var Spider = require('node-spider');
var mysql = require('mysql');
var conn = mysql.createConnection({
    host    :'localhost',
    port : 3306,
    user : 'DB아이디',
    password : '비번',
    database:'데이터베이스'
});
conn.connect();
 
 
 
if(!String.prototype.trim) {
  String.prototype.trim = function () {
    return this.replace(/^[\s\uFEFF\xA0]+|[\s\uFEFF\xA0]+$/g, '');
  };
}
function addslashes(string) {
    return string.replace(/\\/g, '\\\\').
        replace(/\u0008/g, '\\b').
        replace(/\t/g, '\\t').
        replace(/\n/g, '\\n').
        replace(/\f/g, '\\f').
        replace(/\r/g, '\\r').
        replace(/'/g, '\\\'').
        replace(/"/g, '\\"');
}
 
var spider = new Spider({
    // How many requests can be run in parallel
    concurrent: 5,
    // How long to wait after each request
    delay: 0,
    // A stream to where internal logs are sent, optional
    logs: process.stderr,
    // Re-visit visited URLs, false by default
    allowDuplicates: false,
    // If `true` all queued handlers will be try-catch'd, errors go to `error` callback
    catchErrors: true,
    // Called when there's an error, throw will be used if none is provided
    error: function(err, url) {
 
		console.log(err);
    },
    // Called when there are no more requests
    done: function() {
    },
 
    //- All options are passed to `request` module, for example:
    headers: { 'user-agent': 'node-spider' },
    //encoding: 'utf8'
	encoding: null
});
 
var handleRequest = function(doc) {
 
	// doc.res
	var content = doc.$('div.con_inner').text().trim();
	var title = doc.$('#head_title').text().trim();
	var contentHTML = doc.$('div.con_inner').html();
	var member =  doc.$('li#info_name').find('span.member').text().trim();
	var PageUrl = doc.url;
	if(title) title = addslashes(title);
	if(member) member = addslashes(member);
	if(content) content = addslashes(content);
	if(contentHTML) contentHTML = addslashes(contentHTML);
	if(content) {
 
		// new page crawled
		console.log('---시작---');
		console.log(PageUrl); // page url
		conn.query("insert into `_sc` set `member` = '"+member+"', `title` = '"+title+"', `content` = '"+content+"', `html_content` = '"+contentHTML+"', `url` = '"+PageUrl+"', `rdate` = now()");
		console.log('---종료---');
	}
 
    // uses cheerio, check its docs for more info
    doc.$('a').each(function(i, elem) {
 
        // do stuff with element
        var href = elem.attribs.href.split('#')[0];
        var url = doc.resolve(href);
 
		if(url.indexOf('/cm_free/') > -1) { // 자게 주소 패턴이 있을경우만..
 
			// crawl more
			spider.queue(url, handleRequest);
		}
    });
};
 
// start crawling
spider.queue('http://sir.kr/cm_free', handleRequest);