-
Notifications
You must be signed in to change notification settings - Fork 4
/
Copy pathgetdata.js
279 lines (248 loc) · 8.67 KB
/
getdata.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
var $ = require('jQuery'),
nodegrass = require('nodegrass'),
db = require('./lib/db.js'),
db = new db();
/* 7完毕
*parms stdcategoryid1 家居建材= 13:建材, 58:照明工业, 59:五金工具 ,18:户外运动 ,7:数码、电脑 ,96:家纺
312:内衣,
食品农业= 2:食品、饮料
日用百货= 15:日用百货, 17:工艺品、礼品 71:汽摩及配件,67:办公、文教
橡塑橡胶= 55:橡塑
冶金钢材= 9:冶金矿产
化工精细= 56:精细化学品,8:化工
纺织= 4:纺织、皮革
包装= 52:纸业 68:包装
五金机械= 65:机械及行业设备, 72:印刷 ,64:环保 ,12:交通运输
电子电工= 5:电工电气 57:电子元器件 58
照明安防= 10208:仪器仪表,70:安全、防护
医药医疗= 66:医药、保养
stdcategoryid2 10165:男装 ,10166:女装,1043574:,
美妆日化= 1043574:日化用品,1043171:美容、化妆用具 ,1043162:美甲用品,1042634:护肤品 ,82101:彩妆、香水 ,10313:美发造型
1042954:箱包皮具, 1038378:鞋 , 54:服饰配件、饰品 6:家用电器,509:通信产品 ,53:传媒、广电,1813:玩具,1501:母婴用品,311:童装
*/
/*var cid = process.argv[2],
start = process.argv[3],
end = process.argv[4];*/
var start = parseInt(process.argv[2]),
end = parseInt(process.argv[3]);
function getHtml(pageid,cid){
var page = pageid*24+1,
url = 'http://s.hc360.com/company/机械市场.html?e='+page+'&v=4',//慧聪
source = '1688';
// var url = 'http://daili.1688.com/daili/ajax.json?action=list/list_action&event_submit_doQueryFromList=true&pageNum='+pageid+'&pageSize=15&stdcategoryid1='+cid+'&_=1388454529278'
nodegrass.get(url,function(data,status,headers){
// alibabaToDb(data); //阿里巴巴
hc360ToDb(data); //慧聪
return;
},'gbk').on('error', function(e) {
//记录列表错误url
var error = [];
error['url'] = url;
error['type'] = 'list';
error['source'] = source;
matchDb('errorurl',{url:url},function(){
db.sqlInsert('errorurl',error);
})
console.log("Got error: " + e.message);
});
}
//阿里巴巴 代理栏目 规则
function alibabaToDb(data){
var list = eval('('+data+')');
$.each(list.data, function(index, val) {
var companyname = val.companyname;
//入库排重
matchDb('enterprise',{companyname:companyname},function(){
if (companyname) {
var values=[];
values['companyname']= val.companyname;
values['companyintroduction']= val.companyintroduction;
values['stdcategoryid1']= val.stdcategoryid1;
values['tel']= val.tel;
values['winportdomain']= val.winportdomain;
values['brandlogourl']= val.brandlogourl;
values['createdTime']= timestamp();
// db.sqlInsert('enterprise',values);
console.log(result.companyname+'-'+result.tel);
}else{
console.log('null');
}
})
//比对
// db.sqlSelect('enterprise',['id'],{companyname:companyname},function(data){
// // console.log(data.length);
// if (data.length) {
// console.log('存在跳过');
// }else{
// insert(val);
// }
// });
/*var insert = function(result){
if (result.companyname) {
var values=[];
values['companyname']= result.companyname;
values['companyintroduction']= result.companyintroduction;
values['stdcategoryid1']= result.stdcategoryid1;
values['tel']= result.tel;
values['winportdomain']= result.winportdomain;
values['brandlogourl']= result.brandlogourl;
values['createdTime']= timestamp();
// db.sqlInsert('enterprise',values);
console.log(result.companyname+'-'+result.tel);
}else{
console.log('null');
}
}*/
// connection.end();
//filterData(data,pageid);
});
}
//慧聪 企业栏目 规则======================================
function hc360ToDb(data){
var $html = $(data),
company = $html.find('.list_company h3 a');
$.each(company,function(i,item){
var href = $(item).attr('href');
// companyname = $.trim($(item).text());
href = href+'/shop/show.html';
hcCompany(href);
});
process.nextTick(function() {
console.log('=====完成====');
});
}
//慧聪企业详情
function hcCompany(href){
var url = href;
nodegrass.get(url,function(data,status,headers){
var $html = $(data),
companyname = $.trim($html.find('.comName a').text());
if (companyname) {
//入库排重
// matchDb(companyname,function(){
matchDb('enterprise',{companyname:companyname},function(){
// var $html = $(data),
var tableTr = $html.find('.detailsinfo table tr'),
contactbox = $html.find('.contactbox'),
cAbout = $html.find('.cAbout'),
telstr = $(contactbox).find("ul li[title^='手机']").text();
if (!telstr) {
telstr = $(contactbox).find("ul li[title^='电话']").text();
telstr = telstr.split(' ')[0];
}
var tel = telstr.split('：')[1],
nature = $(tableTr).eq(0).find('td').eq(3).text(),
values = [];
values['companyname'] = companyname;
// values['companyintroduction'] = $(cAbout).html()
values['companyintroduction'] = $.trim($(cAbout).text());
values['mainProducts'] = $.trim($(tableTr).eq(0).find('td').eq(1).text());
values['nature'] = $.trim($(tableTr).eq(1).find('td').eq(1).text());
values['businessAddress'] = $.trim($(tableTr).eq(2).find('td').eq(3).text());
values['legalPerson'] = $.trim($(tableTr).eq(3).find('td').eq(3).text());
values['turnover'] = $.trim($(tableTr).eq(4).find('td').eq(3).text());
values['source'] = '1688';
values['founded'] = $.trim($(tableTr).eq(3).find('td').eq(1).text());
values['registeredCapital'] = $.trim($(tableTr).eq(5).find('td').eq(3).text());
values['stdcategoryid1'] = 65;
values['tel'] = $.trim(tel);
values['contact'] = $.trim($html.find('.renName').text());
values['winportdomain'] = href;
var brandlogourl = $(cAbout).find('.cAboutPic #hc_jiaodianmain img').attr('src');
values['brandlogourl'] = (brandlogourl != undefined) ? brandlogourl : null;
values['createdTime'] = timestamp();
if(values['founded'] == '—' && values['legalPerson'] == '—' && values['businessAddress'] == '—'){
console.log('垃圾数据',url);
}else{
db.sqlInsert('enterprise',values);
console.log(companyname);
// window.close();
}
})
}else{
console.log('企业名null,排除'+url);
}
return;
},'gbk').on('error', function(e) {
//记录详情页错误url
var error = [];
error['url'] = url;
error['type'] = 'detail';
error['source'] = 'hc360';
matchDb('errorurl',{url:url},function(){
db.sqlInsert('errorurl',error);
})
console.log("Got error: " + e.message);
});
}
//比对
// function matchDb(companyname,callback){
function matchDb(table,where,callback){
// db.sqlSelect('enterprise',['id'],{companyname:companyname},function(data){
db.sqlSelect(table,['id'],where,function(data){
// console.log(data.length);
if (data.length) {
console.log('存在跳过');
}else{
callback();
}
},0,1);
}
// hcCompany('http://kaimafujian.b2b.hc360.com/shop/show.html');
// hcCompany('http://yhret2012.b2b.hc360.com/','江苏南元机床集团有限公司');
// getHtml(0,1);
// getHtml(2);
function listLoop(start, end){
while (start < end){
getHtml(start,65);
start ++;
}
}
listLoop(start,end);
//删除数据
/*db.sqlDelete('errorurl',{id:2680},function(data){
console.log(data);
})*/
//3360 120 28页
function getErrorDb(start,pageNum){
db.sqlSelect('errorurl',['id','url'],null,function(data){
$.each(data,function(i,item){
console.log(item.url);
hcCompany(item.url);
})
},start,pageNum)
}
// getErrorDb(0,12);
// process.argv.forEach(function(val, index, array) {
// console.log(index + ': ' + val);
// });
// sqlSelect('enterprise',['id'],{companyname:'汉川鑫龙祥圣服装厂'},function(data){
// console.log(data);
// })
//当前时间戳
function timestamp() {
var timestamp = Date.parse(new Date());
return timestamp;
}
//return 2013-12-05 8:30
function CurentTime(){
var now = new Date();
var year = now.getFullYear(); //年
var month = now.getMonth() + 1; //月
var day = now.getDate(); //日
var hh = now.getHours(); //时
var mm = now.getMinutes(); //分
var clock = year + "-";
if(month < 10)
clock += "0";
clock += month + "-";
if(day < 10)
clock += "0";
clock += day + " ";
if(hh < 10)
clock += "0";
clock += hh + ":";
if (mm < 10) clock += '0';
clock += mm;
return(clock);
}