1- "use strict" ;
1+ const cheerio = require ( 'cheerio' ) ;
2+ const async = require ( 'async' ) ;
3+ const rp = require ( 'request-promise' ) ;
4+ const sqlConn = require ( '../../resource/mysqlConnection' ) ;
5+ const globalConfig = require ( '../../config/env' ) ;
26
3- const cheerio = require ( 'cheerio' ) ,
4- async = require ( 'async' ) ,
5- fs = require ( 'fs' ) ,
6- rp = require ( 'request-promise' ) ,
7- sqlConn = require ( '../../resource/mysqlConnection' ) ,
8- globalConfig = require ( '../../config/env' ) ,
9-
10- CONCURRENCY = 10 ,
11- TIMEOUT = 5 * 1000 ,
12- JOBFLAG = '_JOBDONE' ;
7+ const CONCURRENCY = 10 ;
8+ const TIMEOUT = 5 * 1000 ;
9+ const JOBFLAG = '_JOBDONE' ;
1310
1411let finished = null ;
1512const privateFn = {
16- checkNameExist : ( v ) => {
17- if ( v . name . length === 0 ) {
18- let gitList = v . github . split ( '/' ) ;
19- v . name = gitList [ 1 ] ;
20- }
21- } ,
22-
2313 promiseLoop : ( data0 , resolve , reject ) => {
2414 console . log ( `downloading ${ data0 . length } packages...` ) ;
2515 async . mapLimit ( data0 , CONCURRENCY , ( v , cb ) => {
26- privateFn . checkNameExist ( v ) ;
16+ if ( v . name . length === 0 ) {
17+ const gitList = v . github . split ( '/' ) ;
18+ v . name = gitList [ 1 ] ;
19+ }
2720 privateFn . getNum ( v , cb ) ;
2821 } , ( error , data ) => {
29- let unfinished = data . filter ( v => ! v [ JOBFLAG ] ) ;
22+ const unfinished = data . filter ( v => ! v [ JOBFLAG ] ) ;
3023 finished = finished . concat ( data . filter ( v => v [ JOBFLAG ] ) ) ;
31- if ( unfinished . length > 0 ) {
24+ if ( unfinished . length > 0 ) {
3225 return privateFn . promiseLoop ( unfinished , resolve , reject ) ;
33- } else {
34- let result = finished . filter ( v => v [ JOBFLAG ] ) ;
35- resolve ( result ) ;
3626 }
27+ const result = finished . filter ( v => v [ JOBFLAG ] ) ;
28+ resolve ( result ) ;
29+ return 0 ;
3730 } ) ;
3831 } ,
3932
40- getNum : ( obj , cb ) => {
41- //console.log('download ' + obj.name );
33+ getNum : ( obj0 , cb ) => {
34+ const obj = Object . assign ( { } , obj0 ) ;
4235 rp ( {
43- uri : ' https://github.com/' + obj . github ,
36+ uri : ` https://github.com/${ obj . github } ` ,
4437 timeout : TIMEOUT ,
45- transform : function ( body ) {
46- return cheerio . load ( body ) ;
47- }
38+ transform : body => cheerio . load ( body ) ,
4839 } )
49- . then ( function ( $ ) {
50- let elems = $ ( '.social-count.js-social-count' ) ;
51- let starElem = elems [ 0 ] ;
52- let num = starElem . children [ 0 ] . data . trim ( ) ;
53- obj [ ' star' ] = num ;
40+ . then ( ( $ ) => {
41+ const elems = $ ( '.social-count.js-social-count' ) ;
42+ const starElem = elems [ 0 ] ;
43+ const num = starElem . children [ 0 ] . data . trim ( ) ;
44+ obj . star = num ;
5445 obj [ JOBFLAG ] = true ;
5546 cb ( null , obj ) ;
5647 } )
57- . catch ( function ( err ) {
58- console . error ( ' crawler timeout on ' + obj . name ) ;
48+ . catch ( ( ) => {
49+ console . error ( ` crawler timeout on ${ obj . name } ` ) ;
5950 cb ( null , obj ) ;
6051 } ) ;
6152 } ,
@@ -65,19 +56,19 @@ const publicFn = {
6556 start : ( ) => {
6657 finished = [ ] ;
6758 const deferred = new Promise ( ( resolve , reject ) => {
68- var qr = " SELECT * FROM git WHERE `group` IS NOT NULL" ;
69- if ( globalConfig . isDev ) {
70- qr = " SELECT * FROM git WHERE `group` IS NOT NULL AND id > 419" ;
59+ let qr = ' SELECT * FROM git WHERE `group` IS NOT NULL' ;
60+ if ( globalConfig . isDev ) {
61+ qr = ' SELECT * FROM git WHERE `group` IS NOT NULL AND id > 419' ;
7162 }
7263 sqlConn . sqlExecOne ( qr ) . then ( ( db ) => {
73- privateFn . promiseLoop ( db , resolve , reject )
74- } , ( err ) => {
64+ privateFn . promiseLoop ( db , resolve , reject ) ;
65+ } , ( ) => {
7566 console . error ( 'sql executed fails1.' ) ;
7667 } ) ;
7768 } ) ;
7869
7970 return deferred ;
80- }
71+ } ,
8172} ;
8273
8374module . exports = publicFn ;
0 commit comments