Skip to content

Commit a6d05c3

Browse files
zhenbushidashenzhenbushidashen
committed
mod:抽取常量&去除静态类
1 parent 77d6457 commit a6d05c3

12 files changed

+919
-15809
lines changed

‎package-lock.json

-15,054
This file was deleted.

‎package.json

+3-1
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,8 @@
77
"@testing-library/react": "^9.3.2",
88
"@testing-library/user-event": "^7.1.2",
99
"antd": "^4.3.1",
10+
"highlight.js": "^10.0.3",
11+
"marked": "^1.1.0",
1012
"react": "^16.13.1",
1113
"react-dom": "^16.13.1",
1214
"react-markdown": "^4.3.1",
@@ -17,7 +19,7 @@
1719
"build": "react-scripts build",
1820
"test": "react-scripts test",
1921
"eject": "react-scripts eject",
20-
"crawl": "node scripts/curl-leetcode.js && node scripts/generate-leetcode.js"
22+
"crawl": "node scripts/curlleetcode.js && node scripts/generateleetcode.js"
2123
},
2224
"eslintConfig": {
2325
"extends": "react-app"

‎scripts/LeetCodeProvider.js

+53-41
Original file line numberDiff line numberDiff line change
@@ -3,63 +3,75 @@
33
const request = require('request')
44
const Iconv = require('iconv-lite')
55
const cheerio = require('cheerio')
6-
const Logger = require('./Logger')
76

7+
const Logger = require('./logger')
8+
const { PROBLEMS_URL, QUESTION_DOM_SELECTOR, BASE_MARKDWON_DOWNLOAD_URL, ENGLISH_MARKDOWN_SIGN } = require('./constants')
9+
10+
const LeetCodeProvider = {
811

912

10-
class LeetCodeProvider {
11-
1213

13-
static getInstance() {
14-
return this.oInstance || (this.oInstance = new LeetCodeProvider)
15-
}
1614

1715
getProblemsTitle() {
18-
return new Promise((ok,unExpect) => {
16+
return new Promise((ok, unExpect) => {
1917

2018
Logger.success('正在获取问题列表...')
2119

2220
request({
23-
method:'GET',
24-
url:'https://github.com/azl397985856/leetcode/tree/master/problems',
25-
encoding:null},
26-
(error,response,body) => {
27-
if(error) {
28-
unExpect(Logger.error('获取问题列表失败',error) && error)
29-
} else {
30-
let aProblemTitles =[]
31-
let sHtml = Iconv.decode(body, 'utf-8').toString()
32-
cheerio.load(sHtml)('.js-navigation-item .content .js-navigation-open ').each((idx,ele)=> aProblemTitles.push(ele.attribs['title']) )
33-
Logger.success('获取问题列表成功')
34-
ok(aProblemTitles.filter(name => !name.endsWith('.en.md')))
35-
}
36-
})
21+
method: 'GET',
22+
url: PROBLEMS_URL,
23+
encoding: null
24+
},
25+
(error, response, body) => {
26+
if (error) {
27+
28+
unExpect(Logger.error('获取问题列表失败', error) && error)
29+
30+
} else {
31+
32+
let aProblemTitles = []
33+
34+
let sHtml = Iconv.decode(body, 'utf-8').toString()
35+
36+
cheerio.load(sHtml)(QUESTION_DOM_SELECTOR).each((idx, ele) => aProblemTitles.push(ele.attribs['title']))
37+
38+
Logger.success('获取问题列表成功')
39+
40+
ok(aProblemTitles.filter(name => !name.endsWith(ENGLISH_MARKDOWN_SIGN)))
41+
}
42+
})
3743

3844
})
39-
}
45+
},
4046

4147
getProblemDetail(problemNameWithExt) {
48+
4249
return new Promise((ok, unExpect) => {
4350

44-
Logger.success('正在抓取问题:', problemNameWithExt)
45-
46-
request({
47-
method:'GET',
48-
url:`https://raw.githubusercontent.com/azl397985856/leetcode/master/problems/${problemNameWithExt}`,
49-
encoding:null
50-
},
51-
(error,response,body) => {
52-
if (error) {
53-
unExpect(Logger.error(`抓取问题 "${problemNameWithExt}" 失败`,error) && error)
54-
} else {
55-
let markdown = Iconv.decode(body, 'utf-8').toString()
56-
Logger.success(`抓取问题 "${problemNameWithExt}" 成功！`)
57-
ok(markdown)
58-
}
59-
60-
})
61-
})
62-
}
51+
Logger.success('正在抓取问题:', problemNameWithExt)
52+
53+
request({
54+
method: 'GET',
55+
url: `${BASE_MARKDWON_DOWNLOAD_URL}${problemNameWithExt}`,
56+
encoding: null
57+
},
58+
(error, response, body) => {
59+
if (error) {
60+
61+
unExpect(Logger.error(`抓取问题 "${problemNameWithExt}" 失败`, error) && error)
62+
63+
} else {
64+
65+
let markdown = Iconv.decode(body, 'utf-8').toString()
66+
67+
Logger.success(`抓取问题 "${problemNameWithExt}" 成功！`)
68+
69+
ok(markdown)
70+
}
71+
72+
})
73+
})
74+
}
6375

6476
}
6577

‎scripts/Logger.js

+4-7
Original file line numberDiff line numberDiff line change
@@ -7,15 +7,12 @@ logger.level = 'debug'
77

88
logger.category = 'LeetCode'
99

10-
class Logger {
10+
const Logger = {
1111

12-
13-
14-
static success(...args) {
12+
success(...args) {
1513
logger.info(...args)
16-
}
17-
18-
static error(...args) {
14+
},
15+
error(...args) {
1916
logger.error(...args)
2017
}
2118

‎scripts/constants.js

+39-6
Original file line numberDiff line numberDiff line change
@@ -1,24 +1,57 @@
11

22

33

4-
module.exports = {
4+
module.exports = {
5+
56
/**
67
* 需解析的语言类型
78
*/
8-
99
SUPPORT_LANGUAGE: [
1010
'java',
11-
'python',
12-
'js'
11+
'js',
12+
'cpp',
13+
'py'
1314
],
15+
16+
/**
17+
* 是否启用强制更新
18+
* 如开启，会跳过读取本地缓存，拉取最新文件
19+
*/
20+
IS_FORCE_UPDATE_MODE: false,
21+
22+
/**
23+
* 请求处理频率 ms
24+
*/
25+
REQUEST_RATE: 300,
26+
1427
/**
1528
* markdown输出目录
1629
*/
17-
RAW_MARKDOWN_OUTPUT_DIR: 'spider/raw-markdown',
30+
RAW_MARKDOWN_OUTPUT_DIR: 'spider/raw-markdown',
1831

1932
/**
2033
* 转化后的json输出目录
2134
*/
22-
DB_JSON_OUTPUT_DIR: 'spider/yield-db-json'
35+
DB_JSON_OUTPUT_DIR: 'spider/yield-db-json',
36+
37+
/**
38+
* 获取问题列表地址
39+
*/
40+
PROBLEMS_URL: 'https://github.com/azl397985856/leetcode/tree/master/problems',
41+
42+
/**
43+
* 抓取页面问题内容的dom元素选择器
44+
*/
45+
QUESTION_DOM_SELECTOR: '.js-navigation-item .content .js-navigation-open',
46+
47+
/**
48+
* markdwon下载地址
49+
*/
50+
BASE_MARKDWON_DOWNLOAD_URL: 'https://raw.githubusercontent.com/azl397985856/leetcode/master/problems/',
51+
52+
/**
53+
* 过滤英文文档末尾标识
54+
*/
55+
ENGLISH_MARKDOWN_SIGN: '.en.md'
2356

2457
}

‎scripts/curl-leetcode.js

-79
This file was deleted.

‎scripts/curlleetcode.js

+63
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
const LeetCodeProvider = require('./leetcodeprovider')
2+
3+
const Logger = require('./logger')
4+
5+
const Utils = require('./utils')
6+
7+
const { RAW_MARKDOWN_OUTPUT_DIR, REQUEST_RATE, IS_FORCE_UPDATE_MODE } = require('./constants')
8+
9+
10+
/**
11+
* 当前请求问题索引
12+
*/
13+
let requsetNumber = 0
14+
15+
16+
Utils.mkdirSync(RAW_MARKDOWN_OUTPUT_DIR)
17+
18+
const getProblemDetail = (questionsName, requsetNumber) => {
19+
20+
const cachedFilesName = Utils.getDirsFileName(RAW_MARKDOWN_OUTPUT_DIR)
21+
22+
if (!IS_FORCE_UPDATE_MODE && cachedFilesName.includes(questionsName[requsetNumber])) {
23+
24+
Logger.success(`${questionsName[requsetNumber]}命中缓存， 跳过。。。`)
25+
26+
requsetNumber++
27+
28+
29+
getProblemDetail(questionsName, requsetNumber)
30+
31+
}
32+
else {
33+
34+
questionsName[requsetNumber] && LeetCodeProvider.getProblemDetail(questionsName[requsetNumber]).then(markDown => {
35+
36+
Logger.success(`问题: "${questionsName[requsetNumber]}" | 结果： ${JSON.stringify(markDown)}`)
37+
38+
Utils.writeFileSync(RAW_MARKDOWN_OUTPUT_DIR, questionsName[requsetNumber], markDown)
39+
40+
requsetNumber++
41+
42+
}).catch(Logger.error).then(() => {
43+
44+
setTimeout(() => {
45+
46+
questionsName[requsetNumber] && getProblemDetail(questionsName, requsetNumber)
47+
48+
}, REQUEST_RATE)
49+
})
50+
}
51+
52+
}
53+
54+
55+
LeetCodeProvider.getProblemsTitle().then(questionsName => {
56+
57+
getProblemDetail(questionsName, requsetNumber)
58+
59+
})
60+
61+
62+
63+

0 commit comments

Comments
 (0)