Skip to content

Commit 6627306

Browse files
zhenbushidashenzhenbushidashen
committed
mod:抽离请求
1 parent 7befae0 commit 6627306

9 files changed

+16348
-281
lines changed

‎a.out

+4
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
2+
> leetcode-cheat@0.1.0 crawl /Users/zhangzhen/myproject/leetcode-cheat
3+
> node scripts/curlleetcode.js && node scripts/generateleetcode.js
4+

‎package-lock.json

+15,099
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎package.json

+1
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@
3535
]
3636
},
3737
"devDependencies": {
38+
"axios": "^0.19.2",
3839
"cheerio": "^1.0.0-rc.3",
3940
"iconv-lite": "^0.5.1",
4041
"log4js": "^6.3.0",

‎scripts/LeetCodeProvider.js

+23-52
Original file line numberDiff line numberDiff line change
@@ -5,74 +5,45 @@ const Iconv = require('iconv-lite')
55
const cheerio = require('cheerio')
66

77
const Logger = require('./logger')
8+
const Utils = require('./utils')
89
const { PROBLEMS_URL, QUESTION_DOM_SELECTOR, BASE_MARKDWON_DOWNLOAD_URL, ENGLISH_MARKDOWN_SIGN } = require('./constants')
910

10-
const LeetCodeProvider = {
11-
11+
module.exports = LeetCodeProvider = {
1212

1313

1414

1515
getProblemsTitle() {
16-
return new Promise((ok, unExpect) => {
17-
18-
Logger.success('正在获取问题列表...')
19-
20-
request({
21-
method: 'GET',
22-
url: PROBLEMS_URL,
23-
encoding: null
24-
},
25-
(error, response, body) => {
26-
if (error) {
27-
28-
unExpect(Logger.error('获取问题列表失败', error) && error)
29-
30-
} else {
31-
32-
let aProblemTitles = []
33-
34-
let sHtml = Iconv.decode(body, 'utf-8').toString()
35-
36-
cheerio.load(sHtml)(QUESTION_DOM_SELECTOR).each((idx, ele) => aProblemTitles.push(ele.attribs['title']))
37-
38-
Logger.success('获取问题列表成功')
39-
40-
ok(aProblemTitles.filter(name => !name.endsWith(ENGLISH_MARKDOWN_SIGN)))
41-
}
42-
})
43-
16+
17+
return Utils.httpGet(PROBLEMS_URL)
18+
.then((body)=> {
19+
let aProblemTitles = []
20+
let sHtml = Iconv.decode(body, 'utf-8').toString()
21+
cheerio.load(sHtml)(QUESTION_DOM_SELECTOR).each((idx, ele) => aProblemTitles.push(ele.attribs['title']))
22+
Logger.success('获取问题列表成功')
23+
24+
return aProblemTitles.filter(name => !name.endsWith(ENGLISH_MARKDOWN_SIGN))
25+
})
26+
.catch(error => {
27+
Logger.error('获取问题列表失败', error)
4428
})
4529
},
4630

47-
getProblemDetail(problemNameWithExt) {
48-
49-
return new Promise((ok, unExpect) => {
50-
51-
Logger.success('正在抓取问题:', problemNameWithExt)
5231

53-
request({
54-
method: 'GET',
55-
url: `${BASE_MARKDWON_DOWNLOAD_URL}${problemNameWithExt}`,
56-
encoding: null
57-
},
58-
(error, response, body) => {
59-
if (error) {
6032

61-
unExpect(Logger.error(`抓取问题 "${problemNameWithExt}" 失败`, error) && error)
6233

63-
} else {
64-
65-
let markdown = Iconv.decode(body, 'utf-8').toString()
66-
67-
Logger.success(`抓取问题 "${problemNameWithExt}" 成功！`)
34+
getProblemDetail(problemNameWithExt) {
6835

69-
ok(markdown)
70-
}
36+
return Utils.httpGet(`${BASE_MARKDWON_DOWNLOAD_URL}${problemNameWithExt}`)
37+
.then(body => {
7138

72-
})
39+
let markdown = Iconv.decode(body, 'utf-8').toString()
40+
Logger.success(`抓取问题 "${problemNameWithExt}" 成功！`)
41+
return markdown
42+
})
43+
.catch(error => {
44+
Logger.error(`抓取问题 "${problemNameWithExt}" 失败`, error)
7345
})
7446
}
7547

7648
}
7749

78-
module.exports = LeetCodeProvider

‎scripts/constants.js

+2-2
Original file line numberDiff line numberDiff line change
@@ -17,13 +17,13 @@ module.exports = {
1717
* 是否启用强制更新
1818
* 如开启，会跳过读取本地缓存，拉取最新文件
1919
*/
20-
IS_FORCE_UPDATE_MODE: false,
20+
IS_FORCE_UPDATE_MODE: true,
2121

2222
/**
2323
* 请求处理频率 ms
2424
*/
2525
REQUEST_RATE: 300,
26-
26+
2727
/**
2828
* markdown输出目录
2929
*/

‎scripts/curlleetcode.js

+11-6
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ const Logger = require('./logger')
44

55
const Utils = require('./utils')
66

7+
78
const { RAW_MARKDOWN_OUTPUT_DIR, REQUEST_RATE, IS_FORCE_UPDATE_MODE } = require('./constants')
89

910

@@ -32,12 +33,16 @@ const getProblemDetail = (questionsName, requsetNumber) => {
3233
else {
3334

3435
questionsName[requsetNumber] && LeetCodeProvider.getProblemDetail(questionsName[requsetNumber]).then(markDown => {
35-
36-
Logger.success(`问题: "${questionsName[requsetNumber]}" | 结果： ${JSON.stringify(markDown)}`)
37-
38-
Utils.writeFileSync(RAW_MARKDOWN_OUTPUT_DIR, questionsName[requsetNumber], markDown)
39-
40-
requsetNumber++
36+
if (markDown) {
37+
38+
Logger.success(`问题: "${questionsName[requsetNumber]}" | 结果： ${JSON.stringify(markDown)}`)
39+
40+
Utils.writeFileSync(RAW_MARKDOWN_OUTPUT_DIR, questionsName[requsetNumber], markDown)
41+
42+
requsetNumber++
43+
} else {
44+
Logger.error(`获取${questionsName[requsetNumber]} markdown 失败!`)
45+
}
4146

4247
}).catch(Logger.error).then(() => {
4348

‎scripts/generateleetcode.js

+4-4
Original file line numberDiff line numberDiff line change
@@ -12,9 +12,9 @@ const {
1212

1313
const genertateLeetcodeToJson = () => {
1414

15-
const rowMarkdowns = Utils.getDirsFileName(RAW_MARKDOWN_OUTPUT_DIR).filter(name => !name.endsWith(ENGLISH_MARKDOWN_SIGN))
15+
const rawMarkdowns = Utils.getDirsFileName(RAW_MARKDOWN_OUTPUT_DIR).filter(name => !name.endsWith(ENGLISH_MARKDOWN_SIGN))
1616

17-
rowMarkdowns.forEach(filename => {
17+
rawMarkdowns.forEach(filename => {
1818

1919
let languageResloved = []
2020
let preKnowledge = []
@@ -108,15 +108,15 @@ const generateCollectionIndexFile = () => {
108108
Logger.success('开始生产index文件')
109109
const jsonsName = Utils.getDirsFileName(DB_JSON_OUTPUT_DIR)
110110

111-
let sRootContent = `
111+
let rootContent = `
112112
export const db_collection = {
113113
${ jsonsName.reduce((acc, next) => {
114114
return acc + ('"' + next.split('.')[1] + '":' + JSON.stringify(require('../spider/yield-db-json/' + next), null, 4) + ',\n')
115115
}, '')}
116116
}
117117
`
118118

119-
Utils.writeFileSync('src/db', 'root.db.js', sRootContent)
119+
Utils.writeFileSync('src/db', 'root.db.js', rootContent)
120120
Logger.success('index文件生成完毕')
121121

122122
}

‎scripts/utils.js

+27
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,12 @@
11
const fs = require('fs')
2+
const request = require('request')
23
const { resolve: pathResolve } = require('path')
34
const mkdirp = require('mkdirp')
45

56
const PROJECT_ROOT = pathResolve(__dirname, '../')
67

8+
const isFun = input => typeof input === 'function'
9+
710
const Utils = {
811

912

@@ -19,6 +22,30 @@ const Utils = {
1922
return new RegExp(`(?:\`\`\`${lang})((.|[\r\n])*?)(?:\`\`\`)`, 'g')
2023
},
2124

25+
26+
httpGet( url,{ beforeRequest, onSuccess, onError,} = {}) {
27+
return new Promise((ok, unExpect) => {
28+
isFun(beforeRequest) && beforeRequest()
29+
beforeRequest
30+
request({
31+
method:'GET',
32+
url,
33+
encoding:null
34+
}, (error, response, body) => {
35+
if (error) {
36+
37+
unExpect(error)
38+
isFun(onError) && onError()
39+
40+
} else {
41+
42+
isFun(onSuccess) && onSuccess()
43+
ok(body)
44+
}
45+
})
46+
})
47+
},
48+
2249
getSatelliteDataReg() {
2350
return {
2451
pre: /(?<=## 前置知识)([\s\S]*?)(?=##)/g,

0 commit comments

Comments
 (0)