Skip to content

Commit 793f87f

Browse files
kefniarkkefniark
authored
Merge pull request #15 from komodojp/deno
Fix deno issue, Node ESM import and few issues
2 parents 251b89f + 9c54543 commit 793f87f

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

43 files changed

+33029
-34099
lines changed

‎.eslintignore

+1
Original file line numberDiff line numberDiff line change
@@ -2,3 +2,4 @@ node_modules
22
bin
33
dist
44
tests
5+
tests-integration

‎.github/workflows/main.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ jobs:
1414
- name: Use Node.js
1515
uses: actions/setup-node@v2
1616
with:
17-
node-version: '14.x'
17+
node-version: '18.x'
1818
- run: |
1919
yarn
2020
yarn build

‎.prettierignore

+1
Original file line numberDiff line numberDiff line change
@@ -9,3 +9,4 @@ config
99
*.log
1010
package.json
1111
package-lock.json
12+
tests-integration

‎Readme.md

+1-1
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414
- Pure JS, No api call, No dependencies (Node and Browser compatible)
1515
- Blazing fast and low memory footprint (unlike ML methods)
1616
- Train with dataset from [Tatoeba](https://tatoeba.org/en/) and [UDHR](https://unicode.org/udhr/)
17-
- Support [64 languages](./docs/langs.md) (24 for [the web version](./docs/light.md))
17+
- Support [62 languages](./docs/langs.md) (24 for [the web version](./docs/light.md))
1818
- Reliable even for really short texts (chatbot, keywords, ...)
1919
- Support both ISO-639-1 & ISO-639-2
2020
- Available for both `CommonJS` and `ESM`

‎bin/tinyld.js

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
#! /usr/bin/env node
2-
const { detectAll } = require('../dist/tinyld.cjs')
2+
const { detectAll } = require('../dist/tinyld.normal.node.js')
33

44
function main() {
55
const [, , ...args] = process.argv

‎data/bench/cld.json

+16-16
Original file line numberDiff line numberDiff line change
@@ -4,64 +4,64 @@
44
"success_rate": 36.375,
55
"error_rate": 3.25,
66
"unindentified_rate": 60.375,
7-
"execution_time": 0.076
7+
"execution_time": 0.0794
88
},
99
"24": {
1010
"success_rate": 77.7188,
1111
"error_rate": 3.0313,
1212
"unindentified_rate": 19.25,
13-
"execution_time": 0.0566
13+
"execution_time": 0.0572
1414
},
1515
"36": {
1616
"success_rate": 92.8125,
1717
"error_rate": 1.75,
1818
"unindentified_rate": 5.4375,
19-
"execution_time": 0.057
19+
"execution_time": 0.0577
2020
},
2121
"48": {
2222
"success_rate": 96.75,
2323
"error_rate": 1.0938,
2424
"unindentified_rate": 2.1563,
25-
"execution_time": 0.0582
25+
"execution_time": 0.0579
2626
},
2727
"64": {
28-
"success_rate": 98.2813,
28+
"success_rate": 98.3125,
2929
"error_rate": 0.5,
30-
"unindentified_rate": 1.2188,
31-
"execution_time": 0.0594
30+
"unindentified_rate": 1.1875,
31+
"execution_time": 0.0586
3232
},
3333
"128": {
3434
"success_rate": 99.5625,
3535
"error_rate": 0.0938,
3636
"unindentified_rate": 0.3438,
37-
"execution_time": 0.0608
37+
"execution_time": 0.0629
3838
},
3939
"256": {
4040
"success_rate": 99.9375,
4141
"error_rate": 0.0313,
4242
"unindentified_rate": 0.0313,
43-
"execution_time": 0.0672
43+
"execution_time": 0.0688
4444
},
4545
"512": {
4646
"success_rate": 99.9688,
4747
"error_rate": 0,
4848
"unindentified_rate": 0.0313,
49-
"execution_time": 0.0797
49+
"execution_time": 0.0827
5050
},
5151
"1024": {
5252
"success_rate": 100,
5353
"error_rate": 0,
5454
"unindentified_rate": 0,
55-
"execution_time": 0.1063
55+
"execution_time": 0.1096
5656
}
5757
},
5858
"stats": {
5959
"min": 78.39,
6060
"max": 100,
61-
"success_rate": 92.3097,
62-
"error_rate": 1.6328,
63-
"unindentified_rate": 6.0574,
64-
"execution_time": 0.0566
61+
"success_rate": 92.315,
62+
"error_rate": 1.6319,
63+
"unindentified_rate": 6.0532,
64+
"execution_time": 0.0565
6565
},
6666
"languages": {
6767
"jpn": 100,
@@ -78,7 +78,7 @@
7878
"nld": 89.58,
7979
"spa": 87.79,
8080
"ita": 85.9,
81-
"heb": 84.05,
81+
"heb": 84.06,
8282
"ara": 78.39
8383
}
8484
}

‎data/bench/franc-all.json

+28-28
Original file line numberDiff line numberDiff line change
@@ -4,78 +4,78 @@
44
"success_rate": 18.5313,
55
"error_rate": 35,
66
"unindentified_rate": 46.4688,
7-
"execution_time": 0.0918
7+
"execution_time": 0.088
88
},
99
"24": {
1010
"success_rate": 48.4375,
1111
"error_rate": 51.5625,
1212
"unindentified_rate": 0,
13-
"execution_time": 0.2222
13+
"execution_time": 0.215
1414
},
1515
"36": {
16-
"success_rate": 60.625,
17-
"error_rate": 39.375,
16+
"success_rate": 60.5938,
17+
"error_rate": 39.4063,
1818
"unindentified_rate": 0,
19-
"execution_time": 0.2974
19+
"execution_time": 0.2977
2020
},
2121
"48": {
22-
"success_rate": 69.5938,
23-
"error_rate": 30.4063,
22+
"success_rate": 69.5625,
23+
"error_rate": 30.4375,
2424
"unindentified_rate": 0,
25-
"execution_time": 0.3688
25+
"execution_time": 0.3563
2626
},
2727
"64": {
28-
"success_rate": 73.5625,
29-
"error_rate": 26.4375,
28+
"success_rate": 73.5938,
29+
"error_rate": 26.4062,
3030
"unindentified_rate": 0,
31-
"execution_time": 0.4389
31+
"execution_time": 0.4264
3232
},
3333
"128": {
34-
"success_rate": 86.0938,
35-
"error_rate": 13.9063,
34+
"success_rate": 86.1563,
35+
"error_rate": 13.8438,
3636
"unindentified_rate": 0,
37-
"execution_time": 0.6776
37+
"execution_time": 0.6821
3838
},
3939
"256": {
4040
"success_rate": 93.6875,
4141
"error_rate": 6.3125,
4242
"unindentified_rate": 0,
43-
"execution_time": 1.2104
43+
"execution_time": 1.1989
4444
},
4545
"512": {
46-
"success_rate": 96.3125,
47-
"error_rate": 3.6875,
46+
"success_rate": 96.25,
47+
"error_rate": 3.75,
4848
"unindentified_rate": 0,
49-
"execution_time": 2.0938
49+
"execution_time": 2.0734
5050
},
5151
"1024": {
52-
"success_rate": 98.381,
53-
"error_rate": 1.619,
52+
"success_rate": 98.4147,
53+
"error_rate": 1.5853,
5454
"unindentified_rate": 0,
55-
"execution_time": 3.57
55+
"execution_time": 3.5626
5656
}
5757
},
5858
"stats": {
5959
"min": 29.47,
6060
"max": 99.93,
61-
"success_rate": 66.4673,
62-
"error_rate": 33.5327,
61+
"success_rate": 66.4893,
62+
"error_rate": 33.5107,
6363
"unindentified_rate": 0,
64-
"execution_time": 0.3525
64+
"execution_time": 0.3376
6565
},
6666
"languages": {
6767
"jpn": 99.93,
68-
"kor": 99.8261,
68+
"kor": 99.83,
6969
"cmn": 99.35,
7070
"heb": 98.18,
7171
"ara": 91.73,
7272
"deu": 80.77,
7373
"fin": 70.8,
7474
"fra": 67.27,
75-
"hin": 60.4,
76-
"nld": 59.65,
75+
"hin": 60.42,
76+
"nld": 59.64,
7777
"rus": 51.94,
78-
"eng": 49.88,
78+
"eng": 49.9,
7979
"por": 49.39,
8080
"ita": 42.55,
8181
"tur": 32.27,

‎data/bench/franc-min.json

+29-29
Original file line numberDiff line numberDiff line change
@@ -4,80 +4,80 @@
44
"success_rate": 21.4375,
55
"error_rate": 29.375,
66
"unindentified_rate": 49.1875,
7-
"execution_time": 0.0184
7+
"execution_time": 0.0171
88
},
99
"24": {
10-
"success_rate": 53.25,
11-
"error_rate": 40.5,
10+
"success_rate": 53.2188,
11+
"error_rate": 40.5313,
1212
"unindentified_rate": 6.25,
13-
"execution_time": 0.0369
13+
"execution_time": 0.0387
1414
},
1515
"36": {
1616
"success_rate": 66.9375,
1717
"error_rate": 26.8125,
1818
"unindentified_rate": 6.25,
19-
"execution_time": 0.0503
19+
"execution_time": 0.0525
2020
},
2121
"48": {
22-
"success_rate": 73.4063,
23-
"error_rate": 20.3438,
22+
"success_rate": 73.375,
23+
"error_rate": 20.375,
2424
"unindentified_rate": 6.25,
25-
"execution_time": 0.0612
25+
"execution_time": 0.0644
2626
},
2727
"64": {
2828
"success_rate": 75.9063,
2929
"error_rate": 17.875,
3030
"unindentified_rate": 6.2188,
31-
"execution_time": 0.0755
31+
"execution_time": 0.078
3232
},
3333
"128": {
34-
"success_rate": 82.4688,
35-
"error_rate": 11.3438,
34+
"success_rate": 82.5625,
35+
"error_rate": 11.25,
3636
"unindentified_rate": 6.1875,
37-
"execution_time": 0.1183
37+
"execution_time": 0.1275
3838
},
3939
"256": {
4040
"success_rate": 85.1563,
4141
"error_rate": 8.6875,
4242
"unindentified_rate": 6.1563,
43-
"execution_time": 0.2252
43+
"execution_time": 0.2363
4444
},
4545
"512": {
46-
"success_rate": 86.5625,
47-
"error_rate": 7.4063,
46+
"success_rate": 86.5313,
47+
"error_rate": 7.4375,
4848
"unindentified_rate": 6.0313,
49-
"execution_time": 0.4095
49+
"execution_time": 0.4234
5050
},
5151
"1024": {
52-
"success_rate": 86.9841,
53-
"error_rate": 7.0159,
54-
"unindentified_rate": 6,
55-
"execution_time": 0.738
52+
"success_rate": 87.0006,
53+
"error_rate": 7.007,
54+
"unindentified_rate": 5.9924,
55+
"execution_time": 0.7489
5656
}
5757
},
5858
"stats": {
5959
"min": 0.01,
6060
"max": 99.93,
61-
"success_rate": 70.1739,
62-
"error_rate": 23.3584,
63-
"unindentified_rate": 6.4677,
64-
"execution_time": 0.0681
61+
"success_rate": 70.193,
62+
"error_rate": 23.3432,
63+
"unindentified_rate": 6.4639,
64+
"execution_time": 0.0668
6565
},
6666
"languages": {
6767
"jpn": 99.93,
68-
"kor": 99.8261,
68+
"kor": 99.83,
6969
"cmn": 99.35,
7070
"deu": 94.18,
7171
"ara": 91.89,
7272
"fra": 87.32,
7373
"nld": 87.21,
74-
"eng": 81.38,
74+
"eng": 81.39,
7575
"por": 76.8,
7676
"ita": 74.31,
77-
"hin": 68.22,
78-
"spa": 67.38,
77+
"hin": 68.25,
78+
"spa": 67.37,
7979
"tur": 58.11,
80-
"rus": 54.59,
80+
"rus": 54.58,
8181
"fin": 0.01,
8282
"heb": 0.01
8383
}

0 commit comments

Comments
 (0)