Add scripts to generate types declaration as part of the build process

Author: kefniark

.npmignore

@@ -3,3 +3,5 @@
 data
 docs
 tests
+src
+utils

Readme.md

@@ -80,3 +80,28 @@ To summary in one sentence:
 > Better, Faster, Smaller
 
 [More Benchmark Information](./docs/benchmark.md)
+
+--
+
+## Developer
+
+```sh
+# Install
+yarn
+
+# Build
+yarn build
+
+# Test
+yarn test
+
+# Lint / Auto-fix code style problems
+yarn lint
+
+# Optional, used to generate src/profiles/* data from language dataset
+# Warning: This step is time consuming and require to install big datasets (described in ./docs/dev.md)
+yarn train
+
+# Optional, used to generate benchmark data/bench/*
+yarn bench
+```

data/bench/cld.json

@@ -4,19 +4,19 @@
       "success_rate": 36.375,
       "error_rate": 3.25,
       "unindentified_rate": 60.375,
-      "execution_time": 0.0706
+      "execution_time": 0.076
     },
     "24": {
       "success_rate": 77.7188,
       "error_rate": 3.0313,
       "unindentified_rate": 19.25,
-      "execution_time": 0.0548
+      "execution_time": 0.0566
     },
     "36": {
       "success_rate": 92.8125,
       "error_rate": 1.75,
       "unindentified_rate": 5.4375,
-      "execution_time": 0.0545
+      "execution_time": 0.057
     },
     "48": {
       "success_rate": 96.75,
@@ -28,31 +28,31 @@
       "success_rate": 98.2813,
       "error_rate": 0.5,
       "unindentified_rate": 1.2188,
-      "execution_time": 0.0566
+      "execution_time": 0.0594
     },
     "128": {
       "success_rate": 99.5625,
       "error_rate": 0.0938,
       "unindentified_rate": 0.3438,
-      "execution_time": 0.0595
+      "execution_time": 0.0608
     },
     "256": {
       "success_rate": 99.9375,
       "error_rate": 0.0313,
       "unindentified_rate": 0.0313,
-      "execution_time": 0.0662
+      "execution_time": 0.0672
     },
     "512": {
       "success_rate": 99.9688,
       "error_rate": 0,
       "unindentified_rate": 0.0313,
-      "execution_time": 0.0799
+      "execution_time": 0.0797
     },
     "1024": {
       "success_rate": 100,
       "error_rate": 0,
       "unindentified_rate": 0,
-      "execution_time": 0.1108
+      "execution_time": 0.1063
     }
   },
   "stats": {
@@ -61,7 +61,7 @@
     "success_rate": 92.3097,
     "error_rate": 1.6328,
     "unindentified_rate": 6.0574,
-    "execution_time": 0.0543
+    "execution_time": 0.0566
   },
   "languages": {
     "jpn": 100,

data/bench/franc-all.json

@@ -4,55 +4,55 @@
       "success_rate": 18.5313,
       "error_rate": 35,
       "unindentified_rate": 46.4688,
-      "execution_time": 0.104
+      "execution_time": 0.0918
     },
     "24": {
       "success_rate": 48.4375,
       "error_rate": 51.5625,
       "unindentified_rate": 0,
-      "execution_time": 0.2488
+      "execution_time": 0.2222
     },
     "36": {
       "success_rate": 60.625,
       "error_rate": 39.375,
       "unindentified_rate": 0,
-      "execution_time": 0.3291
+      "execution_time": 0.2974
     },
     "48": {
       "success_rate": 69.5938,
       "error_rate": 30.4063,
       "unindentified_rate": 0,
-      "execution_time": 0.4012
+      "execution_time": 0.3688
     },
     "64": {
       "success_rate": 73.5625,
       "error_rate": 26.4375,
       "unindentified_rate": 0,
-      "execution_time": 0.473
+      "execution_time": 0.4389
     },
     "128": {
       "success_rate": 86.0938,
       "error_rate": 13.9063,
       "unindentified_rate": 0,
-      "execution_time": 0.7312
+      "execution_time": 0.6776
     },
     "256": {
       "success_rate": 93.6875,
       "error_rate": 6.3125,
       "unindentified_rate": 0,
-      "execution_time": 1.2803
+      "execution_time": 1.2104
     },
     "512": {
       "success_rate": 96.3125,
       "error_rate": 3.6875,
       "unindentified_rate": 0,
-      "execution_time": 2.2039
+      "execution_time": 2.0938
     },
     "1024": {
       "success_rate": 98.381,
       "error_rate": 1.619,
       "unindentified_rate": 0,
-      "execution_time": 3.7052
+      "execution_time": 3.57
     }
   },
   "stats": {
@@ -61,7 +61,7 @@
     "success_rate": 66.4673,
     "error_rate": 33.5327,
     "unindentified_rate": 0,
-    "execution_time": 0.3554
+    "execution_time": 0.3525
   },
   "languages": {
     "jpn": 99.93,

data/bench/franc-min.json

@@ -4,55 +4,55 @@
       "success_rate": 21.4375,
       "error_rate": 29.375,
       "unindentified_rate": 49.1875,
-      "execution_time": 0.0187
+      "execution_time": 0.0184
     },
     "24": {
       "success_rate": 53.25,
       "error_rate": 40.5,
       "unindentified_rate": 6.25,
-      "execution_time": 0.0368
+      "execution_time": 0.0369
     },
     "36": {
       "success_rate": 66.9375,
       "error_rate": 26.8125,
       "unindentified_rate": 6.25,
-      "execution_time": 0.0513
+      "execution_time": 0.0503
     },
     "48": {
       "success_rate": 73.4063,
       "error_rate": 20.3438,
       "unindentified_rate": 6.25,
-      "execution_time": 0.0628
+      "execution_time": 0.0612
     },
     "64": {
       "success_rate": 75.9063,
       "error_rate": 17.875,
       "unindentified_rate": 6.2188,
-      "execution_time": 0.0753
+      "execution_time": 0.0755
     },
     "128": {
       "success_rate": 82.4688,
       "error_rate": 11.3438,
       "unindentified_rate": 6.1875,
-      "execution_time": 0.1207
+      "execution_time": 0.1183
     },
     "256": {
       "success_rate": 85.1563,
       "error_rate": 8.6875,
       "unindentified_rate": 6.1563,
-      "execution_time": 0.2316
+      "execution_time": 0.2252
     },
     "512": {
       "success_rate": 86.5625,
       "error_rate": 7.4063,
       "unindentified_rate": 6.0313,
-      "execution_time": 0.4174
+      "execution_time": 0.4095
     },
     "1024": {
       "success_rate": 86.9841,
       "error_rate": 7.0159,
       "unindentified_rate": 6,
-      "execution_time": 0.7425
+      "execution_time": 0.738
     }
   },
   "stats": {
@@ -61,7 +61,7 @@
     "success_rate": 70.1739,
     "error_rate": 23.3584,
     "unindentified_rate": 6.4677,
-    "execution_time": 0.0657
+    "execution_time": 0.0681
   },
   "languages": {
     "jpn": 99.93,

data/bench/franc.json

@@ -4,55 +4,55 @@
       "success_rate": 20.6875,
       "error_rate": 32.8438,
       "unindentified_rate": 46.4688,
-      "execution_time": 0.0431
+      "execution_time": 0.0408
     },
     "24": {
       "success_rate": 54.6875,
       "error_rate": 45.3125,
       "unindentified_rate": 0,
-      "execution_time": 0.0943
+      "execution_time": 0.0899
     },
     "36": {
       "success_rate": 69.125,
       "error_rate": 30.875,
       "unindentified_rate": 0,
-      "execution_time": 0.1228
+      "execution_time": 0.1216
     },
     "48": {
       "success_rate": 77.0938,
       "error_rate": 22.9063,
       "unindentified_rate": 0,
-      "execution_time": 0.1526
+      "execution_time": 0.1489
     },
     "64": {
       "success_rate": 80.9375,
       "error_rate": 19.0625,
       "unindentified_rate": 0,
-      "execution_time": 0.1831
+      "execution_time": 0.1784
     },
     "128": {
       "success_rate": 91.1875,
       "error_rate": 8.8125,
       "unindentified_rate": 0,
-      "execution_time": 0.2902
+      "execution_time": 0.2879
     },
     "256": {
       "success_rate": 96.7188,
       "error_rate": 3.2813,
       "unindentified_rate": 0,
-      "execution_time": 0.5402
+      "execution_time": 0.5192
     },
     "512": {
       "success_rate": 98.9375,
       "error_rate": 1.0625,
       "unindentified_rate": 0,
-      "execution_time": 0.971
+      "execution_time": 0.922
     },
     "1024": {
       "success_rate": 99.6825,
       "error_rate": 0.3175,
       "unindentified_rate": 0,
-      "execution_time": 1.6949
+      "execution_time": 1.626
     }
   },
   "stats": {
@@ -61,7 +61,7 @@
     "success_rate": 74.0726,
     "error_rate": 25.9274,
     "unindentified_rate": 0,
-    "execution_time": 0.1482
+    "execution_time": 0.1513
   },
   "languages": {
     "jpn": 99.93,