perf: tiktoken count (#1507)

* perf: tiktoken count

* fix: rerank histories

* fix: rerank histories

* update npmrc
This commit is contained in:
Archer
2024-05-16 15:42:15 +08:00
committed by GitHub
parent c6d9b15897
commit 6067f5aff3
8 changed files with 67 additions and 110 deletions

View File

@@ -13,7 +13,6 @@
"decompress": "^4.2.1",
"domino-ext": "^2.1.4",
"encoding": "^0.1.13",
"fastgpt-js-tiktoken": "^1.0.12",
"file-type": "^19.0.0",
"iconv-lite": "^0.6.3",
"joplin-turndown-plugin-gfm": "^1.0.12",
@@ -29,6 +28,7 @@
"papaparse": "5.4.1",
"pdfjs-dist": "4.0.269",
"pg": "^8.10.0",
"tiktoken": "^1.0.15",
"tunnel": "^0.0.6",
"turndown": "^7.1.2"
},