feat: use Tiktokenizer to count tokens

This commit is contained in:
archer
2023-04-24 15:18:05 +08:00
parent 29c95d24ae
commit adbaa8b37b
8 changed files with 149 additions and 32 deletions

View File

@@ -17,6 +17,7 @@
"@chakra-ui/icons": "^2.0.17",
"@chakra-ui/react": "^2.5.1",
"@chakra-ui/system": "^2.5.5",
"@dqbd/tiktoken": "^1.0.6",
"@emotion/react": "^11.10.6",
"@emotion/styled": "^11.10.6",
"@next/font": "13.1.6",
@@ -29,6 +30,7 @@
"formidable": "^2.1.1",
"framer-motion": "^9.0.6",
"gpt-token-utils": "^1.2.0",
"graphemer": "^1.4.0",
"hyperdown": "^2.4.29",
"immer": "^9.0.19",
"jsonwebtoken": "^9.0.0",