feat: use Tiktokenizer to count tokens
This commit is contained in:
@@ -17,6 +17,7 @@
|
||||
"@chakra-ui/icons": "^2.0.17",
|
||||
"@chakra-ui/react": "^2.5.1",
|
||||
"@chakra-ui/system": "^2.5.5",
|
||||
"@dqbd/tiktoken": "^1.0.6",
|
||||
"@emotion/react": "^11.10.6",
|
||||
"@emotion/styled": "^11.10.6",
|
||||
"@next/font": "13.1.6",
|
||||
@@ -29,6 +30,7 @@
|
||||
"formidable": "^2.1.1",
|
||||
"framer-motion": "^9.0.6",
|
||||
"gpt-token-utils": "^1.2.0",
|
||||
"graphemer": "^1.4.0",
|
||||
"hyperdown": "^2.4.29",
|
||||
"immer": "^9.0.19",
|
||||
"jsonwebtoken": "^9.0.0",
|
||||
|
||||
Reference in New Issue
Block a user