gitea/FastGPT
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: chat content use tiktoken count

Browse Source
This commit is contained in:
archer
2023-04-24 16:46:39 +08:00
parent adbaa8b37b
commit 1f112f7715
23 changed files with 182 additions and 836 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/pages/api/data/splitData.ts
View File

@@ -4,7 +4,7 @@ import { connectToDatabase, DataItem, Data } from '@/service/mongo';
import { authToken } from '@/service/utils/tools';
import { generateQA } from '@/service/events/generateQA';
import { generateAbstract } from '@/service/events/generateAbstract';
import { encode } from 'gpt-token-utils';
import { countChatTokens } from '@/utils/tools';
/* 拆分数据成QA */
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
@@ -34,7 +34,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
chunks.forEach((chunk) => {
splitText += chunk;
const tokens = encode(splitText).length;
const tokens = countChatTokens({ messages: [{ role: 'system', content: splitText }] });
if (tokens >= 780) {
dataItems.push({
userId,