feat: quote change

src/pages/api/openapi/chat/lastChatResult.ts

@@ -1,38 +0,0 @@
-import type { NextApiRequest, NextApiResponse } from 'next';
-import { jsonRes } from '@/service/response';
-import { Chat } from '@/service/mongo';
-import { authUser } from '@/service/utils/auth';
-import { QuoteItemType } from '../kb/appKbSearch';
-
-type Props = {
-  chatId: string;
-};
-export type Response = {
-  quote: QuoteItemType[];
-};
-
-/* 聊天内容存存储 */
-export default async function handler(req: NextApiRequest, res: NextApiResponse) {
-  try {
-    const { chatId } = req.query as Props;
-
-    if (!chatId) {
-      throw new Error('缺少参数');
-    }
-
-    const { userId } = await authUser({ req });
-
-    const chatItem = await Chat.findOne({ _id: chatId, userId }, { content: { $slice: -1 } });
-
-    jsonRes<Response>(res, {
-      data: {
-        quote: chatItem?.content[0]?.quote || []
-      }
-    });
-  } catch (err) {
-    jsonRes(res, {
-      code: 500,
-      error: err
-    });
-  }
-}

src/pages/api/openapi/kb/appKbSearch.ts

@@ -13,7 +13,7 @@ import { openaiEmbedding } from '../plugin/openaiEmbedding';
 import { ModelDataStatusEnum } from '@/constants/model';
 import { modelToolMap } from '@/utils/plugin';
 
-export type QuoteItemType = { id: string; q: string; a: string };
+export type QuoteItemType = { id: string; q: string; a: string; isEdit: boolean };
 type Props = {
   prompts: ChatItemSimpleType[];
   similarity: number;
@@ -97,7 +97,7 @@ export async function appKbSearch({
   // search kb
   const searchRes = await Promise.all(
     promptVectors.map((promptVector) =>
-      PgClient.select<{ id: string; q: string; a: string }>('modelData', {
+      PgClient.select<QuoteItemType>('modelData', {
         fields: ['id', 'q', 'a'],
         where: [
           ['status', ModelDataStatusEnum.ready],

src/pages/api/openapi/text/gptMessagesSlice.ts

@@ -1,13 +1,9 @@
 // Next.js API route support: https://nextjs.org/docs/api-routes/introduction
 import type { NextApiRequest, NextApiResponse } from 'next';
-import { type Tiktoken } from '@dqbd/tiktoken';
 import { jsonRes } from '@/service/response';
 import { authUser } from '@/service/utils/auth';
-import Graphemer from 'graphemer';
 import type { ChatItemSimpleType } from '@/types/chat';
-import { ChatCompletionRequestMessage } from 'openai';
-import { getOpenAiEncMap } from '@/utils/plugin/openai';
-import { adaptChatItem_openAI } from '@/utils/plugin/openai';
+import { countOpenAIToken } from '@/utils/plugin/openai';
 
 type ModelType = 'gpt-3.5-turbo' | 'gpt-4' | 'gpt-4-32k';
 
@@ -52,62 +48,13 @@ export function gpt_chatItemTokenSlice({
   model: ModelType;
   maxToken: number;
 }) {
-  const textDecoder = new TextDecoder();
-  const graphemer = new Graphemer();
-
-  function getChatGPTEncodingText(messages: ChatCompletionRequestMessage[], model: ModelType) {
-    const isGpt3 = model === 'gpt-3.5-turbo';
-
-    const msgSep = isGpt3 ? '\n' : '';
-    const roleSep = isGpt3 ? '\n' : '<|im_sep|>';
-
-    return [
-      messages
-        .map(({ name = '', role, content }) => {
-          return `<|im_start|>${name || role}${roleSep}${content}<|im_end|>`;
-        })
-        .join(msgSep),
-      `<|im_start|>assistant${roleSep}`
-    ].join(msgSep);
-  }
-  function text2TokensLen(encoder: Tiktoken, inputText: string) {
-    const encoding = encoder.encode(inputText, 'all');
-    const segments: { text: string; tokens: { id: number; idx: number }[] }[] = [];
-
-    let byteAcc: number[] = [];
-    let tokenAcc: { id: number; idx: number }[] = [];
-    let inputGraphemes = graphemer.splitGraphemes(inputText);
-
-    for (let idx = 0; idx < encoding.length; idx++) {
-      const token = encoding[idx]!;
-      byteAcc.push(...encoder.decode_single_token_bytes(token));
-      tokenAcc.push({ id: token, idx });
-
-      const segmentText = textDecoder.decode(new Uint8Array(byteAcc));
-      const graphemes = graphemer.splitGraphemes(segmentText);
-
-      if (graphemes.every((item, idx) => inputGraphemes[idx] === item)) {
-        segments.push({ text: segmentText, tokens: tokenAcc });
-
-        byteAcc = [];
-        tokenAcc = [];
-        inputGraphemes = inputGraphemes.slice(graphemes.length);
-      }
-    }
-
-    return segments.reduce((memo, i) => memo + i.tokens.length, 0) ?? 0;
-  }
-  const OpenAiEncMap = getOpenAiEncMap();
-  const enc = OpenAiEncMap[model];
-
   let result: ChatItemSimpleType[] = [];
 
   for (let i = 0; i < messages.length; i++) {
     const msgs = [...result, messages[i]];
-    const tokens = text2TokensLen(
-      enc,
-      getChatGPTEncodingText(adaptChatItem_openAI({ messages }), model)
-    );
+
+    const tokens = countOpenAIToken({ messages: msgs, model });
+
     if (tokens < maxToken) {
       result = msgs;
     } else {