new framwork

2023-06-09 12:57:42 +08:00
parent d9450bd7ee
commit ba9d9c3d5f
263 changed files with 12269 additions and 11599 deletions
--- a/client/src/utils/plugin/openai.ts
+++ b/client/src/utils/plugin/openai.ts
@@ -0,0 +1,162 @@
+import { encoding_for_model, type Tiktoken } from '@dqbd/tiktoken';
+import type { ChatItemSimpleType } from '@/types/chat';
+import { ChatRoleEnum } from '@/constants/chat';
+import { ChatCompletionRequestMessage, ChatCompletionRequestMessageRoleEnum } from 'openai';
+import { OpenAiChatEnum } from '@/constants/model';
+import Graphemer from 'graphemer';
+
+const textDecoder = new TextDecoder();
+const graphemer = new Graphemer();
+
+export const getOpenAiEncMap = () => {
+  if (typeof window !== 'undefined') {
+    window.OpenAiEncMap = window.OpenAiEncMap || {
+      'gpt-3.5-turbo': encoding_for_model('gpt-3.5-turbo', {
+        '<|im_start|>': 100264,
+        '<|im_end|>': 100265,
+        '<|im_sep|>': 100266
+      }),
+      'gpt-4': encoding_for_model('gpt-4', {
+        '<|im_start|>': 100264,
+        '<|im_end|>': 100265,
+        '<|im_sep|>': 100266
+      }),
+      'gpt-4-32k': encoding_for_model('gpt-4-32k', {
+        '<|im_start|>': 100264,
+        '<|im_end|>': 100265,
+        '<|im_sep|>': 100266
+      })
+    };
+    return window.OpenAiEncMap;
+  }
+  if (typeof global !== 'undefined') {
+    global.OpenAiEncMap = global.OpenAiEncMap || {
+      'gpt-3.5-turbo': encoding_for_model('gpt-3.5-turbo', {
+        '<|im_start|>': 100264,
+        '<|im_end|>': 100265,
+        '<|im_sep|>': 100266
+      }),
+      'gpt-4': encoding_for_model('gpt-4', {
+        '<|im_start|>': 100264,
+        '<|im_end|>': 100265,
+        '<|im_sep|>': 100266
+      }),
+      'gpt-4-32k': encoding_for_model('gpt-4-32k', {
+        '<|im_start|>': 100264,
+        '<|im_end|>': 100265,
+        '<|im_sep|>': 100266
+      })
+    };
+    return global.OpenAiEncMap;
+  }
+  return {
+    'gpt-3.5-turbo': encoding_for_model('gpt-3.5-turbo', {
+      '<|im_start|>': 100264,
+      '<|im_end|>': 100265,
+      '<|im_sep|>': 100266
+    }),
+    'gpt-4': encoding_for_model('gpt-4', {
+      '<|im_start|>': 100264,
+      '<|im_end|>': 100265,
+      '<|im_sep|>': 100266
+    }),
+    'gpt-4-32k': encoding_for_model('gpt-4-32k', {
+      '<|im_start|>': 100264,
+      '<|im_end|>': 100265,
+      '<|im_sep|>': 100266
+    })
+  };
+};
+
+export const adaptChatItem_openAI = ({
+  messages
+}: {
+  messages: ChatItemSimpleType[];
+}): ChatCompletionRequestMessage[] => {
+  const map = {
+    [ChatRoleEnum.AI]: ChatCompletionRequestMessageRoleEnum.Assistant,
+    [ChatRoleEnum.Human]: ChatCompletionRequestMessageRoleEnum.User,
+    [ChatRoleEnum.System]: ChatCompletionRequestMessageRoleEnum.System
+  };
+  return messages.map((item) => ({
+    role: map[item.obj] || ChatCompletionRequestMessageRoleEnum.System,
+    content: item.value || ''
+  }));
+};
+
+export function countOpenAIToken({
+  messages,
+  model
+}: {
+  messages: ChatItemSimpleType[];
+  model: `${OpenAiChatEnum}`;
+}) {
+  function getChatGPTEncodingText(
+    messages: {
+      role: 'system' | 'user' | 'assistant';
+      content: string;
+      name?: string;
+    }[],
+    model: 'gpt-3.5-turbo' | 'gpt-4' | 'gpt-4-32k'
+  ) {
+    const isGpt3 = model === 'gpt-3.5-turbo';
+
+    const msgSep = isGpt3 ? '\n' : '';
+    const roleSep = isGpt3 ? '\n' : '<|im_sep|>';
+
+    return [
+      messages
+        .map(({ name = '', role, content }) => {
+          return `<|im_start|>${name || role}${roleSep}${content}<|im_end|>`;
+        })
+        .join(msgSep),
+      `<|im_start|>assistant${roleSep}`
+    ].join(msgSep);
+  }
+  function text2TokensLen(encoder: Tiktoken, inputText: string) {
+    const encoding = encoder.encode(inputText, 'all');
+    const segments: { text: string; tokens: { id: number; idx: number }[] }[] = [];
+
+    let byteAcc: number[] = [];
+    let tokenAcc: { id: number; idx: number }[] = [];
+    let inputGraphemes = graphemer.splitGraphemes(inputText);
+
+    for (let idx = 0; idx < encoding.length; idx++) {
+      const token = encoding[idx]!;
+      byteAcc.push(...encoder.decode_single_token_bytes(token));
+      tokenAcc.push({ id: token, idx });
+
+      const segmentText = textDecoder.decode(new Uint8Array(byteAcc));
+      const graphemes = graphemer.splitGraphemes(segmentText);
+
+      if (graphemes.every((item, idx) => inputGraphemes[idx] === item)) {
+        segments.push({ text: segmentText, tokens: tokenAcc });
+
+        byteAcc = [];
+        tokenAcc = [];
+        inputGraphemes = inputGraphemes.slice(graphemes.length);
+      }
+    }
+
+    return segments.reduce((memo, i) => memo + i.tokens.length, 0) ?? 0;
+  }
+
+  const adaptMessages = adaptChatItem_openAI({ messages });
+
+  return text2TokensLen(getOpenAiEncMap()[model], getChatGPTEncodingText(adaptMessages, model));
+}
+
+export const openAiSliceTextByToken = ({
+  model = 'gpt-3.5-turbo',
+  text,
+  length
+}: {
+  model: `${OpenAiChatEnum}`;
+  text: string;
+  length: number;
+}) => {
+  const enc = getOpenAiEncMap()[model];
+  const encodeText = enc.encode(text);
+  const decoder = new TextDecoder();
+  return decoder.decode(enc.decode(encodeText.slice(0, length)));
+};