feat: chat content use tiktoken count

src/constants/model.ts

@@ -5,24 +5,28 @@ export enum ModelDataStatusEnum {
   waiting = 'waiting'
 }
 
-export enum ChatModelNameEnum {
-  GPT35 = 'gpt-3.5-turbo',
-  VECTOR_GPT = 'VECTOR_GPT',
-  VECTOR = 'text-embedding-ada-002'
+export const embeddingModel = 'text-embedding-ada-002';
+export enum ChatModelEnum {
+  'GPT35' = 'gpt-3.5-turbo',
+  'GPT4' = 'gpt-4',
+  'GPT432k' = 'gpt-4-32k'
 }
 
-export const ChatModelNameMap = {
-  [ChatModelNameEnum.GPT35]: 'gpt-3.5-turbo',
-  [ChatModelNameEnum.VECTOR_GPT]: 'gpt-3.5-turbo',
-  [ChatModelNameEnum.VECTOR]: 'text-embedding-ada-002'
+export enum ModelNameEnum {
+  GPT35 = 'gpt-3.5-turbo',
+  VECTOR_GPT = 'VECTOR_GPT'
+}
+
+export const Model2ChatModelMap: Record<`${ModelNameEnum}`, `${ChatModelEnum}`> = {
+  [ModelNameEnum.GPT35]: 'gpt-3.5-turbo',
+  [ModelNameEnum.VECTOR_GPT]: 'gpt-3.5-turbo'
 };
 
 export type ModelConstantsData = {
   icon: 'model' | 'dbModel';
   name: string;
-  model: `${ChatModelNameEnum}`;
+  model: `${ModelNameEnum}`;
   trainName: string; // 空字符串代表不能训练
-  maxToken: number;
   contextMaxToken: number;
   maxTemperature: number;
   price: number; // 多少钱 / 1token，单位: 0.00001元
@@ -32,20 +36,18 @@ export const modelList: ModelConstantsData[] = [
   {
     icon: 'model',
     name: 'chatGPT',
-    model: ChatModelNameEnum.GPT35,
+    model: ModelNameEnum.GPT35,
     trainName: '',
-    maxToken: 4000,
-    contextMaxToken: 7000,
+    contextMaxToken: 4096,
     maxTemperature: 1.5,
     price: 3
   },
   {
     icon: 'dbModel',
     name: '知识库',
-    model: ChatModelNameEnum.VECTOR_GPT,
+    model: ModelNameEnum.VECTOR_GPT,
     trainName: 'vector',
-    maxToken: 4000,
-    contextMaxToken: 7000,
+    contextMaxToken: 4096,
     maxTemperature: 1,
     price: 3
   }
@@ -133,8 +135,8 @@ export const defaultModel: ModelSchema = {
   },
   service: {
     trainId: '',
-    chatModel: ChatModelNameEnum.GPT35,
-    modelName: ChatModelNameEnum.GPT35
+    chatModel: ModelNameEnum.GPT35,
+    modelName: ModelNameEnum.GPT35
   },
   security: {
     domain: ['*'],