feat: content check

src/pages/api/chat/chat.ts

@@ -10,6 +10,7 @@ import { resStreamResponse } from '@/service/utils/chat';
 import { searchKb } from '@/service/plugins/searchKb';
 import { ChatRoleEnum } from '@/constants/chat';
 import { BillTypeEnum } from '@/constants/user';
+import { sensitiveCheck } from '@/service/api/text';
 
 /* 发送提示词 */
 export default async function handler(req: NextApiRequest, res: NextApiResponse) {
@@ -44,6 +45,10 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
 
     // 读取对话内容
     const prompts = [...content, prompt];
+    let systemPrompts: {
+      obj: ChatRoleEnum;
+      value: string;
+    }[] = [];
 
     // 使用了知识库搜索
     if (model.chat.relatedKbs.length > 0) {
@@ -60,16 +65,23 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
         return res.send(searchPrompts[0]?.value);
       }
 
-      prompts.splice(prompts.length - 3, 0, ...searchPrompts);
-    } else {
-      // 没有用知识库搜索，仅用系统提示词
-      model.chat.systemPrompt &&
-        prompts.splice(prompts.length - 3, 0, {
+      systemPrompts = searchPrompts;
+    } else if (model.chat.systemPrompt) {
+      systemPrompts = [
+        {
           obj: ChatRoleEnum.System,
           value: model.chat.systemPrompt
-        });
+        }
+      ];
     }
 
+    prompts.splice(prompts.length - 3, 0, ...systemPrompts);
+
+    // content check
+    await sensitiveCheck({
+      input: [...systemPrompts, prompt].map((item) => item.value).join('')
+    });
+
     // 计算温度
     const temperature = (modelConstantsData.maxTemperature * (model.chat.temperature / 10)).toFixed(
       2

src/pages/api/chat/shareChat/chat.ts

@@ -10,6 +10,7 @@ import { resStreamResponse } from '@/service/utils/chat';
 import { searchKb } from '@/service/plugins/searchKb';
 import { ChatRoleEnum } from '@/constants/chat';
 import { BillTypeEnum } from '@/constants/user';
+import { sensitiveCheck } from '@/service/api/text';
 
 /* 发送提示词 */
 export default async function handler(req: NextApiRequest, res: NextApiResponse) {
@@ -41,6 +42,11 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
 
     const modelConstantsData = ChatModelMap[model.chat.chatModel];
 
+    let systemPrompts: {
+      obj: ChatRoleEnum;
+      value: string;
+    }[] = [];
+
     // 使用了知识库搜索
     if (model.chat.relatedKbs.length > 0) {
       const { code, searchPrompts } = await searchKb({
@@ -56,16 +62,23 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
         return res.send(searchPrompts[0]?.value);
       }
 
-      prompts.splice(prompts.length - 3, 0, ...searchPrompts);
-    } else {
-      // 没有用知识库搜索，仅用系统提示词
-      model.chat.systemPrompt &&
-        prompts.splice(prompts.length - 3, 0, {
+      systemPrompts = searchPrompts;
+    } else if (model.chat.systemPrompt) {
+      systemPrompts = [
+        {
           obj: ChatRoleEnum.System,
           value: model.chat.systemPrompt
-        });
+        }
+      ];
     }
 
+    prompts.splice(prompts.length - 3, 0, ...systemPrompts);
+
+    // content check
+    await sensitiveCheck({
+      input: [...systemPrompts, prompts[prompts.length - 1]].map((item) => item.value).join('')
+    });
+
     // 计算温度
     const temperature = (modelConstantsData.maxTemperature * (model.chat.temperature / 10)).toFixed(
       2

src/pages/api/openapi/chat/chat.ts

@@ -10,6 +10,7 @@ import { searchKb } from '@/service/plugins/searchKb';
 import { ChatRoleEnum } from '@/constants/chat';
 import { withNextCors } from '@/service/utils/tools';
 import { BillTypeEnum } from '@/constants/user';
+import { sensitiveCheck } from '@/service/api/text';
 
 /* 发送提示词 */
 export default withNextCors(async function handler(req: NextApiRequest, res: NextApiResponse) {
@@ -62,13 +63,16 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
 
     const modelConstantsData = ChatModelMap[model.chat.chatModel];
 
+    let systemPrompts: {
+      obj: ChatRoleEnum;
+      value: string;
+    }[] = [];
+
     // 使用了知识库搜索
     if (model.chat.relatedKbs.length > 0) {
-      const similarity = ModelVectorSearchModeMap[model.chat.searchMode]?.similarity || 0.22;
-
       const { code, searchPrompts } = await searchKb({
         prompts,
-        similarity,
+        similarity: ModelVectorSearchModeMap[model.chat.searchMode]?.similarity,
         model,
         userId
       });
@@ -77,18 +81,29 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
       if (code === 201) {
         return isStream
           ? res.send(searchPrompts[0]?.value)
-          : jsonRes(res, { data: searchPrompts[0]?.value });
+          : jsonRes(res, {
+              data: searchPrompts[0]?.value,
+              message: searchPrompts[0]?.value
+            });
       }
-      prompts.splice(prompts.length - 3, 0, ...searchPrompts);
-    } else {
-      // 没有用知识库搜索，仅用系统提示词
-      model.chat.systemPrompt &&
-        prompts.splice(prompts.length - 3, 0, {
+
+      systemPrompts = searchPrompts;
+    } else if (model.chat.systemPrompt) {
+      systemPrompts = [
+        {
           obj: ChatRoleEnum.System,
           value: model.chat.systemPrompt
-        });
+        }
+      ];
     }
 
+    prompts.splice(prompts.length - 3, 0, ...systemPrompts);
+
+    // content check
+    await sensitiveCheck({
+      input: [...systemPrompts, prompts[prompts.length - 1]].map((item) => item.value).join('')
+    });
+
     // 计算温度
     const temperature = (modelConstantsData.maxTemperature * (model.chat.temperature / 10)).toFixed(
       2

src/pages/api/openapi/text/sensitiveCheck.ts

@@ -0,0 +1,48 @@
+// Next.js API route support: https://nextjs.org/docs/api-routes/introduction
+import type { NextApiRequest, NextApiResponse } from 'next';
+import { jsonRes } from '@/service/response';
+import { authUser, getSystemOpenAiKey } from '@/service/utils/auth';
+import type { TextPluginRequestParams } from '@/types/plugin';
+import axios from 'axios';
+import { axiosConfig } from '@/service/utils/tools';
+
+export default async function handler(req: NextApiRequest, res: NextApiResponse) {
+  try {
+    if (process.env.SENSITIVE_CHECK !== '1') {
+      return jsonRes(res);
+    }
+
+    await authUser({ req });
+
+    const { input } = req.body as TextPluginRequestParams;
+
+    const response = await axios({
+      ...axiosConfig(getSystemOpenAiKey()),
+      method: 'POST',
+      url: `/moderations`,
+      data: {
+        input
+      }
+    });
+
+    const data = (response.data.results?.[0]?.category_scores as Record<string, number>) || {};
+
+    const values = Object.values(data);
+
+    for (const val of values) {
+      if (val > 0.2) {
+        return jsonRes(res, {
+          code: 500,
+          message: '您的内容不合规'
+        });
+      }
+    }
+
+    jsonRes(res);
+  } catch (err) {
+    jsonRes(res, {
+      code: 500,
+      error: err
+    });
+  }
+}