perf: search prompt, upload step and psw len

2023-08-10 11:49:32 +08:00
parent 9ea19b8eaa
commit 63c832d883
14 changed files with 113 additions and 58 deletions
--- a/client/src/pages/api/openapi/kb/pushData.ts
+++ b/client/src/pages/api/openapi/kb/pushData.ts
@@ -90,15 +90,14 @@ export async function pushDataToKb({
  data.forEach((item) => {
    const text = item.q + item.a;

-    if (mode === TrainingModeEnum.qa) {
-      // count token
-      const token = modelToolMap.countTokens({
-        model: 'gpt-3.5-turbo-16k',
-        messages: [{ obj: 'System', value: item.q }]
-      });
-      if (token > modeMaxToken[TrainingModeEnum.qa]) {
-        return;
-      }
+    // count token
+    const token = modelToolMap.countTokens({
+      model: 'gpt-3.5-turbo',
+      messages: [{ obj: 'System', value: item.q }]
+    });
+
+    if (token > modeMaxToken[TrainingModeEnum.qa]) {
+      return;
    }

    if (!set.has(text)) {