perf: token count;feat: chunk size

2023-06-23 15:08:30 +08:00
parent 9aace871ff
commit ae1f7a888e
5 changed files with 85 additions and 148 deletions
--- a/client/src/utils/file.ts
+++ b/client/src/utils/file.ts
@@ -148,15 +148,9 @@ export const fileDownload = ({
 * slideLen - The size of the before and after Text
 * maxLen > slideLen
 */
-export const splitText_token = ({
-  text,
-  maxLen,
-  slideLen
-}: {
-  text: string;
-  maxLen: number;
-  slideLen: number;
-}) => {
+export const splitText_token = ({ text, maxLen }: { text: string; maxLen: number }) => {
+  const slideLen = Math.floor(maxLen * 0.3);
+
  try {
    const enc = getOpenAiEncMap()[OpenAiChatEnum.GPT35];
    // filter empty text. encode sentence