Add whisper and tts ui (#484)

Co-authored-by: heheer <71265218+newfish-cmyk@users.noreply.github.com>

projects/app/src/pages/api/core/chat/item/getSpeech.ts

@@ -4,7 +4,7 @@ import { connectToDatabase } from '@/service/mongo';
 import { GetChatSpeechProps } from '@/global/core/chat/api.d';
 import { text2Speech } from '@fastgpt/service/core/ai/audio/speech';
 import { pushAudioSpeechBill } from '@/service/support/wallet/bill/push';
-import { authCert } from '@fastgpt/service/support/permission/auth/common';
+import { authCertAndShareId } from '@fastgpt/service/support/permission/auth/common';
 import { authType2BillSource } from '@/service/support/wallet/bill/utils';
 import { getAudioSpeechModel } from '@/service/core/ai/model';
 import { MongoTTSBuffer } from '@fastgpt/service/common/buffer/tts/schema';
@@ -19,16 +19,16 @@ import { MongoTTSBuffer } from '@fastgpt/service/common/buffer/tts/schema';
 export default async function handler(req: NextApiRequest, res: NextApiResponse) {
   try {
     await connectToDatabase();
-    const { ttsConfig, input } = req.body as GetChatSpeechProps;
+    const { ttsConfig, input, shareId } = req.body as GetChatSpeechProps;
 
     if (!ttsConfig.model || !ttsConfig.voice) {
       throw new Error('model or voice not found');
     }
 
-    const { teamId, tmbId, authType } = await authCert({ req, authToken: true });
+    const { teamId, tmbId, authType } = await authCertAndShareId({ req, authToken: true, shareId });
 
     const ttsModel = getAudioSpeechModel(ttsConfig.model);
-    const voiceData = ttsModel.voices.find((item) => item.value === ttsConfig.voice);
+    const voiceData = ttsModel.voices?.find((item) => item.value === ttsConfig.voice);
 
     if (!voiceData) {
       throw new Error('voice not found');
@@ -37,7 +37,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
     const ttsBuffer = await MongoTTSBuffer.findOne(
       {
         bufferId: voiceData.bufferId,
-        text: input
+        text: JSON.stringify({ text: input, speed: ttsConfig.speed })
       },
       'buffer'
     );
@@ -51,6 +51,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
       input,
       model: ttsConfig.model,
       voice: ttsConfig.voice,
+      speed: ttsConfig.speed,
       props: {
         // temp code
         baseUrl: ttsModel.baseUrl || '',
@@ -68,7 +69,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
 
           await MongoTTSBuffer.create({
             bufferId: voiceData.bufferId,
-            text: input,
+            text: JSON.stringify({ text: input, speed: ttsConfig.speed }),
             buffer
           });
         } catch (error) {}

projects/app/src/pages/api/system/getInitData.ts

@@ -2,7 +2,7 @@ import type { FeConfigsType, SystemEnvType } from '@fastgpt/global/common/system
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@fastgpt/service/common/response';
 import { readFileSync } from 'fs';
-import type { InitDateResponse } from '@/global/common/api/systemRes';
+import type { ConfigFileType, InitDateResponse } from '@/global/common/api/systemRes';
 import { formatPrice } from '@fastgpt/global/support/wallet/bill/tools';
 import { getTikTokenEnc } from '@fastgpt/global/common/string/tiktoken';
 import { initHttpAgent } from '@fastgpt/service/common/middle/httpAgent';
@@ -13,15 +13,9 @@ import {
   defaultExtractModels,
   defaultQGModels,
   defaultVectorModels,
-  defaultAudioSpeechModels
+  defaultAudioSpeechModels,
+  defaultWhisperModel
 } from '@fastgpt/global/core/ai/model';
-import {
-  AudioSpeechModelType,
-  ChatModelItemType,
-  FunctionModelItemType,
-  LLMModelItemType,
-  VectorModelItemType
-} from '@fastgpt/global/core/ai/model.d';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse) {
   getInitConfig();
@@ -83,60 +77,39 @@ export function getInitConfig() {
 
     const filename =
       process.env.NODE_ENV === 'development' ? 'data/config.local.json' : '/app/data/config.json';
-    const res = JSON.parse(readFileSync(filename, 'utf-8')) as {
-      FeConfig: FeConfigsType;
-      SystemParams: SystemEnvType;
-      ChatModels: ChatModelItemType[];
-      QAModels: LLMModelItemType[];
-      CQModels: FunctionModelItemType[];
-      ExtractModels: FunctionModelItemType[];
-      QGModels: LLMModelItemType[];
-      VectorModels: VectorModelItemType[];
-      AudioSpeechModels: AudioSpeechModelType[];
-    };
+    const res = JSON.parse(readFileSync(filename, 'utf-8')) as ConfigFileType;
 
     console.log(`System Version: ${global.systemVersion}`);
 
-    console.log(res);
-
-    global.systemEnv = res.SystemParams
-      ? { ...defaultSystemEnv, ...res.SystemParams }
-      : defaultSystemEnv;
-    global.feConfigs = res.FeConfig
-      ? { ...defaultFeConfigs, ...res.FeConfig, isPlus: !!res.SystemParams?.pluginBaseUrl }
-      : defaultFeConfigs;
-
-    global.chatModels = res.ChatModels || defaultChatModels;
-    global.qaModels = res.QAModels || defaultQAModels;
-    global.cqModels = res.CQModels || defaultCQModels;
-    global.extractModels = res.ExtractModels || defaultExtractModels;
-    global.qgModels = res.QGModels || defaultQGModels;
-
-    global.vectorModels = res.VectorModels || defaultVectorModels;
-
-    global.audioSpeechModels = res.AudioSpeechModels || defaultAudioSpeechModels;
+    setDefaultData(res);
   } catch (error) {
     setDefaultData();
     console.log('get init config error, set default', error);
   }
 }
 
-export function setDefaultData() {
-  global.systemEnv = defaultSystemEnv;
-  global.feConfigs = defaultFeConfigs;
+export function setDefaultData(res?: ConfigFileType) {
+  global.systemEnv = res?.SystemParams
+    ? { ...defaultSystemEnv, ...res.SystemParams }
+    : defaultSystemEnv;
+  global.feConfigs = res?.FeConfig
+    ? { ...defaultFeConfigs, ...res.FeConfig, isPlus: !!res.SystemParams?.pluginBaseUrl }
+    : defaultFeConfigs;
 
-  global.chatModels = defaultChatModels;
-  global.qaModels = defaultQAModels;
-  global.cqModels = defaultCQModels;
-  global.extractModels = defaultExtractModels;
-  global.qgModels = defaultQGModels;
+  global.chatModels = res?.ChatModels || defaultChatModels;
+  global.qaModels = res?.QAModels || defaultQAModels;
+  global.cqModels = res?.CQModels || defaultCQModels;
+  global.extractModels = res?.ExtractModels || defaultExtractModels;
+  global.qgModels = res?.QGModels || defaultQGModels;
 
-  global.vectorModels = defaultVectorModels;
-  global.audioSpeechModels = defaultAudioSpeechModels;
+  global.vectorModels = res?.VectorModels || defaultVectorModels;
+
+  global.audioSpeechModels = res?.AudioSpeechModels || defaultAudioSpeechModels;
+
+  global.whisperModel = res?.WhisperModel || defaultWhisperModel;
 
   global.priceMd = '';
 
-  console.log('use default config');
   console.log(global);
 }
 
@@ -178,6 +151,10 @@ ${global.extractModels
 ${global.qgModels
   ?.map((item) => `| 下一步指引-${item.name} | ${formatPrice(item.price, 1000)} |`)
   .join('\n')}
+${global.audioSpeechModels
+  ?.map((item) => `| 语音播放-${item.name} | ${formatPrice(item.price, 1000)} |`)
+  .join('\n')}
+${`| 语音输入-${global.whisperModel.name} | ${global.whisperModel.price}/分钟 |`}
 `;
   console.log(global.priceMd);
 }

projects/app/src/pages/api/v1/audio/transcriptions.ts

@@ -1,10 +1,11 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@fastgpt/service/common/response';
-import { authCert } from '@fastgpt/service/support/permission/auth/common';
+import { authCert, authCertAndShareId } from '@fastgpt/service/support/permission/auth/common';
 import { withNextCors } from '@fastgpt/service/common/middle/cors';
 import { getUploadModel } from '@fastgpt/service/common/file/upload/multer';
 import fs from 'fs';
 import { getAIApi } from '@fastgpt/service/core/ai/config';
+import { pushWhisperBill } from '@/service/support/wallet/bill/push';
 
 const upload = getUploadModel({
   maxSize: 2
@@ -12,9 +13,16 @@ const upload = getUploadModel({
 
 export default withNextCors(async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
   try {
+    const {
+      files,
+      metadata: { duration, shareId }
+    } = await upload.doUpload<{ duration: number; shareId?: string }>(req, res);
+
     const { teamId, tmbId } = await authCert({ req, authToken: true });
 
-    const { files } = await upload.doUpload(req, res);
+    if (!global.whisperModel) {
+      throw new Error('whisper model not found');
+    }
 
     const file = files[0];
 
@@ -26,7 +34,13 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
 
     const result = await ai.audio.transcriptions.create({
       file: fs.createReadStream(file.path),
-      model: 'whisper-1'
+      model: global.whisperModel.model
+    });
+
+    pushWhisperBill({
+      teamId,
+      tmbId,
+      duration
     });
 
     jsonRes(res, {