feat: modeldata接口。fix: 部分权限校验bug

src/pages/api/model/train/getTrainings.ts

@@ -0,0 +1,52 @@
+import type { NextApiRequest, NextApiResponse } from 'next';
+import { jsonRes } from '@/service/response';
+import { connectToDatabase, Training } from '@/service/mongo';
+import { authToken } from '@/service/utils/tools';
+
+// 关闭next默认的bodyParser处理方式
+export const config = {
+  api: {
+    bodyParser: false
+  }
+};
+
+/* 获取模型训练记录 */
+export default async function handler(req: NextApiRequest, res: NextApiResponse) {
+  try {
+    const { authorization } = req.headers;
+
+    if (!authorization) {
+      throw new Error('无权操作');
+    }
+    const { modelId } = req.query;
+    if (!modelId) {
+      throw new Error('参数错误');
+    }
+    await authToken(authorization);
+
+    await connectToDatabase();
+
+    /* 获取 modelId 下的 training 记录 */
+    const records = await Training.find({
+      modelId
+    });
+
+    jsonRes(res, {
+      data: records
+    });
+  } catch (err: any) {
+    /* 清除上传的文件，关闭训练记录 */
+    // @ts-ignore
+    if (openai) {
+      // @ts-ignore
+      uploadFileId && openai.deleteFile(uploadFileId);
+      // @ts-ignore
+      trainId && openai.cancelFineTune(trainId);
+    }
+
+    jsonRes(res, {
+      code: 500,
+      error: err
+    });
+  }
+}

src/pages/api/model/train/putTrainStatus.ts

@@ -0,0 +1,104 @@
+import type { NextApiRequest, NextApiResponse } from 'next';
+import { jsonRes } from '@/service/response';
+import { connectToDatabase, Model, Training } from '@/service/mongo';
+import { getOpenAIApi } from '@/service/utils/chat';
+import { authToken, getUserOpenaiKey } from '@/service/utils/tools';
+import type { ModelSchema } from '@/types/mongoSchema';
+import { TrainingItemType } from '@/types/training';
+import { ModelStatusEnum, TrainingStatusEnum } from '@/constants/model';
+import { OpenAiTuneStatusEnum } from '@/service/constants/training';
+import { httpsAgent } from '@/service/utils/tools';
+
+/* 更新训练状态 */
+export default async function handler(req: NextApiRequest, res: NextApiResponse) {
+  try {
+    const { authorization } = req.headers;
+
+    if (!authorization) {
+      throw new Error('无权操作');
+    }
+    const { modelId } = req.query as { modelId: string };
+    if (!modelId) {
+      throw new Error('参数错误');
+    }
+    const userId = await authToken(authorization);
+
+    await connectToDatabase();
+
+    // 获取模型
+    const model = await Model.findById<ModelSchema>(modelId);
+
+    if (!model || model.status !== 'training') {
+      throw new Error('模型不在训练中');
+    }
+
+    // 查询正在训练中的训练记录
+    const training: TrainingItemType | null = await Training.findOne({
+      modelId,
+      status: 'pending'
+    });
+
+    if (!training) {
+      throw new Error('找不到训练记录');
+    }
+
+    // 用户的 openai 实例
+    const openai = getOpenAIApi(await getUserOpenaiKey(userId));
+
+    // 获取 openai 的训练情况
+    const { data } = await openai.retrieveFineTune(training.tuneId, { httpsAgent });
+    // console.log(data);
+    if (data.status === OpenAiTuneStatusEnum.succeeded) {
+      // 删除训练文件
+      openai.deleteFile(data.training_files[0].id, { httpsAgent });
+
+      // 更新模型状态和模型内容
+      await Model.findByIdAndUpdate(modelId, {
+        status: ModelStatusEnum.running,
+        updateTime: new Date(),
+        service: {
+          ...model.service,
+          trainId: data.fine_tuned_model, // 训练完后，再次训练和对话使用的 model 是一样的
+          chatModel: data.fine_tuned_model
+        }
+      });
+      // 更新训练数据
+      await Training.findByIdAndUpdate(training._id, {
+        status: TrainingStatusEnum.succeed
+      });
+
+      return jsonRes(res, {
+        data: '模型微调完成'
+      });
+    }
+
+    /* 取消微调 */
+    if (data.status === OpenAiTuneStatusEnum.cancelled) {
+      // 删除训练文件
+      openai.deleteFile(data.training_files[0].id, { httpsAgent });
+
+      // 更新模型
+      await Model.findByIdAndUpdate(modelId, {
+        status: ModelStatusEnum.running,
+        updateTime: new Date()
+      });
+      // 更新训练数据
+      await Training.findByIdAndUpdate(training._id, {
+        status: TrainingStatusEnum.canceled
+      });
+
+      return jsonRes(res, {
+        data: '模型微调已取消'
+      });
+    }
+
+    jsonRes(res, {
+      data: '模型还在训练中'
+    });
+  } catch (err: any) {
+    jsonRes(res, {
+      code: 500,
+      error: err
+    });
+  }
+}

src/pages/api/model/train/train.ts

@@ -0,0 +1,129 @@
+// Next.js API route support: https://nextjs.org/docs/api-routes/introduction
+import type { NextApiRequest, NextApiResponse } from 'next';
+import { jsonRes } from '@/service/response';
+import { connectToDatabase, Model, Training } from '@/service/mongo';
+import { getOpenAIApi } from '@/service/utils/chat';
+import formidable from 'formidable';
+import { authToken, getUserOpenaiKey } from '@/service/utils/tools';
+import { join } from 'path';
+import fs from 'fs';
+import type { ModelSchema } from '@/types/mongoSchema';
+import type { OpenAIApi } from 'openai';
+import { ModelStatusEnum, TrainingStatusEnum } from '@/constants/model';
+import { httpsAgent } from '@/service/utils/tools';
+
+// 关闭next默认的bodyParser处理方式
+export const config = {
+  api: {
+    bodyParser: false
+  }
+};
+
+/* 上传文件，开始微调 */
+export default async function handler(req: NextApiRequest, res: NextApiResponse) {
+  let openai: OpenAIApi, trainId: string, uploadFileId: string;
+
+  try {
+    const { authorization } = req.headers;
+
+    if (!authorization) {
+      throw new Error('无权操作');
+    }
+    const { modelId } = req.query;
+
+    if (!modelId) {
+      throw new Error('参数错误');
+    }
+    const userId = await authToken(authorization);
+
+    await connectToDatabase();
+
+    // 获取模型的状态
+    const model = await Model.findById<ModelSchema>(modelId);
+
+    if (!model || model.status !== 'running') {
+      throw new Error('模型正忙');
+    }
+
+    // const trainingType = model.service.modelType
+    const trainingType = model.service.trainId; // 目前都默认是 openai text-davinci-03
+
+    // 获取用户的 API Key 实例化后的对象
+    openai = getOpenAIApi(await getUserOpenaiKey(userId));
+
+    // 接收文件并保存
+    const form = formidable({
+      uploadDir: join(process.cwd(), 'public/trainData'),
+      keepExtensions: true
+    });
+
+    const { files } = await new Promise<{
+      fields: formidable.Fields;
+      files: formidable.Files;
+    }>((resolve, reject) => {
+      form.parse(req, (err, fields, files) => {
+        if (err) return reject(err);
+        resolve({ fields, files });
+      });
+    });
+    const file = files.file;
+
+    // 上传文件到 openai
+    // @ts-ignore
+    const uploadRes = await openai.createFile(
+      // @ts-ignore
+      fs.createReadStream(file.filepath),
+      'fine-tune',
+      { httpsAgent }
+    );
+    uploadFileId = uploadRes.data.id; // 记录上传文件的 ID
+
+    // 开始训练
+    const trainRes = await openai.createFineTune(
+      {
+        training_file: uploadFileId,
+        model: trainingType,
+        suffix: model.name,
+        n_epochs: 4
+      },
+      { httpsAgent }
+    );
+
+    trainId = trainRes.data.id; // 记录训练 ID
+
+    // 创建训练记录
+    await Training.create({
+      serviceName: 'openai',
+      tuneId: trainId,
+      status: TrainingStatusEnum.pending,
+      modelId
+    });
+
+    // 修改模型状态
+    await Model.findByIdAndUpdate(modelId, {
+      $inc: {
+        trainingTimes: +1
+      },
+      updateTime: new Date(),
+      status: ModelStatusEnum.training
+    });
+
+    jsonRes(res, {
+      data: 'start training'
+    });
+  } catch (err: any) {
+    /* 清除上传的文件，关闭训练记录 */
+    // @ts-ignore
+    if (openai) {
+      // @ts-ignore
+      uploadFileId && openai.deleteFile(uploadFileId, { httpsAgent });
+      // @ts-ignore
+      trainId && openai.cancelFineTune(trainId, { httpsAgent });
+    }
+
+    jsonRes(res, {
+      code: 500,
+      error: err
+    });
+  }
+}