4.6.8-alpha (#804)

* perf: redirect request and err log replace perf: dataset openapi feat: session fix: retry input error feat: 468 doc sub page feat: standard sub perf: rerank tip perf: rerank tip perf: api sdk perf: openapi sub plan perf: sub ui fix: ts * perf: init log * fix: variable select * sub page * icon * perf: llm model config * perf: menu ux * perf: system store * perf: publish app name * fix: init data * perf: flow edit ux * fix: value type format and ux * fix prompt editor default value (#13) * fix prompt editor default value * fix prompt editor update when not focus * add key with variable --------- Co-authored-by: Archer <545436317@qq.com> * fix: value type * doc * i18n * import path * home page * perf: mongo session running * fix: ts * perf: use toast * perf: flow edit * perf: sse response * slider ui * fetch error * fix prompt editor rerender when not focus by key defaultvalue (#14) * perf: prompt editor * feat: dataset search concat * perf: doc * fix:ts * perf: doc * fix json editor onblur value (#15) * faq * vector model default config * ipv6 --------- Co-authored-by: heheer <71265218+newfish-cmyk@users.noreply.github.com>
2024-02-01 21:57:41 +08:00
parent fc19c4cf09
commit 34602b25df
285 changed files with 10345 additions and 11223 deletions
--- a/packages/service/common/file/image/controller.ts
+++ b/packages/service/common/file/image/controller.ts
@@ -1,6 +1,7 @@
 import { UploadImgProps } from '@fastgpt/global/common/file/api';
 import { imageBaseUrl } from '@fastgpt/global/common/file/image/constants';
 import { MongoImage } from './schema';
+import { ClientSession } from '../../../common/mongo';

 export function getMongoImgUrl(id: string) {
  return `${imageBaseUrl}${id}`;
@@ -48,15 +49,20 @@ export async function readMongoImg({ id }: { id: string }) {

 export async function delImgByRelatedId({
  teamId,
-  relateIds
+  relateIds,
+  session
 }: {
  teamId: string;
  relateIds: string[];
+  session: ClientSession;
 }) {
  if (relateIds.length === 0) return;

-  return MongoImage.deleteMany({
-    teamId,
-    'metadata.relatedId': { $in: relateIds.map((id) => String(id)) }
-  });
+  return MongoImage.deleteMany(
+    {
+      teamId,
+      'metadata.relatedId': { $in: relateIds.map((id) => String(id)) }
+    },
+    { session }
+  );
 }
--- a/packages/service/common/mongo/sessionRun.ts
+++ b/packages/service/common/mongo/sessionRun.ts
@@ -1,21 +1,21 @@
-import mongoose, { connectionMongo } from './index';
+import { connectionMongo, ClientSession } from './index';

-export async function mongoSessionTask(
-  fn: (session: mongoose.mongo.ClientSession) => Promise<any>
-) {
+export const mongoSessionRun = async <T = unknown>(fn: (session: ClientSession) => Promise<T>) => {
  const session = await connectionMongo.startSession();
+  session.startTransaction();

  try {
-    session.startTransaction();
-
-    await fn(session);
+    const result = await fn(session);

    await session.commitTransaction();
-    await session.endSession();
+    session.endSession();
+
+    return result as T;
  } catch (error) {
+    console.log(error);
+
    await session.abortTransaction();
-    await session.endSession();
-    console.error(error);
+    session.endSession();
    return Promise.reject(error);
  }
-}
+};
--- a/packages/service/common/system/tools.ts
+++ b/packages/service/common/system/tools.ts
@@ -0,0 +1,7 @@
+import { isIPv6 } from 'net';
+
+export const SERVICE_LOCAL_PORT = `${process.env.PORT || 3000}`;
+export const SERVICE_LOCAL_HOST =
+  process.env.HOSTNAME && isIPv6(process.env.HOSTNAME)
+    ? `[${process.env.HOSTNAME}]:${SERVICE_LOCAL_PORT}`
+    : `${process.env.HOSTNAME || 'localhost'}:${SERVICE_LOCAL_PORT}`;
--- a/packages/service/common/vectorStore/controller.d.ts
+++ b/packages/service/common/vectorStore/controller.d.ts
@@ -16,4 +16,5 @@ export type InsertVectorProps = {
 export type EmbeddingRecallProps = {
  datasetIds: string[];
  similarity?: number;
+  efSearch?: number;
 };
--- a/packages/service/common/vectorStore/controller.ts
+++ b/packages/service/common/vectorStore/controller.ts
@@ -2,6 +2,7 @@
 import { PgVector } from './pg/class';
 import { getVectorsByText } from '../../core/ai/embedding';
 import { InsertVectorProps } from './controller.d';
+import { VectorModelItemType } from '@fastgpt/global/core/ai/model.d';

 const getVectorObj = () => {
  return new PgVector();
@@ -20,7 +21,7 @@ export const insertDatasetDataVector = async ({
  ...props
 }: InsertVectorProps & {
  query: string;
-  model: string;
+  model: VectorModelItemType;
 }) => {
  const { vectors, charsLength } = await getVectorsByText({
    model,
@@ -43,7 +44,7 @@ export const updateDatasetDataVector = async ({
 }: InsertVectorProps & {
  id: string;
  query: string;
-  model: string;
+  model: VectorModelItemType;
 }) => {
  // insert new vector
  const { charsLength, insertId } = await insertDatasetDataVector(props);
--- a/packages/service/common/vectorStore/pg/controller.ts
+++ b/packages/service/common/vectorStore/pg/controller.ts
@@ -121,12 +121,12 @@ export const embeddingRecall = async (
 ): Promise<{
  results: EmbeddingRecallItemType[];
 }> => {
-  const { datasetIds, vectors, limit, similarity = 0, retry = 2 } = props;
+  const { datasetIds, vectors, limit, similarity = 0, retry = 2, efSearch = 100 } = props;

  try {
    const results: any = await PgClient.query(
      `BEGIN;
-        SET LOCAL hnsw.ef_search = ${global.systemEnv.pgHNSWEfSearch || 100};
+        SET LOCAL hnsw.ef_search = ${efSearch};
        select id, collection_id, (vector <#> '[${vectors[0]}]') * -1 AS score 
          from ${PgDatasetTableName} 
          where dataset_id IN (${datasetIds.map((id) => `'${String(id)}'`).join(',')})
--- a/packages/service/core/ai/config.ts
+++ b/packages/service/core/ai/config.ts
@@ -6,10 +6,14 @@ export const baseUrl = process.env.ONEAPI_URL || openaiBaseUrl;

 export const systemAIChatKey = process.env.CHAT_API_KEY || '';

-export const getAIApi = (props?: UserModelSchema['openaiAccount'], timeout = 60000) => {
+export const getAIApi = (props?: {
+  userKey?: UserModelSchema['openaiAccount'];
+  timeout?: number;
+}) => {
+  const { userKey, timeout } = props || {};
  return new OpenAI({
-    apiKey: props?.key || systemAIChatKey,
-    baseURL: props?.baseUrl || baseUrl,
+    apiKey: userKey?.key || systemAIChatKey,
+    baseURL: userKey?.baseUrl || baseUrl,
    httpAgent: global.httpsAgent,
    timeout,
    maxRetries: 2
--- a/packages/service/core/ai/embedding/index.ts
+++ b/packages/service/core/ai/embedding/index.ts
@@ -1,15 +1,13 @@
+import { VectorModelItemType } from '@fastgpt/global/core/ai/model.d';
 import { getAIApi } from '../config';

-export type GetVectorProps = {
-  model: string;
+type GetVectorProps = {
+  model: VectorModelItemType;
  input: string;
 };

 // text to vector
-export async function getVectorsByText({
-  model = 'text-embedding-ada-002',
-  input
-}: GetVectorProps) {
+export async function getVectorsByText({ model, input }: GetVectorProps) {
  if (!input) {
    return Promise.reject({
      code: 500,
@@ -23,7 +21,8 @@ export async function getVectorsByText({
    // input text to vector
    const result = await ai.embeddings
      .create({
-        model,
+        ...model.defaultConfig,
+        model: model.model,
        input: [input]
      })
      .then(async (res) => {
--- a/packages/service/core/ai/functions/createQuestionGuide.ts
+++ b/packages/service/core/ai/functions/createQuestionGuide.ts
@@ -10,10 +10,12 @@ export async function createQuestionGuide({
  messages: ChatMessageItemType[];
  model: string;
 }) {
-  const ai = getAIApi(undefined, 480000);
+  const ai = getAIApi({
+    timeout: 480000
+  });
  const data = await ai.chat.completions.create({
    model: model,
-    temperature: 0,
+    temperature: 0.1,
    max_tokens: 200,
    messages: [
      ...messages,
--- a/packages/service/core/ai/functions/queryExtension.ts
+++ b/packages/service/core/ai/functions/queryExtension.ts
@@ -17,7 +17,9 @@ OUTPUT:
 `;

 export const searchQueryExtension = async ({ query, model }: { query: string; model: string }) => {
-  const ai = getAIApi(undefined, 480000);
+  const ai = getAIApi({
+    timeout: 480000
+  });

  const result = await ai.chat.completions.create({
    model,
--- a/packages/service/core/chat/chatItemSchema.ts
+++ b/packages/service/core/chat/chatItemSchema.ts
@@ -90,7 +90,7 @@ try {
     close custom feedback; 
  */
  ChatItemSchema.index({ appId: 1, chatId: 1, dataId: 1 }, { background: true });
-  ChatItemSchema.index({ time: -1 }, { background: true });
+  ChatItemSchema.index({ time: -1, obj: 1 }, { background: true });
  ChatItemSchema.index({ userGoodFeedback: 1 }, { background: true });
  ChatItemSchema.index({ userBadFeedback: 1 }, { background: true });
  ChatItemSchema.index({ customFeedbacks: 1 }, { background: true });
--- a/packages/service/core/dataset/collection/controller.ts
+++ b/packages/service/core/dataset/collection/controller.ts
@@ -15,6 +15,7 @@ import { delImgByRelatedId } from '../../../common/file/image/controller';
 import { deleteDatasetDataVector } from '../../../common/vectorStore/controller';
 import { delFileByFileIdList } from '../../../common/file/gridfs/controller';
 import { BucketNameEnum } from '@fastgpt/global/common/file/constants';
+import { ClientSession } from '../../../common/mongo';

 export async function createOneCollection({
  teamId,
@@ -35,41 +36,53 @@ export async function createOneCollection({
  hashRawText,
  rawTextLength,
  metadata = {},
+  session,
  ...props
-}: CreateDatasetCollectionParams & { teamId: string; tmbId: string; [key: string]: any }) {
-  const { _id } = await MongoDatasetCollection.create({
-    ...props,
-    teamId,
-    tmbId,
-    parentId: parentId || null,
-    datasetId,
-    name,
-    type,
+}: CreateDatasetCollectionParams & {
+  teamId: string;
+  tmbId: string;
+  [key: string]: any;
+  session?: ClientSession;
+}) {
+  const [collection] = await MongoDatasetCollection.create(
+    [
+      {
+        ...props,
+        teamId,
+        tmbId,
+        parentId: parentId || null,
+        datasetId,
+        name,
+        type,

-    trainingType,
-    chunkSize,
-    chunkSplitter,
-    qaPrompt,
+        trainingType,
+        chunkSize,
+        chunkSplitter,
+        qaPrompt,

-    fileId,
-    rawLink,
+        fileId,
+        rawLink,

-    rawTextLength,
-    hashRawText,
-    metadata
-  });
+        rawTextLength,
+        hashRawText,
+        metadata
+      }
+    ],
+    { session }
+  );

  // create default collection
  if (type === DatasetCollectionTypeEnum.folder) {
    await createDefaultCollection({
      datasetId,
-      parentId: _id,
+      parentId: collection._id,
      teamId,
-      tmbId
+      tmbId,
+      session
    });
  }

-  return _id;
+  return collection;
 }

 // create default collection
@@ -78,34 +91,43 @@ export function createDefaultCollection({
  datasetId,
  parentId,
  teamId,
-  tmbId
+  tmbId,
+  session
 }: {
  name?: '手动录入' | '手动标注';
  datasetId: string;
  parentId?: string;
  teamId: string;
  tmbId: string;
+  session?: ClientSession;
 }) {
-  return MongoDatasetCollection.create({
-    name,
-    teamId,
-    tmbId,
-    datasetId,
-    parentId,
-    type: DatasetCollectionTypeEnum.virtual,
-    trainingType: TrainingModeEnum.chunk,
-    chunkSize: 0,
-    updateTime: new Date('2099')
-  });
+  return MongoDatasetCollection.create(
+    [
+      {
+        name,
+        teamId,
+        tmbId,
+        datasetId,
+        parentId,
+        type: DatasetCollectionTypeEnum.virtual,
+        trainingType: TrainingModeEnum.chunk,
+        chunkSize: 0,
+        updateTime: new Date('2099')
+      }
+    ],
+    { session }
+  );
 }

 /**
 * delete collection and it related data
 */
 export async function delCollectionAndRelatedSources({
-  collections
+  collections,
+  session
 }: {
  collections: (CollectionWithDatasetType | DatasetCollectionSchemaType)[];
+  session: ClientSession;
 }) {
  if (collections.length === 0) return;

@@ -128,24 +150,25 @@ export async function delCollectionAndRelatedSources({
  await delay(2000);

  // delete dataset.datas
-  await MongoDatasetData.deleteMany({ teamId, collectionId: { $in: collectionIds } });
-  // delete pg data
-  await deleteDatasetDataVector({ teamId, collectionIds });
-
-  // delete file and imgs
-  await Promise.all([
-    delImgByRelatedId({
-      teamId,
-      relateIds: relatedImageIds
-    }),
-    delFileByFileIdList({
-      bucketName: BucketNameEnum.dataset,
-      fileIdList
-    })
-  ]);
-
+  await MongoDatasetData.deleteMany({ teamId, collectionId: { $in: collectionIds } }, { session });
+  // delete imgs
+  await delImgByRelatedId({
+    teamId,
+    relateIds: relatedImageIds,
+    session
+  });
  // delete collections
-  await MongoDatasetCollection.deleteMany({
-    _id: { $in: collectionIds }
+  await MongoDatasetCollection.deleteMany(
+    {
+      _id: { $in: collectionIds }
+    },
+    { session }
+  );
+
+  // no session delete: delete files, vector data
+  await deleteDatasetDataVector({ teamId, collectionIds });
+  await delFileByFileIdList({
+    bucketName: BucketNameEnum.dataset,
+    fileIdList
  });
 }
--- a/packages/service/core/dataset/collection/utils.ts
+++ b/packages/service/core/dataset/collection/utils.ts
@@ -9,6 +9,7 @@ import {
  TrainingModeEnum
 } from '@fastgpt/global/core/dataset/constants';
 import { hashStr } from '@fastgpt/global/common/string/tools';
+import { ClientSession } from '../../../common/mongo';

 /**
 * get all collection by top collectionId
@@ -149,17 +150,17 @@ export const getCollectionAndRawText = async ({

 /* link collection start load data */
 export const reloadCollectionChunks = async ({
-  collectionId,
  collection,
  tmbId,
  billId,
-  rawText
+  rawText,
+  session
 }: {
-  collectionId?: string;
-  collection?: CollectionWithDatasetType;
+  collection: CollectionWithDatasetType;
  tmbId: string;
  billId?: string;
  rawText?: string;
+  session: ClientSession;
 }) => {
  const {
    title,
@@ -168,7 +169,6 @@ export const reloadCollectionChunks = async ({
    isSameRawText
  } = await getCollectionAndRawText({
    collection,
-    collectionId,
    newRawText: rawText
  });

@@ -186,6 +186,7 @@ export const reloadCollectionChunks = async ({
    if (col.trainingType === TrainingModeEnum.qa) return col.datasetId.agentModel;
    return Promise.reject('Training model error');
  })();
+
  await MongoDatasetTraining.insertMany(
    chunks.map((item, i) => ({
      teamId: col.teamId,
@@ -199,13 +200,18 @@ export const reloadCollectionChunks = async ({
      q: item,
      a: '',
      chunkIndex: i
-    }))
+    })),
+    { session }
  );

  // update raw text
-  await MongoDatasetCollection.findByIdAndUpdate(col._id, {
-    ...(title && { name: title }),
-    rawTextLength: newRawText.length,
-    hashRawText: hashStr(newRawText)
-  });
+  await MongoDatasetCollection.findByIdAndUpdate(
+    col._id,
+    {
+      ...(title && { name: title }),
+      rawTextLength: newRawText.length,
+      hashRawText: hashStr(newRawText)
+    },
+    { session }
+  );
 };
--- a/packages/service/core/dataset/controller.ts
+++ b/packages/service/core/dataset/controller.ts
@@ -2,6 +2,7 @@ import { CollectionWithDatasetType, DatasetSchemaType } from '@fastgpt/global/co
 import { MongoDatasetCollection } from './collection/schema';
 import { MongoDataset } from './schema';
 import { delCollectionAndRelatedSources } from './collection/controller';
+import { ClientSession } from '../../common/mongo';

 /* ============= dataset ========== */
 /* find all datasetId by top datasetId */
@@ -55,7 +56,13 @@ export async function getCollectionWithDataset(collectionId: string) {
 }

 /* delete all data by datasetIds */
-export async function delDatasetRelevantData({ datasets }: { datasets: DatasetSchemaType[] }) {
+export async function delDatasetRelevantData({
+  datasets,
+  session
+}: {
+  datasets: DatasetSchemaType[];
+  session: ClientSession;
+}) {
  if (!datasets.length) return;

  const teamId = datasets[0].teamId;
@@ -70,5 +77,5 @@ export async function delDatasetRelevantData({ datasets }: { datasets: DatasetSc
    '_id teamId fileId metadata'
  ).lean();

-  await delCollectionAndRelatedSources({ collections });
+  await delCollectionAndRelatedSources({ collections, session });
 }
--- a/packages/service/core/dataset/training/controller.ts
+++ b/packages/service/core/dataset/training/controller.ts
@@ -40,12 +40,12 @@ export async function pushDataListToTrainingQueue({
  trainingMode = TrainingModeEnum.chunk,

  vectorModelList = [],
-  qaModelList = []
+  datasetModelList = []
 }: {
  teamId: string;
  tmbId: string;
  vectorModelList: VectorModelItemType[];
-  qaModelList: LLMModelItemType[];
+  datasetModelList: LLMModelItemType[];
 } & PushDatasetDataProps): Promise<PushDatasetDataResponse> {
  const {
    datasetId: { _id: datasetId, vectorModel, agentModel }
@@ -68,7 +68,7 @@ export async function pushDataListToTrainingQueue({
    }

    if (trainingMode === TrainingModeEnum.qa) {
-      const qaModelData = qaModelList?.find((item) => item.model === agentModel);
+      const qaModelData = datasetModelList?.find((item) => item.model === agentModel);
      if (!qaModelData) {
        return Promise.reject(`Model ${agentModel} is inValid`);
      }
@@ -150,7 +150,7 @@ export async function pushDataListToTrainingQueue({
          model,
          q: item.q,
          a: item.a,
-          chunkIndex: item.chunkIndex ?? i,
+          chunkIndex: item.chunkIndex ?? 0,
          weight: weight ?? 0,
          indexes: item.indexes
        }))
--- a/packages/service/package.json
+++ b/packages/service/package.json
@@ -15,6 +15,7 @@
    "nextjs-cors": "^2.1.2",
    "node-cron": "^3.0.3",
    "pg": "^8.10.0",
+    "date-fns": "^2.30.0",
    "tunnel": "^0.0.6"
  },
  "devDependencies": {
--- a/packages/service/support/openapi/auth.ts
+++ b/packages/service/support/openapi/auth.ts
@@ -17,6 +17,7 @@ export async function authOpenApiKey({ apikey }: { apikey: string }) {
    }

    // auth limit
+    // @ts-ignore
    if (global.feConfigs?.isPlus) {
      await POST('/support/openapi/authLimit', { openApi } as AuthOpenApiLimitProps);
    }
--- a/packages/service/support/openapi/schema.ts
+++ b/packages/service/support/openapi/schema.ts
@@ -68,6 +68,13 @@ const OpenApiSchema = new Schema(
  }
 );

+try {
+  OpenApiSchema.index({ teamId: 1 });
+  OpenApiSchema.index({ apiKey: 1 });
+} catch (error) {
+  console.log(error);
+}
+
 export const MongoOpenApi: Model<OpenApiSchema> =
  models['openapi'] || model('openapi', OpenApiSchema);
 MongoOpenApi.syncIndexes();
--- a/packages/service/support/permission/auth/common.ts
+++ b/packages/service/support/permission/auth/common.ts
@@ -2,7 +2,7 @@ import { AuthUserTypeEnum } from '@fastgpt/global/support/permission/constant';
 import { parseHeaderCert } from '../controller';
 import { AuthModeType } from '../type';
 import { authOutLinkValid } from './outLink';
-import { isIPv6 } from 'net';
+import { SERVICE_LOCAL_HOST } from '../../../common/system/tools';

 export const authCert = async (props: AuthModeType) => {
  const result = await parseHeaderCert(props);
@@ -35,12 +35,7 @@ export async function authCertOrShareId({

 /* auth the request from local service */
 export const authRequestFromLocal = ({ req }: AuthModeType) => {
-  const host =
-    process.env.HOSTNAME && isIPv6(process.env.HOSTNAME)
-      ? `[${process.env.HOSTNAME}]:${process.env.PORT || 3000}`
-      : `${process.env.HOSTNAME || 'localhost'}:${process.env.PORT || 3000}`;
-
-  if (host !== req.headers.host) {
+  if (req.headers.host !== SERVICE_LOCAL_HOST) {
    return Promise.reject('Invalid request');
  }
 };
--- a/packages/service/support/permission/controller.ts
+++ b/packages/service/support/permission/controller.ts
@@ -111,8 +111,7 @@ export async function parseHeaderCert({
    }
  }

-  const { cookie, token, apikey, rootkey, authorization } = (req.headers ||
-    {}) as ReqHeaderAuthType;
+  const { cookie, token, rootkey, authorization } = (req.headers || {}) as ReqHeaderAuthType;

  const { uid, teamId, tmbId, appId, openApiKey, authType } = await (async () => {
    if (authApiKey && authorization) {
@@ -151,19 +150,6 @@ export async function parseHeaderCert({
        authType: AuthUserTypeEnum.root
      };
    }
-    // apikey: abandon
-    if (authApiKey && apikey) {
-      // apikey
-      const parseResult = await authOpenApiKey({ apikey });
-      return {
-        uid: parseResult.userId,
-        teamId: parseResult.teamId,
-        tmbId: parseResult.tmbId,
-        appId: parseResult.appId,
-        openApiKey: parseResult.apikey,
-        authType: AuthUserTypeEnum.apikey
-      };
-    }

    return Promise.reject(ERROR_ENUM.unAuthorization);
  })();
--- a/packages/service/support/permission/limit/dataset.ts
+++ b/packages/service/support/permission/limit/dataset.ts
@@ -1,17 +1,18 @@
+import { StandSubPlanLevelMapType } from '@fastgpt/global/support/wallet/sub/type';
 import { getVectorCountByTeamId } from '../../../common/vectorStore/controller';
-import { getTeamDatasetValidSub } from '../../wallet/sub/utils';
+import { getTeamDatasetMaxSize } from '../../wallet/sub/utils';

 export const checkDatasetLimit = async ({
  teamId,
-  freeSize = Infinity,
-  insertLen = 0
+  insertLen = 0,
+  standardPlans
 }: {
  teamId: string;
-  freeSize?: number;
  insertLen?: number;
+  standardPlans?: StandSubPlanLevelMapType;
 }) => {
  const [{ maxSize }, usedSize] = await Promise.all([
-    getTeamDatasetValidSub({ teamId, freeSize }),
+    getTeamDatasetMaxSize({ teamId, standardPlans }),
    getVectorCountByTeamId(teamId)
  ]);

--- a/packages/service/support/user/team/controller.ts
+++ b/packages/service/support/user/team/controller.ts
@@ -1,5 +1,5 @@
 import { TeamItemType, TeamMemberWithTeamSchema } from '@fastgpt/global/support/user/team/type';
-import { Types } from '../../../common/mongo';
+import { ClientSession, Types } from '../../../common/mongo';
 import {
  TeamMemberRoleEnum,
  TeamMemberStatusEnum,
@@ -55,13 +55,15 @@ export async function createDefaultTeam({
  teamName = 'My Team',
  avatar = '/icon/logo.svg',
  balance,
-  maxSize = 5
+  maxSize = 5,
+  session
 }: {
  userId: string;
  teamName?: string;
  avatar?: string;
  balance?: number;
  maxSize?: number;
+  session: ClientSession;
 }) {
  // auth default team
  const tmb = await MongoTeamMember.findOne({
@@ -73,23 +75,33 @@ export async function createDefaultTeam({
    console.log('create default team', userId);

    // create
-    const { _id: insertedId } = await MongoTeam.create({
-      ownerId: userId,
-      name: teamName,
-      avatar,
-      balance,
-      maxSize,
-      createTime: new Date()
-    });
-    await MongoTeamMember.create({
-      teamId: insertedId,
-      userId,
-      name: 'Owner',
-      role: TeamMemberRoleEnum.owner,
-      status: TeamMemberStatusEnum.active,
-      createTime: new Date(),
-      defaultTeam: true
-    });
+    const [{ _id: insertedId }] = await MongoTeam.create(
+      [
+        {
+          ownerId: userId,
+          name: teamName,
+          avatar,
+          balance,
+          maxSize,
+          createTime: new Date()
+        }
+      ],
+      { session }
+    );
+    await MongoTeamMember.create(
+      [
+        {
+          teamId: insertedId,
+          userId,
+          name: 'Owner',
+          role: TeamMemberRoleEnum.owner,
+          status: TeamMemberStatusEnum.active,
+          createTime: new Date(),
+          defaultTeam: true
+        }
+      ],
+      { session }
+    );
  } else {
    console.log('default team exist', userId);
    await MongoTeam.findByIdAndUpdate(tmb.teamId, {
--- a/packages/service/support/user/team/teamSchema.ts
+++ b/packages/service/support/user/team/teamSchema.ts
@@ -3,7 +3,6 @@ const { Schema, model, models } = connectionMongo;
 import { TeamSchema as TeamType } from '@fastgpt/global/support/user/team/type.d';
 import { userCollectionName } from '../../user/schema';
 import { TeamCollectionName } from '@fastgpt/global/support/user/team/constant';
-import { PRICE_SCALE } from '@fastgpt/global/support/wallet/bill/constants';

 const TeamSchema = new Schema({
  name: {
--- a/packages/service/support/wallet/bill/controller.ts
+++ b/packages/service/support/wallet/bill/controller.ts
@@ -1,5 +1,6 @@
 import { BillSourceEnum } from '@fastgpt/global/support/wallet/bill/constants';
 import { MongoBill } from './schema';
+import { ClientSession } from '../../../common/mongo';

 export const createTrainingBill = async ({
  teamId,
@@ -7,7 +8,8 @@ export const createTrainingBill = async ({
  appName,
  billSource,
  vectorModel,
-  agentModel
+  agentModel,
+  session
 }: {
  teamId: string;
  tmbId: string;
@@ -15,28 +17,34 @@ export const createTrainingBill = async ({
  billSource: `${BillSourceEnum}`;
  vectorModel: string;
  agentModel: string;
+  session?: ClientSession;
 }) => {
-  const { _id } = await MongoBill.create({
-    teamId,
-    tmbId,
-    appName,
-    source: billSource,
-    list: [
+  const [{ _id }] = await MongoBill.create(
+    [
      {
-        moduleName: 'wallet.moduleName.index',
-        model: vectorModel,
-        charsLength: 0,
-        amount: 0
-      },
-      {
-        moduleName: 'wallet.moduleName.qa',
-        model: agentModel,
-        charsLength: 0,
-        amount: 0
+        teamId,
+        tmbId,
+        appName,
+        source: billSource,
+        list: [
+          {
+            moduleName: 'wallet.moduleName.index',
+            model: vectorModel,
+            charsLength: 0,
+            amount: 0
+          },
+          {
+            moduleName: 'wallet.moduleName.qa',
+            model: agentModel,
+            charsLength: 0,
+            amount: 0
+          }
+        ],
+        total: 0
      }
    ],
-    total: 0
-  });
+    { session }
+  );

  return { billId: String(_id) };
 };
--- a/packages/service/support/wallet/sub/schema.ts
+++ b/packages/service/support/wallet/sub/schema.ts
@@ -29,6 +29,15 @@ const SubSchema = new Schema({
    required: true
  },
  mode: {
+    type: String,
+    enum: Object.keys(subModeMap)
+  },
+  currentMode: {
+    type: String,
+    enum: Object.keys(subModeMap),
+    required: true
+  },
+  nextMode: {
    type: String,
    enum: Object.keys(subModeMap),
    required: true
@@ -46,6 +55,10 @@ const SubSchema = new Schema({
    type: Number,
    required: true
  },
+  pointPrice: {
+    // stand level point total price
+    type: Number
+  },

  // sub content
  currentSubLevel: {
@@ -56,6 +69,9 @@ const SubSchema = new Schema({
    type: String,
    enum: Object.keys(standardSubLevelMap)
  },
+  totalPoints: {
+    type: Number
+  },

  currentExtraDatasetSize: {
    type: Number
@@ -72,48 +88,44 @@ const SubSchema = new Schema({
  },

  // standard sub limit
-  maxTeamMember: {
-    type: Number
-  },
-  maxAppAmount: {
-    type: Number
-  },
-  maxDatasetAmount: {
-    type: Number
-  },
-  chatHistoryStoreDuration: {
-    // n day
-    type: Number
-  },
-  maxDatasetSize: {
-    type: Number
-  },
-  trainingWeight: {
-    // 0 1 2 3
-    type: Number
-  },
-  customApiKey: {
-    type: Boolean
-  },
-  customCopyright: {
-    type: Boolean
-  },
-  exportDatasetInterval: {
-    // hours
-    type: Number
-  },
-  websiteSyncInterval: {
-    // hours
-    type: Number
-  },
-  reRankWeight: {
-    // 0 1 2 3
-    type: Number
-  },
-  totalPoints: {
-    // record standard sub points
-    type: Number
-  },
+  // maxTeamMember: {
+  //   type: Number
+  // },
+  // maxAppAmount: {
+  //   type: Number
+  // },
+  // maxDatasetAmount: {
+  //   type: Number
+  // },
+  // chatHistoryStoreDuration: {
+  //   // n day
+  //   type: Number
+  // },
+  // maxDatasetSize: {
+  //   type: Number
+  // },
+  // trainingWeight: {
+  //   // 0 1 2 3
+  //   type: Number
+  // },
+  // customApiKey: {
+  //   type: Boolean
+  // },
+  // customCopyright: {
+  //   type: Boolean
+  // },
+  // websiteSyncInterval: {
+  //   // hours
+  //   type: Number
+  // },
+  // reRankWeight: {
+  //   // 0 1 2 3
+  //   type: Number
+  // },
+  // totalPoints: {
+  //   // record standard sub points
+  //   type: Number
+  // },

  surplusPoints: {
    // standard sub / extra points sub
--- a/packages/service/support/wallet/sub/utils.ts
+++ b/packages/service/support/wallet/sub/utils.ts
@@ -1,28 +1,87 @@
 import { SubTypeEnum } from '@fastgpt/global/support/wallet/sub/constants';
 import { MongoTeamSub } from './schema';
+import { addHours } from 'date-fns';
+import { FeTeamSubType, StandSubPlanLevelMapType } from '@fastgpt/global/support/wallet/sub/type.d';
+import { getVectorCountByTeamId } from '../../../common/vectorStore/controller';

-/* get team dataset size */
-export const getTeamDatasetValidSub = async ({
+/* get team dataset max size */
+export const getTeamDatasetMaxSize = async ({
  teamId,
-  freeSize = Infinity
+  standardPlans
 }: {
  teamId: string;
-  freeSize?: number;
+  standardPlans?: StandSubPlanLevelMapType;
 }) => {
-  const sub = await MongoTeamSub.findOne({
+  if (!standardPlans) {
+    return {
+      maxSize: Infinity,
+      sub: null
+    };
+  }
+
+  const plans = await MongoTeamSub.find({
    teamId,
-    type: SubTypeEnum.extraDatasetSize,
-    expiredTime: { $gte: new Date() }
+    expiredTime: { $gte: addHours(new Date(), -3) }
  }).lean();

-  const maxSize = (() => {
-    if (!sub || !sub.currentExtraDatasetSize) return freeSize;
+  const standard = plans.find((plan) => plan.type === SubTypeEnum.standard);
+  const extraDatasetSize = plans.find((plan) => plan.type === SubTypeEnum.extraDatasetSize);

-    return sub.currentExtraDatasetSize + freeSize;
-  })();
+  const standardMaxDatasetSize =
+    standard?.currentSubLevel && standardPlans
+      ? standardPlans[standard.currentSubLevel]?.maxDatasetSize || Infinity
+      : Infinity;
+  const totalDatasetSize =
+    standardMaxDatasetSize + (extraDatasetSize?.currentExtraDatasetSize || 0);

  return {
-    maxSize,
-    sub
+    maxSize: totalDatasetSize,
+    sub: extraDatasetSize
+  };
+};
+
+export const getTeamSubPlanStatus = async ({
+  teamId,
+  standardPlans
+}: {
+  teamId: string;
+  standardPlans?: StandSubPlanLevelMapType;
+}): Promise<FeTeamSubType> => {
+  const [plans, usedDatasetSize] = await Promise.all([
+    MongoTeamSub.find({ teamId }).lean(),
+    getVectorCountByTeamId(teamId)
+  ]);
+
+  const standard = plans.find((plan) => plan.type === SubTypeEnum.standard);
+  const extraDatasetSize = plans.find((plan) => plan.type === SubTypeEnum.extraDatasetSize);
+  const extraPoints = plans.find((plan) => plan.type === SubTypeEnum.extraPoints);
+
+  const standardMaxDatasetSize =
+    standard?.currentSubLevel && standardPlans
+      ? standardPlans[standard.currentSubLevel]?.maxDatasetSize || Infinity
+      : Infinity;
+  const totalDatasetSize =
+    standardMaxDatasetSize + (extraDatasetSize?.currentExtraDatasetSize || 0);
+
+  const standardMaxPoints =
+    standard?.currentSubLevel && standardPlans
+      ? standardPlans[standard.currentSubLevel]?.totalPoints || Infinity
+      : Infinity;
+  const totalPoints = standardMaxPoints + (extraPoints?.currentExtraPoints || 0);
+
+  const surplusPoints = (standard?.surplusPoints || 0) + (extraPoints?.surplusPoints || 0);
+
+  return {
+    [SubTypeEnum.standard]: standard,
+    [SubTypeEnum.extraDatasetSize]: extraDatasetSize,
+    [SubTypeEnum.extraPoints]: extraPoints,
+
+    standardMaxDatasetSize,
+    datasetMaxSize: totalDatasetSize,
+    usedDatasetSize,
+
+    standardMaxPoints,
+    totalPoints,
+    usedPoints: totalPoints - surplusPoints
  };
 };