From fadb3e3ceba988ac2d751a74916a70e277aac211 Mon Sep 17 00:00:00 2001 From: Archer <545436317@qq.com> Date: Fri, 10 Jan 2025 14:41:29 +0800 Subject: [PATCH] feat: tmp upgrade code (#3559) * feat: tmp upgrade code * fulltext search test * update action --- .github/workflows/preview-fastgpt-image.yml | 4 ++-- packages/global/common/error/code/user.ts | 5 ----- packages/service/core/dataset/data/dataTextSchema.ts | 12 +++++++----- packages/service/core/dataset/data/schema.ts | 4 ++-- packages/service/core/dataset/search/controller.ts | 1 + packages/service/support/permission/auth/team.ts | 2 +- projects/app/src/pages/api/admin/initv4818.ts | 12 ++++++------ .../api/support/user/account/loginByPassword.ts | 2 +- .../app/src/service/core/dataset/data/controller.ts | 6 ++++-- 9 files changed, 24 insertions(+), 24 deletions(-) diff --git a/.github/workflows/preview-fastgpt-image.yml b/.github/workflows/preview-fastgpt-image.yml index 4db6e43ad..9e5780112 100644 --- a/.github/workflows/preview-fastgpt-image.yml +++ b/.github/workflows/preview-fastgpt-image.yml @@ -36,7 +36,7 @@ jobs: password: ${{ secrets.GH_PAT }} - name: Set DOCKER_REPO_TAGGED based on branch or tag run: | - echo "DOCKER_REPO_TAGGED=ghcr.io/${{ github.repository_owner }}/fastgpt-pr:${{ github.event.pull_request.number }}" >> $GITHUB_ENV + echo "DOCKER_REPO_TAGGED=ghcr.io/${{ github.repository_owner }}/fastgpt-pr:${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV - name: Build image for PR env: DOCKER_REPO_TAGGED: ${{ env.DOCKER_REPO_TAGGED }} @@ -44,7 +44,7 @@ jobs: docker buildx build \ -f projects/app/Dockerfile \ --label "org.opencontainers.image.source=https://github.com/${{ github.repository_owner }}/FastGPT" \ - --label "org.opencontainers.image.description=fastgpt-pr imae" \ + --label "org.opencontainers.image.description=fastgpt-pr image" \ --label "org.opencontainers.image.licenses=Apache" \ --push \ --cache-from=type=local,src=/tmp/.buildx-cache \ diff --git a/packages/global/common/error/code/user.ts b/packages/global/common/error/code/user.ts index ebd782ddf..df9475e93 100644 --- a/packages/global/common/error/code/user.ts +++ b/packages/global/common/error/code/user.ts @@ -2,17 +2,12 @@ import { ErrType } from '../errorCode'; import { i18nT } from '../../../../web/i18n/utils'; /* team: 503000 */ export enum UserErrEnum { - unAuthUser = 'unAuthUser', unAuthRole = 'unAuthRole', binVisitor = 'binVisitor', balanceNotEnough = 'balanceNotEnough', unAuthSso = 'unAuthSso' } const errList = [ - { - statusText: UserErrEnum.unAuthUser, - message: i18nT('common:code_error.user_error.un_auth_user') - }, { statusText: UserErrEnum.binVisitor, message: i18nT('common:code_error.user_error.bin_visitor') diff --git a/packages/service/core/dataset/data/dataTextSchema.ts b/packages/service/core/dataset/data/dataTextSchema.ts index c7384392a..a3c57c2b2 100644 --- a/packages/service/core/dataset/data/dataTextSchema.ts +++ b/packages/service/core/dataset/data/dataTextSchema.ts @@ -29,14 +29,16 @@ const DatasetDataTextSchema = new Schema({ ref: DatasetDataCollectionName, required: true }, - fullTextToken: { - type: String, - required: true - } + fullTextToken: String }); try { - DatasetDataTextSchema.index({ teamId: 1, datasetId: 1, fullTextToken: 'text' }); + DatasetDataTextSchema.index( + { teamId: 1, datasetId: 1, fullTextToken: 'text' }, + { + partialFilterExpression: { fullTextToken: { $exists: true } } + } + ); DatasetDataTextSchema.index({ dataId: 1 }, { unique: true }); } catch (error) { console.log(error); diff --git a/packages/service/core/dataset/data/schema.ts b/packages/service/core/dataset/data/schema.ts index c21cf479f..3b7ac8819 100644 --- a/packages/service/core/dataset/data/schema.ts +++ b/packages/service/core/dataset/data/schema.ts @@ -86,8 +86,8 @@ try { chunkIndex: 1, updateTime: -1 }); - // full text index - // DatasetDataSchema.index({ teamId: 1, datasetId: 1, fullTextToken: 'text' }); + // FullText tmp full text index + DatasetDataSchema.index({ teamId: 1, datasetId: 1, fullTextToken: 'text' }); // Recall vectors after data matching DatasetDataSchema.index({ teamId: 1, datasetId: 1, collectionId: 1, 'indexes.dataId': 1 }); DatasetDataSchema.index({ updateTime: 1 }); diff --git a/packages/service/core/dataset/search/controller.ts b/packages/service/core/dataset/search/controller.ts index 45c5889bd..0de4c2884 100644 --- a/packages/service/core/dataset/search/controller.ts +++ b/packages/service/core/dataset/search/controller.ts @@ -636,6 +636,7 @@ export async function searchDatasetData(props: SearchDatasetDataProps) { forbidCollectionIdList, filterCollectionIdList }), + // FullText tmp fullTextRecall2({ query, limit: fullTextLimit, diff --git a/packages/service/support/permission/auth/team.ts b/packages/service/support/permission/auth/team.ts index b659f2ffd..5bec5635a 100644 --- a/packages/service/support/permission/auth/team.ts +++ b/packages/service/support/permission/auth/team.ts @@ -18,7 +18,7 @@ export async function getUserChatInfoAndAuthTeamPoints(tmbId: string) { ]) .lean(); - if (!tmb) return Promise.reject(UserErrEnum.unAuthUser); + if (!tmb) return Promise.reject(UserErrEnum.binVisitor); await checkTeamAIPoints(tmb.team._id); diff --git a/projects/app/src/pages/api/admin/initv4818.ts b/projects/app/src/pages/api/admin/initv4818.ts index dab3ed704..196ecd589 100644 --- a/projects/app/src/pages/api/admin/initv4818.ts +++ b/projects/app/src/pages/api/admin/initv4818.ts @@ -59,12 +59,12 @@ const initData = async (batchSize: number) => { })), { ordered: false, session, lean: true } ); - // 把成功插入的新数据的 dataId 更新为已初始化 - await MongoDatasetData.updateMany( - { _id: { $in: result.map((item) => item.dataId) } }, - { $set: { initFullText: true }, $unset: { fullTextToken: 1 } }, - { session } - ); + // FullText tmp 把成功插入的新数据的 dataId 更新为已初始化 + // await MongoDatasetData.updateMany( + // { _id: { $in: result.map((item) => item.dataId) } }, + // { $set: { initFullText: true }, $unset: { fullTextToken: 1 } }, + // { session } + // ); success += result.length; diff --git a/projects/app/src/pages/api/support/user/account/loginByPassword.ts b/projects/app/src/pages/api/support/user/account/loginByPassword.ts index 82d1c3e26..fd5ee3f02 100644 --- a/projects/app/src/pages/api/support/user/account/loginByPassword.ts +++ b/projects/app/src/pages/api/support/user/account/loginByPassword.ts @@ -25,7 +25,7 @@ async function handler(req: NextApiRequest, res: NextApiResponse) { 'status' ); if (!authCert) { - return Promise.reject(UserErrEnum.unAuthUser); + return Promise.reject(UserErrEnum.binVisitor); } if (authCert.status === UserStatusEnum.forbidden) { diff --git a/projects/app/src/service/core/dataset/data/controller.ts b/projects/app/src/service/core/dataset/data/controller.ts index abedf5817..c3a607d77 100644 --- a/projects/app/src/service/core/dataset/data/controller.ts +++ b/projects/app/src/service/core/dataset/data/controller.ts @@ -89,7 +89,8 @@ export async function insertData2Dataset({ collectionId, q, a, - // fullTextToken: jiebaSplit({ text: qaStr }), + // FullText tmp + fullTextToken: jiebaSplit({ text: qaStr }), chunkIndex, indexes: indexes?.map((item, i) => ({ ...item, @@ -241,7 +242,8 @@ export async function updateData2Dataset({ // update mongo other data mongoData.q = q || mongoData.q; mongoData.a = a ?? mongoData.a; - // mongoData.fullTextToken = jiebaSplit({ text: `${mongoData.q}\n${mongoData.a}`.trim() }); + // FullText tmp + mongoData.fullTextToken = jiebaSplit({ text: `${mongoData.q}\n${mongoData.a}`.trim() }); // @ts-ignore mongoData.indexes = newIndexes; await mongoData.save({ session });