gitea/FastGPT
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

v4.4.1 (#294)

Browse Source 
* move file

* perf: dataset file manage

* v441 description

* fix: qa csv update file

* feat: rename file

* frontend show system-version
This commit is contained in:
Archer
2023-09-13 17:00:17 +08:00
committed by GitHub
parent be3b680bc6
commit a19afca148
53 changed files with 570 additions and 301 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
client/src/service/lib/gridfs.ts
View File

@@ -17,6 +17,9 @@ export class GridFSStorage {
this.bucket = bucket;
this.uid = String(uid);
}
Collection() {
return mongoose.connection.db.collection(`${this.bucket}.files`);
}
GridFSBucket() {
return new mongoose.mongo.GridFSBucket(mongoose.connection.db, {
bucketName: this.bucket

4
client/src/service/moduleDispatch/kb/search.ts
View File

@@ -5,7 +5,7 @@ import { getVector } from '@/pages/api/openapi/plugin/vector';
import { countModelPrice } from '@/service/events/pushBill';
import type { SelectedKbType } from '@/types/plugin';
import type { QuoteItemType } from '@/types/chat';
import { PgTrainingTableName } from '@/constants/plugin';
import { PgDatasetTableName } from '@/constants/plugin';
type KBSearchProps = {
kbList: SelectedKbType;
@@ -42,7 +42,7 @@ export async function dispatchKBSearch(props: Record<string, any>): Promise<KBSe
const res: any = await PgClient.query(
`BEGIN;
SET LOCAL ivfflat.probes = ${global.systemEnv.pgIvfflatProbe || 10};
select kb_id,id,q,a,source,file_id from ${PgTrainingTableName} where kb_id IN (${kbList
select kb_id,id,q,a,source,file_id from ${PgDatasetTableName} where kb_id IN (${kbList
.map((item) => `'${item.kbId}'`)
.join(',')}) AND vector <#> '[${vectors[0]}]' < -${similarity} order by vector <#> '[${
vectors[0]

12
client/src/service/pg.ts
View File

@@ -1,6 +1,6 @@
import { Pool } from 'pg';
import type { QueryResultRow } from 'pg';
import { PgTrainingTableName } from '@/constants/plugin';
import { PgDatasetTableName } from '@/constants/plugin';
import { addLog } from './utils/tools';
import { DatasetItemType } from '@/types/plugin';
@@ -174,7 +174,7 @@ export const insertKbItem = ({
vector: number[];
})[];
}) => {
return PgClient.insert(PgTrainingTableName, {
return PgClient.insert(PgDatasetTableName, {
values: data.map((item) => [
{ key: 'user_id', value: userId },
{ key: 'kb_id', value: kbId },
@@ -192,7 +192,7 @@ export async function initPg() {
await connectPg();
await PgClient.query(`
CREATE EXTENSION IF NOT EXISTS vector;
CREATE TABLE IF NOT EXISTS ${PgTrainingTableName} (
CREATE TABLE IF NOT EXISTS ${PgDatasetTableName} (
id BIGSERIAL PRIMARY KEY,
vector VECTOR(1536) NOT NULL,
user_id VARCHAR(50) NOT NULL,
@@ -202,9 +202,9 @@ export async function initPg() {
q TEXT NOT NULL,
a TEXT
);
CREATE INDEX IF NOT EXISTS modelData_userId_index ON ${PgTrainingTableName} USING HASH (user_id);
CREATE INDEX IF NOT EXISTS modelData_kbId_index ON ${PgTrainingTableName} USING HASH (kb_id);
CREATE INDEX IF NOT EXISTS idx_model_data_md5_q_a_user_id_kb_id ON ${PgTrainingTableName} (md5(q), md5(a), user_id, kb_id);
CREATE INDEX IF NOT EXISTS modelData_userId_index ON ${PgDatasetTableName} USING HASH (user_id);
CREATE INDEX IF NOT EXISTS modelData_kbId_index ON ${PgDatasetTableName} USING HASH (kb_id);
CREATE INDEX IF NOT EXISTS idx_model_data_md5_q_a_user_id_kb_id ON ${PgDatasetTableName} (md5(q), md5(a), user_id, kb_id);
`);
console.log('init pg successful');
} catch (error) {