gitea/FastGPT
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: kb crud

Browse Source
This commit is contained in:
archer
2023-05-17 19:30:43 +08:00
parent 021add2af4
commit a79429fdcd
57 changed files with 1186 additions and 788 deletions
Download Patch File Download Diff File Expand all files Collapse all files

34
src/pages/api/model/data/fetchingUrlData.ts
View File

@@ -1,34 +0,0 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase } from '@/service/mongo';
import { authToken } from '@/service/utils/auth';
import axios from 'axios';
import { axiosConfig } from '@/service/utils/tools';
/**
* 读取网站的内容
*/
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
const { url } = req.body as { url: string };
if (!url) {
throw new Error('缺少 url');
}
await connectToDatabase();
await authToken(req);
const data = await axios
.get(url, {
httpsAgent: axiosConfig().httpsAgent
})
.then((res) => res.data as string);
jsonRes(res, { data });
} catch (err) {
jsonRes(res, {
code: 500,
error: err
});
}
}

54
src/pages/api/model/data/pushModelDataInput.ts
View File

@@ -1,54 +0,0 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase } from '@/service/mongo';
import { authToken } from '@/service/utils/auth';
import { ModelDataSchema } from '@/types/mongoSchema';
import { generateVector } from '@/service/events/generateVector';
import { PgClient } from '@/service/pg';
import { authModel } from '@/service/utils/auth';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
const { modelId, data } = req.body as {
modelId: string;
data: { a: ModelDataSchema['a']; q: ModelDataSchema['q'] }[];
};
if (!modelId || !Array.isArray(data)) {
throw new Error('缺少参数');
}
// 凭证校验
const userId = await authToken(req);
await connectToDatabase();
// 验证是否是该用户的 model
await authModel({
userId,
modelId
});
// 插入记录
await PgClient.insert('modelData', {
values: data.map((item) => [
{ key: 'user_id', value: userId },
{ key: 'model_id', value: modelId },
{ key: 'q', value: item.q },
{ key: 'a', value: item.a },
{ key: 'status', value: 'waiting' }
])
});
generateVector();
jsonRes(res, {
data: 0
});
} catch (err) {
jsonRes(res, {
code: 500,
error: err
});
}
}

6
src/pages/api/model/del.ts
View File

@@ -2,7 +2,6 @@ import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { Chat, Model, connectToDatabase, Collection, ShareChat } from '@/service/mongo';
import { authToken } from '@/service/utils/auth';
import { PgClient } from '@/service/pg';
import { authModel } from '@/service/utils/auth';
/* 获取我的模型 */
@@ -25,11 +24,6 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
userId
});
// 删除 pg 中所有该模型的数据
await PgClient.delete('modelData', {
where: [['user_id', userId], 'AND', ['model_id', modelId]]
});
// 删除对应的聊天
await Chat.deleteMany({
modelId

0
src/pages/api/model/data/delModelDataById.ts → src/pages/api/openapi/kb/delDataById.ts
View File

59
src/pages/api/model/data/pushModelDataCsv.ts → src/pages/api/openapi/kb/pushData.ts
View File

@@ -1,20 +1,25 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import type { KbDataItemType } from '@/types/plugin';
import { jsonRes } from '@/service/response';
import { connectToDatabase } from '@/service/mongo';
import { authToken } from '@/service/utils/auth';
import { generateVector } from '@/service/events/generateVector';
import { ModelDataStatusEnum } from '@/constants/model';
import { PgClient } from '@/service/pg';
import { authModel } from '@/service/utils/auth';
import { authKb } from '@/service/utils/auth';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
const { modelId, data } = req.body as {
modelId: string;
data: string[][];
const {
kbId,
data,
formatLineBreak = true
} = req.body as {
kbId: string;
formatLineBreak?: boolean;
data: { a: KbDataItemType['a']; q: KbDataItemType['q'] }[];
};
if (!modelId || !Array.isArray(data)) {
if (!kbId || !Array.isArray(data)) {
throw new Error('缺少参数');
}
@@ -23,31 +28,27 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
await connectToDatabase();
// 验证是否是该用户的 model
await authModel({
await authKb({
userId,
modelId
kbId
});
// 去重
// 过滤重复的内容
const searchRes = await Promise.allSettled(
data.map(async ([q, a = '']) => {
data.map(async ({ q, a = '' }) => {
if (!q) {
return Promise.reject('q为空');
}
try {
if (formatLineBreak) {
q = q.replace(/\\n/g, '\n');
a = a.replace(/\\n/g, '\n');
}
// Exactly the same data, not push
try {
const count = await PgClient.count('modelData', {
where: [
['user_id', userId],
'AND',
['model_id', modelId],
'AND',
['q', q],
'AND',
['a', a]
]
where: [['user_id', userId], 'AND', ['kb_id', kbId], 'AND', ['q', q], 'AND', ['a', a]]
});
if (count > 0) {
return Promise.reject('已经存在');
@@ -61,25 +62,25 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
});
})
);
// 过滤重复的内容
const filterData = searchRes
.filter((item) => item.status === 'fulfilled')
.map<{ q: string; a: string }>((item: any) => item.value);
// 插入 pg
// 插入记录
const insertRes = await PgClient.insert('modelData', {
values: filterData.map((item) => [
{ key: 'user_id', value: userId },
{ key: 'model_id', value: modelId },
{ key: 'kb_id', value: kbId },
{ key: 'q', value: item.q },
{ key: 'a', value: item.a },
{ key: 'status', value: ModelDataStatusEnum.waiting }
{ key: 'status', value: 'waiting' }
])
});
generateVector();
jsonRes(res, {
message: `共插入 ${insertRes.rowCount} 条数据`,
data: insertRes.rowCount
});
} catch (err) {
@@ -89,11 +90,3 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
});
}
}
export const config = {
api: {
bodyParser: {
sizeLimit: '100mb'
}
}
};

2
src/pages/api/model/data/putModelData.ts → src/pages/api/openapi/kb/updateData.ts
View File

@@ -16,7 +16,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
// 凭证校验
const userId = await authToken(req);
// 更新 pg 内容
// 更新 pg 内容.仅修改a，不需要更新向量。
await PgClient.update('modelData', {
where: [['id', dataId], 'AND', ['user_id', userId]],
values: [

26
src/pages/api/model/data/splitData.ts → src/pages/api/openapi/text/splitText.ts
View File

@@ -1,21 +1,22 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase, SplitData } from '@/service/mongo';
import { authModel, authToken } from '@/service/utils/auth';
import { authKb, authToken } from '@/service/utils/auth';
import { generateVector } from '@/service/events/generateVector';
import { generateQA } from '@/service/events/generateQA';
import { PgClient } from '@/service/pg';
import { SplitTextTypEnum } from '@/constants/plugin';
/* 拆分数据成QA */
/* split text */
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
const { chunks, modelId, prompt, mode } = req.body as {
modelId: string;
const { chunks, kbId, prompt, mode } = req.body as {
kbId: string;
chunks: string[];
prompt: string;
mode: 'qa' | 'subsection';
mode: `${SplitTextTypEnum}`;
};
if (!chunks || !modelId || !prompt) {
if (!chunks || !kbId || !prompt) {
throw new Error('参数错误');
}
await connectToDatabase();
@@ -23,27 +24,28 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
const userId = await authToken(req);
// 验证是否是该用户的 model
await authModel({
modelId,
await authKb({
kbId,
userId
});
if (mode === 'qa') {
if (mode === SplitTextTypEnum.qa) {
// 批量QA拆分插入数据
await SplitData.create({
userId,
modelId,
kbId,
textList: chunks,
prompt
});
generateQA();
} else if (mode === 'subsection') {
} else if (mode === SplitTextTypEnum.subsection) {
// 待优化，直接调用另一个接口
// 插入记录
await PgClient.insert('modelData', {
values: chunks.map((item) => [
{ key: 'user_id', value: userId },
{ key: 'model_id', value: modelId },
{ key: 'kb_id', value: kbId },
{ key: 'q', value: item },
{ key: 'a', value: '' },
{ key: 'status', value: 'waiting' }

35
src/pages/api/plugins/kb/create.ts Normal file
View File

@@ -0,0 +1,35 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase, KB } from '@/service/mongo';
import { authToken } from '@/service/utils/auth';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
const { name, tags } = req.body as {
name: string;
tags: string[];
};
if (!name) {
throw new Error('缺少参数');
}
// 凭证校验
const userId = await authToken(req);
await connectToDatabase();
const { _id } = await KB.create({
name,
userId,
tags
});
jsonRes(res, { data: _id });
} catch (err) {
jsonRes(res, {
code: 500,
error: err
});
}
}

10
src/pages/api/model/data/exportModelData.ts → src/pages/api/plugins/kb/data/exportModelData.ts
View File

@@ -6,11 +6,11 @@ import { PgClient } from '@/service/pg';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
let { modelId } = req.query as {
modelId: string;
let { kbId } = req.query as {
kbId: string;
};
if (!modelId) {
if (!kbId) {
throw new Error('缺少参数');
}
@@ -21,11 +21,11 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
// 统计数据
const count = await PgClient.count('modelData', {
where: [['model_id', modelId], 'AND', ['user_id', userId]]
where: [['kb_id', kbId], 'AND', ['user_id', userId]]
});
// 从 pg 中获取所有数据
const pgData = await PgClient.select<{ q: string; a: string }>('modelData', {
where: [['model_id', modelId], 'AND', ['user_id', userId]],
where: [['kb_id', kbId], 'AND', ['user_id', userId]],
fields: ['q', 'a'],
order: [{ field: 'id', mode: 'DESC' }],
limit: count

32
src/pages/api/model/data/getModelData.ts → src/pages/api/plugins/kb/data/getDataList.ts
View File

@@ -3,27 +3,22 @@ import { jsonRes } from '@/service/response';
import { connectToDatabase } from '@/service/mongo';
import { authToken } from '@/service/utils/auth';
import { PgClient } from '@/service/pg';
import type { PgModelDataItemType } from '@/types/pg';
import { authModel } from '@/service/utils/auth';
import type { PgKBDataItemType } from '@/types/pg';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
let {
modelId,
kbId,
pageNum = 1,
pageSize = 10,
searchText = ''
} = req.query as {
modelId: string;
pageNum: string;
pageSize: string;
} = req.body as {
kbId: string;
pageNum: number;
pageSize: number;
searchText: string;
};
pageNum = +pageNum;
pageSize = +pageSize;
if (!modelId) {
if (!kbId) {
throw new Error('缺少参数');
}
@@ -32,19 +27,14 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
await connectToDatabase();
const { model } = await authModel({
userId,
modelId,
authOwner: false
});
const where: any = [
...(model.share.isShareDetail ? [] : [['user_id', userId], 'AND']),
['model_id', modelId],
['user_id', userId],
'AND',
['kb_id', kbId],
...(searchText ? ['AND', `(q LIKE '%${searchText}%' OR a LIKE '%${searchText}%')`] : [])
];
const searchRes = await PgClient.select<PgModelDataItemType>('modelData', {
const searchRes = await PgClient.select<PgKBDataItemType>('modelData', {
fields: ['id', 'q', 'a', 'status'],
where,
order: [{ field: 'id', mode: 'DESC' }],

26
src/pages/api/model/data/getTrainingData.ts → src/pages/api/plugins/kb/data/getTrainingData.ts
View File

@@ -8,8 +8,8 @@ import { PgClient } from '@/service/pg';
/* 拆分数据成QA */
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
const { modelId } = req.query as { modelId: string };
if (!modelId) {
const { kbId } = req.query as { kbId: string };
if (!kbId) {
throw new Error('参数错误');
}
await connectToDatabase();
@@ -19,25 +19,25 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
// split queue data
const data = await SplitData.find({
userId,
modelId,
kbId,
textList: { $exists: true, $not: { $size: 0 } }
});
// embedding queue data
const where: any = [
['user_id', userId],
'AND',
['model_id', modelId],
'AND',
['status', ModelDataStatusEnum.waiting]
];
const embeddingData = await PgClient.count('modelData', {
where: [
['user_id', userId],
'AND',
['kb_id', kbId],
'AND',
['status', ModelDataStatusEnum.waiting]
]
});
jsonRes(res, {
data: {
splitDataQueue: data.map((item) => item.textList).flat().length,
embeddingQueue: await PgClient.count('modelData', {
where
})
embeddingQueue: embeddingData
}
});
} catch (err) {

43
src/pages/api/plugins/kb/delete.ts Normal file
View File

@@ -0,0 +1,43 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase, KB } from '@/service/mongo';
import { authToken } from '@/service/utils/auth';
import { PgClient } from '@/service/pg';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
const { id } = req.query as {
id: string;
};
if (!id) {
throw new Error('缺少参数');
}
// 凭证校验
const userId = await authToken(req);
await connectToDatabase();
// delete mongo data
await KB.findOneAndDelete({
_id: id,
userId
});
// delete all pg data
// 删除 pg 中所有该模型的数据
await PgClient.delete('modelData', {
where: [['user_id', userId], 'AND', ['kb_id', id]]
});
// delete related model
jsonRes(res);
} catch (err) {
jsonRes(res, {
code: 500,
error: err
});
}
}

42
src/pages/api/plugins/kb/list.ts Normal file
View File

@@ -0,0 +1,42 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase, KB } from '@/service/mongo';
import { authToken } from '@/service/utils/auth';
import { PgClient } from '@/service/pg';
import { KbItemType } from '@/types/plugin';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
// 凭证校验
const userId = await authToken(req);
await connectToDatabase();
const kbList = await KB.find({
userId
}).sort({ updateTime: -1 });
const data = await Promise.all(
kbList.map(async (item) => ({
_id: item._id,
avatar: item.avatar,
name: item.name,
userId: item.userId,
updateTime: item.updateTime,
tags: item.tags.join(' '),
totalData: await PgClient.count('modelData', {
where: [['user_id', userId], 'AND', ['kb_id', item._id]]
})
}))
);
jsonRes<KbItemType[]>(res, {
data
});
} catch (err) {
jsonRes(res, {
code: 500,
error: err
});
}
}

39
src/pages/api/plugins/kb/update.ts Normal file
View File

@@ -0,0 +1,39 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase, KB } from '@/service/mongo';
import { authToken } from '@/service/utils/auth';
import type { KbUpdateParams } from '@/api/plugins/kb';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
const { id, name, tags, avatar } = req.body as KbUpdateParams;
if (!id || !name) {
throw new Error('缺少参数');
}
// 凭证校验
const userId = await authToken(req);
await connectToDatabase();
await KB.findOneAndUpdate(
{
_id: id,
userId
},
{
avatar,
name,
tags: tags.split(' ').filter((item) => item)
}
);
jsonRes(res);
} catch (err) {
jsonRes(res, {
code: 500,
error: err
});
}
}