feat: pg引入

This commit is contained in:
archer
2023-04-18 22:35:55 +08:00
parent a540ee944a
commit 9e951fbc15
17 changed files with 260 additions and 150 deletions

View File

@@ -6,12 +6,9 @@ import type { ChatCompletionRequestMessage } from 'openai';
import { ChatModelNameEnum } from '@/constants/model';
import { pushSplitDataBill } from '@/service/events/pushBill';
import { generateVector } from './generateVector';
import { connectRedis } from '../redis';
import { VecModelDataPrefix } from '@/constants/redis';
import { customAlphabet } from 'nanoid';
import { openaiError2 } from '../errorCode';
import { connectPg } from '@/service/pg';
import { ModelSplitDataSchema } from '@/types/mongoSchema';
const nanoid = customAlphabet('abcdefghijklmnopqrstuvwxyz1234567890', 12);
export async function generateQA(next = false): Promise<any> {
if (process.env.queueTask !== '1') {
@@ -25,7 +22,7 @@ export async function generateQA(next = false): Promise<any> {
let dataId = null;
try {
const redis = await connectRedis();
const pg = await connectPg();
// 找出一个需要生成的 dataItem
const data = await SplitData.aggregate([
{ $match: { textList: { $exists: true, $ne: [] } } },
@@ -139,23 +136,17 @@ export async function generateQA(next = false): Promise<any> {
SplitData.findByIdAndUpdate(dataItem._id, {
textList: dataItem.textList.slice(0, -5)
}), // 删掉后5个数据
...resultList.map((item) => {
// 插入 redis
return redis.sendCommand([
'HMSET',
`${VecModelDataPrefix}:${nanoid()}`,
'userId',
String(dataItem.userId),
'modelId',
String(dataItem.modelId),
'q',
item.q,
'text',
item.a,
'status',
'waiting'
]);
})
// 生成的内容插入 pg
pg.query(`INSERT INTO modelData (user_id, model_id, q, a, status) VALUES ${resultList
.map(
(item) =>
`('${String(dataItem.userId)}', '${String(dataItem.modelId)}', '${item.q.replace(
/\'/g,
'"'
)}', '${item.a.replace(/\'/g, '"')}', 'waiting')`
)
.join(',')}
`)
]);
console.log('生成QA成功time:', `${(Date.now() - startTime) / 1000}s`);