Compare commits

..

18 Commits

Author SHA1 Message Date
Archer
c5db894e3a update doc (#4460)
* update preview action

* update doc

* remove

* update

* schema
2025-04-07 17:12:53 +08:00
archer
4fc96f6534 preview doc action
add docs preview permission

update preview action

udpate action
2025-04-07 13:57:54 +08:00
Archer
7ee616515d fix: ts (#4458) 2025-04-07 10:55:53 +08:00
archer
4207ecaaaa action 2025-04-07 10:29:19 +08:00
heheer
32ae14f281 fix external var ui (#4444) 2025-04-07 09:55:15 +08:00
archer
9670cba607 Update doc 2025-04-07 09:55:15 +08:00
archer
95d5c01db1 update doc 2025-04-07 09:55:14 +08:00
Archer
915569fe79 Test mongo log (#4443)
* feat: mongodb-log (#4426)

* perf: mongo log

* feat: completions stop reasoner

* mongo db log

---------

Co-authored-by: Finley Ge <32237950+FinleyGe@users.noreply.github.com>
2025-04-07 09:55:14 +08:00
archer
252ad8a611 package 2025-04-07 09:55:14 +08:00
Archer
85ce23869d Test completion v2 (#4438)
* add v2 completions (#4364)

* add v2 completions

* completion config

* config version

* fix

* frontend

* doc

* fix

* fix: completions v2 api

---------

Co-authored-by: heheer <heheer@sealos.io>
2025-04-07 09:55:13 +08:00
Archer
e4c4941a50 perf: mobile voice input (#4437)
* update:Mobile voice interaction (#4362)

* Add files via upload

* Add files via upload

* Update ollama.md

* Update ollama.md

* Add files via upload

* Update useSpeech.ts

* Update ChatInput.tsx

* Update useSpeech.ts

* Update ChatInput.tsx

* Update useSpeech.ts

* Update constants.ts

* Add files via upload

* Update ChatInput.tsx

* Update useSpeech.ts

* Update useSpeech.ts

* Update useSpeech.ts

* Update ChatInput.tsx

* Add files via upload

* Update common.json

* Update VoiceInput.tsx

* Update ChatInput.tsx

* Update VoiceInput.tsx

* Update useSpeech.ts

* Update useSpeech.ts

* Update common.json

* Update common.json

* Update common.json

* Update VoiceInput.tsx

* Update VoiceInput.tsx

* Update ChatInput.tsx

* Update VoiceInput.tsx

* Update ChatInput.tsx

* Update VoiceInput.tsx

* Update ChatInput.tsx

* Update useSpeech.ts

* Update common.json

* Update chat.json

* Update common.json

* Update chat.json

* Update common.json

* Update chat.json

* Update VoiceInput.tsx

* Update ChatInput.tsx

* Update useSpeech.ts

* Update VoiceInput.tsx

* speech ui

* 优化语音输入组件,调整输入框显示逻辑,修复语音输入遮罩层样式,更新画布背景透明度,增强用户交互体验。 (#4435)

* perf: mobil voice input

---------

Co-authored-by: dreamer6680 <1468683855@qq.com>
2025-04-07 09:55:13 +08:00
Archer
c2e088cf39 Redis cache (#4436)
* perf: add Redis cache for vector counting (#4432)

* feat: cache

* perf: get cache key

---------

Co-authored-by: a.e. <49438478+I-Info@users.noreply.github.com>
2025-04-07 09:55:13 +08:00
archer
dc73f47486 feat: config chat file expired time 2025-04-07 09:55:13 +08:00
archer
2f9b83d924 perf: remove loading ui 2025-04-07 09:55:12 +08:00
Archer
9fe95da126 pro migration (#4388) (#4433)
* pro migration

* reuse customPdfParseType

Co-authored-by: gggaaallleee <91131304+gggaaallleee@users.noreply.github.com>
2025-04-07 09:55:12 +08:00
Archer
d171b2d3d8 website sync feature (#4429)
* perf: introduce BullMQ for website sync (#4403)

* perf: introduce BullMQ for website sync

* feat: new redis module

* fix: remove graceful shutdown

* perf: improve UI in dataset detail

- Updated the "change" icon SVG file.
- Modified i18n strings.
- Added new i18n string "immediate_sync".
- Improved UI in dataset detail page, including button icons and
background colors.

* refactor: Add chunkSettings to DatasetSchema

* perf: website sync ux

* env template

* fix: clean up website dataset when updating chunk settings (#4420)

* perf: check setting updated

* perf: worker currency

* feat: init script for website sync refactor (#4425)

* website feature doc

---------

Co-authored-by: a.e. <49438478+I-Info@users.noreply.github.com>
2025-04-07 09:55:11 +08:00
archer
e54fe1eed6 doc 2025-04-07 09:55:10 +08:00
Archer
27332743c7 Training status (#4424)
* dataset data training state (#4311)

* dataset data training state

* fix

* fix ts

* fix

* fix api format

* fix

* fix

* perf: count training

* format

* fix: dataset training state (#4417)

* fix

* add test

* fix

* fix

* fix test

* fix test

* perf: training count

* count

* loading status

---------

Co-authored-by: heheer <heheer@sealos.io>
2025-04-07 09:55:07 +08:00
45 changed files with 710 additions and 646 deletions

View File

@@ -1,4 +1,4 @@
yangchuansheng/fastgpt-imgs:
- source: docSite/assets/imgs/
dest: imgs/
deleteOrphaned: true
deleteOrphaned: true

View File

@@ -1,4 +1,4 @@
name: Deploy doc image to cf
name: Deploy doc image to vercel
on:
workflow_dispatch:
@@ -63,6 +63,18 @@ jobs:
- name: Build
run: cd docSite && hugo mod get -u github.com/colinwilson/lotusdocs@6d0568e && hugo -v --minify
# Step 5 - Push our generated site to Vercel
- name: Deploy to Vercel
uses: amondnet/vercel-action@v25
id: vercel-action
with:
vercel-token: ${{ secrets.VERCEL_TOKEN }} # Required
vercel-org-id: ${{ secrets.VERCEL_ORG_ID }} #Required
vercel-project-id: ${{ secrets.VERCEL_PROJECT_ID }} #Required
github-comment: false
vercel-args: '--prod --local-config ../vercel.json' # Optional
working-directory: docSite/public
- name: Deploy to GitHub Pages
uses: peaceiris/actions-gh-pages@v4
if: github.ref == 'refs/heads/main'

View File

@@ -1,7 +1,7 @@
name: Preview FastGPT docs
on:
pull_request_target:
pull_request:
paths:
- 'docSite/**'
workflow_dispatch:
@@ -50,6 +50,10 @@ jobs:
- 'docSite/content/docs/**'
base: main
- name: Add cdn for images
run: |
sed -i "s#\](/imgs/#\](https://cdn.jsdelivr.net/gh/yangchuansheng/fastgpt-imgs@main/imgs/#g" $(grep -rl "\](/imgs/" docSite/content/zh-cn/docs)
# Step 3 - Install Hugo (specific version)
- name: Install Hugo
uses: peaceiris/actions-hugo@v2
@@ -61,6 +65,9 @@ jobs:
- name: Build
run: cd docSite && hugo mod get -u github.com/colinwilson/lotusdocs@6d0568e && hugo -v --minify
- name: Test
run: ls ./docSite/public
# Step 5 - Push our generated site to Cloudflare
- name: Deploy to Cloudflare Pages
id: deploy

View File

@@ -34,94 +34,6 @@ weight: 852
### 请求
{{< tabs tabTotal="3" >}}
{{< tab tabName="基础请求示例" >}}
{{< markdownify >}}
```bash
curl --location --request POST 'http://localhost:3000/api/v1/chat/completions' \
--header 'Authorization: Bearer fastgpt-xxxxxx' \
--header 'Content-Type: application/json' \
--data-raw '{
"chatId": "my_chatId",
"stream": false,
"detail": false,
"responseChatItemId": "my_responseChatItemId",
"variables": {
"uid": "asdfadsfasfd2323",
"name": "张三"
},
"messages": [
{
"role": "user",
"content": "导演是谁"
}
]
}'
```
{{< /markdownify >}}
{{< /tab >}}
{{< tab tabName="图片/文件请求示例" >}}
{{< markdownify >}}
*`messages`有部分区别,其他参数一致。
* 目前不支持上传文件,需上传到自己的对象存储中,获取对应的文件链接。
```bash
curl --location --request POST 'http://localhost:3000/api/v1/chat/completions' \
--header 'Authorization: Bearer fastgpt-xxxxxx' \
--header 'Content-Type: application/json' \
--data-raw '{
"chatId": "abcd",
"stream": false,
"messages": [
{
"role": "user",
"content": [
{
"type": "text",
"text": "导演是谁"
},
{
"type": "image_url",
"image_url": {
"url": "图片链接"
}
},
{
"type": "file_url",
"name": "文件名",
"url": "文档链接,支持 txt md html word pdf ppt csv excel"
}
]
}
]
}'
```
{{< /markdownify >}}
{{< /tab >}}
{{< tab tabName="参数说明" >}}
{{< markdownify >}}
{{% alert context="info" %}}
- headers.Authorization: Bearer {{apikey}}
- chatId: string | undefined 。
-`undefined` 时(不传入),不使用 FastGpt 提供的上下文功能,完全通过传入的 messages 构建上下文。
-`非空字符串`时,意味着使用 chatId 进行对话,自动从 FastGpt 数据库取历史记录,并使用 messages 数组最后一个内容作为用户问题,其余 message 会被忽略。请自行确保 chatId 唯一长度小于250通常可以是自己系统的对话框ID。
- messages: 结构与 [GPT接口](https://platform.openai.com/docs/api-reference/chat/object) chat模式一致。
- responseChatItemId: string | undefined 。如果传入,则会将该值作为本次对话的响应消息的 IDFastGPT 会自动将该 ID 存入数据库。请确保,在当前`chatId`下,`responseChatItemId`是唯一的。
- detail: 是否返回中间值(模块状态,响应的完整结果等),`stream模式`下会通过`event`进行区分,`非stream模式`结果保存在`responseData`中。
- variables: 模块变量,一个对象,会替换模块中,输入框内容里的`{{key}}`
{{% /alert %}}
{{< /markdownify >}}
{{< /tab >}}
{{< /tabs >}}
<!-- #### v2
v1,v2 接口请求参数一致,仅请求地址不一样。
@@ -216,7 +128,93 @@ curl --location --request POST 'http://localhost:3000/api/v2/chat/completions' \
#### v1
{{< tabs tabTotal="3" >}}
{{< tab tabName="基础请求示例" >}}
{{< markdownify >}}
```bash
curl --location --request POST 'http://localhost:3000/api/v1/chat/completions' \
--header 'Authorization: Bearer fastgpt-xxxxxx' \
--header 'Content-Type: application/json' \
--data-raw '{
"chatId": "my_chatId",
"stream": false,
"detail": false,
"responseChatItemId": "my_responseChatItemId",
"variables": {
"uid": "asdfadsfasfd2323",
"name": "张三"
},
"messages": [
{
"role": "user",
"content": "导演是谁"
}
]
}'
```
{{< /markdownify >}}
{{< /tab >}}
{{< tab tabName="图片/文件请求示例" >}}
{{< markdownify >}}
*`messages`有部分区别,其他参数一致。
* 目前不支持上传文件,需上传到自己的对象存储中,获取对应的文件链接。
```bash
curl --location --request POST 'http://localhost:3000/api/v1/chat/completions' \
--header 'Authorization: Bearer fastgpt-xxxxxx' \
--header 'Content-Type: application/json' \
--data-raw '{
"chatId": "abcd",
"stream": false,
"messages": [
{
"role": "user",
"content": [
{
"type": "text",
"text": "导演是谁"
},
{
"type": "image_url",
"image_url": {
"url": "图片链接"
}
},
{
"type": "file_url",
"name": "文件名",
"url": "文档链接,支持 txt md html word pdf ppt csv excel"
}
]
}
]
}'
```
{{< /markdownify >}}
{{< /tab >}}
{{< tab tabName="参数说明" >}}
{{< markdownify >}}
{{% alert context="info" %}}
- headers.Authorization: Bearer {{apikey}}
- chatId: string | undefined 。
-`undefined` 时(不传入),不使用 FastGpt 提供的上下文功能,完全通过传入的 messages 构建上下文。
-`非空字符串`时,意味着使用 chatId 进行对话,自动从 FastGpt 数据库取历史记录,并使用 messages 数组最后一个内容作为用户问题,其余 message 会被忽略。请自行确保 chatId 唯一长度小于250通常可以是自己系统的对话框ID。
- messages: 结构与 [GPT接口](https://platform.openai.com/docs/api-reference/chat/object) chat模式一致。
- responseChatItemId: string | undefined 。如果传入,则会将该值作为本次对话的响应消息的 IDFastGPT 会自动将该 ID 存入数据库。请确保,在当前`chatId`下,`responseChatItemId`是唯一的。
- detail: 是否返回中间值(模块状态,响应的完整结果等),`stream模式`下会通过`event`进行区分,`非stream模式`结果保存在`responseData`中。
- variables: 模块变量,一个对象,会替换模块中,输入框内容里的`{{key}}`
{{% /alert %}}
{{< /markdownify >}}
{{< /tab >}}
{{< /tabs >}}
### 响应
@@ -747,6 +745,8 @@ curl --location --request POST 'https://api.fastgpt.in/api/v1/chat/completions'
### 请求示例
#### v1
```bash
curl --location --request POST 'http://localhost:3000/api/v1/chat/completions' \
--header 'Authorization: Bearer test-xxxxx' \
@@ -760,8 +760,25 @@ curl --location --request POST 'http://localhost:3000/api/v1/chat/completions' \
}'
```
#### v2
```bash
curl --location --request POST 'http://localhost:3000/api/v2/chat/completions' \
--header 'Authorization: Bearer test-xxxxx' \
--header 'Content-Type: application/json' \
--data-raw '{
"stream": false,
"chatId": "test",
"variables": {
"query":"你好"
}
}'
```
### 响应示例
#### v1
{{< tabs tabTotal="3" >}}
{{< tab tabName="detail=true,stream=false 响应" >}}
@@ -920,6 +937,151 @@ event取值
{{< /tab >}}
{{< /tabs >}}
#### v2
{{< tabs tabTotal="3" >}}
{{< tab tabName="detail=true,stream=false 响应" >}}
{{< markdownify >}}
* 插件的输出可以通过查找`responseData`中, `moduleType=pluginOutput`的元素,其`pluginOutput`是插件的输出。
* 流输出,仍可以通过`choices`进行获取。
```json
{
"responseData": [
{
"id": "bsH1ZdbYkz9iJwYa",
"nodeId": "pluginInput",
"moduleName": "workflow:template.plugin_start",
"moduleType": "pluginInput",
"runningTime": 0
},
{
"id": "zDgfqSPhbYZFHVIn",
"nodeId": "h4Gr4lJtFVQ6qI4c",
"moduleName": "AI 对话",
"moduleType": "chatNode",
"runningTime": 1.44,
"totalPoints": 0,
"model": "GPT-4o-mini",
"tokens": 34,
"inputTokens": 8,
"outputTokens": 26,
"query": "你好",
"reasoningText": "",
"historyPreview": [
{
"obj": "Human",
"value": "你好"
},
{
"obj": "AI",
"value": "你好!有什么我可以帮助你的吗?"
}
],
"contextTotalLen": 2
},
{
"id": "uLLwKKRZvufXzgF4",
"nodeId": "pluginOutput",
"moduleName": "common:core.module.template.self_output",
"moduleType": "pluginOutput",
"runningTime": 0,
"totalPoints": 0,
"pluginOutput": {
"result": "你好!有什么我可以帮助你的吗?"
}
}
],
"newVariables": {
},
"id": "test",
"model": "",
"usage": {
"prompt_tokens": 1,
"completion_tokens": 1,
"total_tokens": 1
},
"choices": [
{
"message": {
"role": "assistant",
"content": "你好!有什么我可以帮助你的吗?"
},
"finish_reason": "stop",
"index": 0
}
]
}
```
{{< /markdownify >}}
{{< /tab >}}
{{< tab tabName="detail=true,stream=true 响应" >}}
{{< markdownify >}}
* 插件的输出可以通过获取`event=flowResponses`中的字符串,并将其反序列化后得到一个数组。同样的,查找 `moduleType=pluginOutput`的元素,其`pluginOutput`是插件的输出。
* 流输出,仍和对话接口一样获取。
```bash
data: {"event":"flowNodeResponse","data":"{\"id\":\"q8ablUOqHGgqLIRM\",\"nodeId\":\"pluginInput\",\"moduleName\":\"workflow:template.plugin_start\",\"moduleType\":\"pluginInput\",\"runningTime\":0}"}
data: {"event":"flowNodeStatus","data":"{\"status\":\"running\",\"name\":\"AI 对话\"}"}
data: {"event":"answer","data":"{\"id\":\"\",\"object\":\"\",\"created\":0,\"model\":\"\",\"choices\":[{\"delta\":{\"role\":\"assistant\",\"content\":\"你好\"},\"index\":0,\"finish_reason\":null}]}"}
data: {"event":"answer","data":"{\"id\":\"\",\"object\":\"\",\"created\":0,\"model\":\"\",\"choices\":[{\"delta\":{\"role\":\"assistant\",\"content\":\"\"},\"index\":0,\"finish_reason\":null}]}"}
data: {"event":"answer","data":"{\"id\":\"\",\"object\":\"\",\"created\":0,\"model\":\"\",\"choices\":[{\"delta\":{\"role\":\"assistant\",\"content\":\"有什么\"},\"index\":0,\"finish_reason\":null}]}"}
data: {"event":"answer","data":"{\"id\":\"\",\"object\":\"\",\"created\":0,\"model\":\"\",\"choices\":[{\"delta\":{\"role\":\"assistant\",\"content\":\"我\"},\"index\":0,\"finish_reason\":null}]}"}
data: {"event":"answer","data":"{\"id\":\"\",\"object\":\"\",\"created\":0,\"model\":\"\",\"choices\":[{\"delta\":{\"role\":\"assistant\",\"content\":\"可以\"},\"index\":0,\"finish_reason\":null}]}"}
data: {"event":"answer","data":"{\"id\":\"\",\"object\":\"\",\"created\":0,\"model\":\"\",\"choices\":[{\"delta\":{\"role\":\"assistant\",\"content\":\"帮助\"},\"index\":0,\"finish_reason\":null}]}"}
data: {"event":"answer","data":"{\"id\":\"\",\"object\":\"\",\"created\":0,\"model\":\"\",\"choices\":[{\"delta\":{\"role\":\"assistant\",\"content\":\"你\"},\"index\":0,\"finish_reason\":null}]}"}
data: {"event":"answer","data":"{\"id\":\"\",\"object\":\"\",\"created\":0,\"model\":\"\",\"choices\":[{\"delta\":{\"role\":\"assistant\",\"content\":\"的吗\"},\"index\":0,\"finish_reason\":null}]}"}
data: {"event":"answer","data":"{\"id\":\"\",\"object\":\"\",\"created\":0,\"model\":\"\",\"choices\":[{\"delta\":{\"role\":\"assistant\",\"content\":\"\"},\"index\":0,\"finish_reason\":null}]}"}
data: {"event":"flowNodeResponse","data":"{\"id\":\"rqlXLUap8QeiN7Kf\",\"nodeId\":\"h4Gr4lJtFVQ6qI4c\",\"moduleName\":\"AI 对话\",\"moduleType\":\"chatNode\",\"runningTime\":1.79,\"totalPoints\":0,\"model\":\"GPT-4o-mini\",\"tokens\":137,\"inputTokens\":111,\"outputTokens\":26,\"query\":\"你好\",\"reasoningText\":\"\",\"historyPreview\":[{\"obj\":\"Human\",\"value\":\"[{\\\"renderTypeList\\\":[\\\"reference\\\"],\\\"selectedTypeInde\\n\\n...[hide 174 chars]...\\n\\ncanSelectImg\\\":true,\\\"required\\\":false,\\\"value\\\":\\\"你好\\\"}]\"},{\"obj\":\"AI\",\"value\":\"你好!有什么我可以帮助你的吗?\"},{\"obj\":\"Human\",\"value\":\"你好\"},{\"obj\":\"AI\",\"value\":\"你好!有什么我可以帮助你的吗?\"}],\"contextTotalLen\":4}"}
data: {"event":"flowNodeResponse","data":"{\"id\":\"lHCpHI0MrM00HQlX\",\"nodeId\":\"pluginOutput\",\"moduleName\":\"common:core.module.template.self_output\",\"moduleType\":\"pluginOutput\",\"runningTime\":0,\"totalPoints\":0,\"pluginOutput\":{\"result\":\"你好!有什么我可以帮助你的吗?\"}}"}
data: {"event":"answer","data":"{\"id\":\"\",\"object\":\"\",\"created\":0,\"model\":\"\",\"choices\":[{\"delta\":{\"role\":\"assistant\",\"content\":null},\"index\":0,\"finish_reason\":\"stop\"}]}"}
data: {"event":"answer","data":"[DONE]"}
```
{{< /markdownify >}}
{{< /tab >}}
{{< tab tabName="输出获取" >}}
{{< markdownify >}}
event取值
- answer: 返回给客户端的文本(最终会算作回答)
- fastAnswer: 指定回复返回给客户端的文本(最终会算作回答)
- toolCall: 执行工具
- toolParams: 工具参数
- toolResponse: 工具返回
- flowNodeStatus: 运行到的节点状态
- flowNodeResponse: 单个节点详细响应
- updateVariables: 更新变量
- error: 报错
{{< /markdownify >}}
{{< /tab >}}
{{< /tabs >}}
# 对话 CRUD
{{% alert icon="🤖 " context="success" %}}

View File

@@ -22,10 +22,6 @@ weight: 796
### 2. 更新镜像 tag
- 更新 FastGPT 镜像 tag: v4.9.4-alpha
- 更新 FastGPT 商业版镜像 tag: v4.9.4-alpha
- Sandbox 无需更新
- AIProxy 无需更新
### 3. 执行升级脚本
@@ -51,7 +47,6 @@ curl --location --request POST 'https://{{host}}/api/admin/initv494' \
4. 利用 redis 进行部分数据缓存。
5. 站点同步支持配置训练参数。
6. AI 对话/工具调用,增加返回模型 finish_reason 字段。
7. 移动端语音输入交互调整
## ⚙️ 优化
@@ -61,6 +56,4 @@ curl --location --request POST 'https://{{host}}/api/admin/initv494' \
## 🐛 修复
1. 搜索应用/知识库时,无法点击目录进入下一层。
2. 重新训练时,参数未成功初始化。
3. package/service 部分请求在多 app 中不一致。
1. 搜索应用/知识库时,无法点击目录进入下一层。

View File

@@ -7,7 +7,7 @@ toc: true
weight: -10
---
FastGPT 是一个AI Agent 构建平台,提供开箱即用的数据处理、模型调用等能力,同时可以通过 Flow 可视化进行工作流编排,从而实现复杂的应用场景!
FastGPT 是一个AI Agent 构建平台,提供开箱即用的数据处理、模型调用等能力,同时可以通过 Flow 可视化进行工作流编排,从而实现复杂的应用场景!t
{{% alert icon="🤖 " context="success" %}}
FastGPT 在线使用:[https://tryfastgpt.ai](https://tryfastgpt.ai)

View File

@@ -51,8 +51,7 @@ export const DatasetTypeMap = {
export enum DatasetStatusEnum {
active = 'active',
syncing = 'syncing',
waiting = 'waiting',
error = 'error'
waiting = 'waiting'
}
export const DatasetStatusMap = {
[DatasetStatusEnum.active]: {
@@ -63,9 +62,6 @@ export const DatasetStatusMap = {
},
[DatasetStatusEnum.waiting]: {
label: i18nT('common:core.dataset.status.waiting')
},
[DatasetStatusEnum.error]: {
label: i18nT('dataset:status_error')
}
};

View File

@@ -209,7 +209,6 @@ export type DatasetListItemType = {
export type DatasetItemType = Omit<DatasetSchemaType, 'vectorModel' | 'agentModel' | 'vlmModel'> & {
status: `${DatasetStatusEnum}`;
errorMsg?: string;
vectorModel: EmbeddingModelItemType;
agentModel: LLMModelItemType;
vlmModel?: LLMModelItemType;

View File

@@ -1,6 +1,7 @@
import { POST } from './plusRequest';
export const postTextCensor = (data: { text: string }) =>
global
.textCensorHandler(data)
POST<{ code?: number; message: string }>('/common/censor/check', data)
.then((res) => {
if (res?.code === 5000) {
return Promise.reject(res);

View File

@@ -1,29 +0,0 @@
import { FeishuServer, YuqueServer } from '@fastgpt/global/core/dataset/apiDataset';
import {
DeepRagSearchProps,
SearchDatasetDataResponse
} from '../../core/dataset/search/controller';
import { AuthOpenApiLimitProps } from '../../support/openapi/auth';
import { CreateUsageProps, ConcatUsageProps } from '@fastgpt/global/support/wallet/usage/api';
import {
GetProApiDatasetFileContentParams,
GetProApiDatasetFileListParams,
GetProApiDatasetFilePreviewUrlParams
} from '../../core/dataset/apiDataset/proApi';
declare global {
var textCensorHandler: (params: { text: string }) => Promise<{ code: number; message?: string }>;
var deepRagHandler: (data: DeepRagSearchProps) => Promise<SearchDatasetDataResponse>;
var authOpenApiHandler: (data: AuthOpenApiLimitProps) => Promise<any>;
var createUsageHandler: (data: CreateUsageProps) => Promise<void>;
var concatUsageHandler: (data: ConcatUsageProps) => Promise<void>;
// API dataset
var getProApiDatasetFileList: (data: GetProApiDatasetFileListParams) => Promise<APIFileItem[]>;
var getProApiDatasetFileContent: (
data: GetProApiDatasetFileContentParams
) => Promise<ApiFileReadContentResponse>;
var getProApiDatasetFilePreviewUrl: (
data: GetProApiDatasetFilePreviewUrlParams
) => Promise<string>;
}

View File

@@ -69,11 +69,6 @@ export function getWorker<DataType, ReturnType = void>(
newWorker.on('error', (error) => {
addLog.error(`MQ Worker [${name}]: ${error.message}`, error);
});
newWorker.on('failed', (jobId, error) => {
addLog.error(`MQ Worker [${name}]: ${error.message}`, error);
});
workers.set(name, newWorker);
return newWorker;
}
export * from 'bullmq';

View File

@@ -1,3 +1,4 @@
export const FastGPTProUrl = process.env.PRO_URL ? `${process.env.PRO_URL}/api` : '';
export const isFastGPTMainService = !!process.env.PRO_URL;
// @ts-ignore
export const isFastGPTProService = () => !!global.systemConfig;

View File

@@ -7,8 +7,6 @@ import { EmbeddingModelItemType } from '@fastgpt/global/core/ai/model.d';
import { MILVUS_ADDRESS, PG_ADDRESS, OCEANBASE_ADDRESS } from './constants';
import { MilvusCtrl } from './milvus/class';
import { setRedisCache, getRedisCache, delRedisCache, CacheKeyEnum } from '../redis/cache';
import { throttle } from 'lodash';
import { retryFn } from '@fastgpt/global/common/system/utils';
const getVectorObj = () => {
if (PG_ADDRESS) return new PgVectorCtrl();
@@ -17,12 +15,7 @@ const getVectorObj = () => {
return new PgVectorCtrl();
};
const getChcheKey = (teamId: string) => `${CacheKeyEnum.team_vector_count}:${teamId}`;
const onDelCache = throttle((teamId: string) => delRedisCache(getChcheKey(teamId)), 30000, {
leading: true,
trailing: true
});
const Vector = getVectorObj();
@@ -56,28 +49,26 @@ export const insertDatasetDataVector = async ({
query: string;
model: EmbeddingModelItemType;
}) => {
return retryFn(async () => {
const { vectors, tokens } = await getVectorsByText({
model,
input: query,
type: 'db'
});
const { insertId } = await Vector.insert({
...props,
vector: vectors[0]
});
onDelCache(props.teamId);
return {
tokens,
insertId
};
const { vectors, tokens } = await getVectorsByText({
model,
input: query,
type: 'db'
});
const { insertId } = await Vector.insert({
...props,
vector: vectors[0]
});
delRedisCache(getChcheKey(props.teamId));
return {
tokens,
insertId
};
};
export const deleteDatasetDataVector = async (props: DelDatasetVectorCtrlProps) => {
const result = await Vector.delete(props);
onDelCache(props.teamId);
delRedisCache(getChcheKey(props.teamId));
return result;
};

View File

@@ -1,25 +0,0 @@
import { ParentIdType } from '@fastgpt/global/common/parentFolder/type';
import { FeishuServer, YuqueServer } from '@fastgpt/global/core/dataset/apiDataset';
export enum ProApiDatasetOperationTypeEnum {
LIST = 'list',
READ = 'read',
CONTENT = 'content'
}
export type ProApiDatasetCommonParams = {
feishuServer?: FeishuServer;
yuqueServer?: YuqueServer;
};
export type GetProApiDatasetFileListParams = ProApiDatasetCommonParams & {
parentId?: ParentIdType;
};
export type GetProApiDatasetFileContentParams = ProApiDatasetCommonParams & {
apiFileId: string;
};
export type GetProApiDatasetFilePreviewUrlParams = ProApiDatasetCommonParams & {
apiFileId: string;
};

View File

@@ -9,6 +9,7 @@ import { readRawContentByFileBuffer } from '../../common/file/read/utils';
import { parseFileExtensionFromUrl } from '@fastgpt/global/common/string/tools';
import { APIFileServer, FeishuServer, YuqueServer } from '@fastgpt/global/core/dataset/apiDataset';
import { useApiDatasetRequest } from './apiDataset/api';
import { POST } from '../../common/api/plusRequest';
export const readFileRawTextByUrl = async ({
teamId,
@@ -167,7 +168,11 @@ export const readApiServerFileContent = async ({
}
if (feishuServer || yuqueServer) {
return global.getProApiDatasetFileContent({
return POST<{
title?: string;
rawText: string;
}>(`/core/dataset/systemApiDataset`, {
type: 'content',
feishuServer,
yuqueServer,
apiFileId

View File

@@ -24,6 +24,7 @@ import { MongoDatasetCollectionTags } from '../tag/schema';
import { readFromSecondary } from '../../../common/mongo/utils';
import { MongoDatasetDataText } from '../data/dataTextSchema';
import { ChatItemType } from '@fastgpt/global/core/chat/type';
import { POST } from '../../../common/api/plusRequest';
import { NodeInputKeyEnum } from '@fastgpt/global/core/workflow/constants';
import { datasetSearchQueryExtension } from './utils';
import type { RerankModelItemType } from '@fastgpt/global/core/ai/model.d';
@@ -849,4 +850,5 @@ export type DeepRagSearchProps = SearchDatasetDataProps & {
[NodeInputKeyEnum.datasetDeepSearchMaxTimes]?: number;
[NodeInputKeyEnum.datasetDeepSearchBg]?: string;
};
export const deepRagSearch = (data: DeepRagSearchProps) => global.deepRagHandler(data);
export const deepRagSearch = (data: DeepRagSearchProps) =>
POST<SearchDatasetDataResponse>('/core/dataset/deepRag', data);

View File

@@ -34,44 +34,23 @@ export const addWebsiteSyncJob = (data: WebsiteSyncJobData) => {
export const getWebsiteSyncDatasetStatus = async (datasetId: string) => {
const jobId = await websiteSyncQueue.getDeduplicationJobId(datasetId);
if (!jobId) {
return {
status: DatasetStatusEnum.active,
errorMsg: undefined
};
return DatasetStatusEnum.active;
}
const job = await websiteSyncQueue.getJob(jobId);
if (!job) {
return {
status: DatasetStatusEnum.active,
errorMsg: undefined
};
return DatasetStatusEnum.active;
}
const jobState = await job.getState();
if (jobState === 'failed' || jobState === 'unknown') {
return {
status: DatasetStatusEnum.error,
errorMsg: job.failedReason
};
}
if (['waiting-children', 'waiting'].includes(jobState)) {
return {
status: DatasetStatusEnum.waiting,
errorMsg: undefined
};
return DatasetStatusEnum.waiting;
}
if (jobState === 'active') {
return {
status: DatasetStatusEnum.syncing,
errorMsg: undefined
};
return DatasetStatusEnum.syncing;
}
return {
status: DatasetStatusEnum.active,
errorMsg: undefined
};
return DatasetStatusEnum.active;
};
// Scheduler setting

View File

@@ -29,9 +29,9 @@ import { InteractiveNodeResponseType } from '@fastgpt/global/core/workflow/templ
import { getFileContentFromLinks, getHistoryFileLinks } from '../../tools/readFiles';
import { parseUrlToFileType } from '@fastgpt/global/common/file/tools';
import { FlowNodeTypeEnum } from '@fastgpt/global/core/workflow/node/constant';
import { postTextCensor } from '../../../../../common/api/requestPlusApi';
import { ModelTypeEnum } from '@fastgpt/global/core/ai/model';
import { getDocumentQuotePrompt } from '@fastgpt/global/core/ai/prompt/AIChat';
import { postTextCensor } from '../../../../chat/postTextCensor';
type Response = DispatchNodeResultType<{
[NodeOutputKeyEnum.answerText]: string;

View File

@@ -13,6 +13,7 @@ import type {
} from '@fastgpt/global/core/ai/type.d';
import { formatModelChars2Points } from '../../../../support/wallet/usage/utils';
import type { LLMModelItemType } from '@fastgpt/global/core/ai/model.d';
import { postTextCensor } from '../../../../common/api/requestPlusApi';
import { ChatCompletionRequestMessageRoleEnum } from '@fastgpt/global/core/ai/constants';
import type {
ChatDispatchProps,
@@ -50,7 +51,6 @@ import { getFileContentFromLinks, getHistoryFileLinks } from '../tools/readFiles
import { parseUrlToFileType } from '@fastgpt/global/common/file/tools';
import { i18nT } from '../../../../../web/i18n/utils';
import { ModelTypeEnum } from '@fastgpt/global/core/ai/model';
import { postTextCensor } from '../../../chat/postTextCensor';
export type ChatProps = ModuleDispatchProps<
AIChatNodeProps & {

View File

@@ -33,7 +33,7 @@ export const dispatchLoop = async (props: Props): Promise<Response> => {
? Number(process.env.WORKFLOW_MAX_LOOP_TIMES)
: 50;
if (loopInputArray.length > maxLength) {
return Promise.reject(`Input array length cannot be greater than ${maxLength}`);
return Promise.reject('Input array length cannot be greater than 50');
}
const outputValueArr = [];

View File

@@ -34,7 +34,7 @@
"node-cron": "^3.0.3",
"node-xlsx": "^0.24.0",
"papaparse": "5.4.1",
"pdfjs-dist": "4.10.38",
"pdfjs-dist": "4.4.168",
"pg": "^8.10.0",
"request-ip": "^3.3.0",
"tiktoken": "1.0.17",

View File

@@ -1,6 +1,7 @@
import { ERROR_ENUM } from '@fastgpt/global/common/error/errorCode';
import { updateApiKeyUsedTime } from './tools';
import { MongoOpenApi } from './schema';
import { POST } from '../../common/api/plusRequest';
import type { OpenApiSchema } from '@fastgpt/global/support/openapi/type';
export type AuthOpenApiLimitProps = { openApi: OpenApiSchema };
@@ -16,10 +17,11 @@ export async function authOpenApiKey({ apikey }: { apikey: string }) {
}
// auth limit
// @ts-ignore
if (global.feConfigs?.isPlus) {
await global.authOpenApiHandler({
await POST('/support/openapi/authLimit', {
openApi
});
} as AuthOpenApiLimitProps);
}
updateApiKeyUsedTime(openApi._id);

View File

@@ -1,21 +1,67 @@
import { UsageSourceEnum } from '@fastgpt/global/support/wallet/usage/constants';
import { MongoUsage } from './schema';
import { ClientSession } from '../../../common/mongo';
import { ClientSession, Types } from '../../../common/mongo';
import { addLog } from '../../../common/system/log';
import { ChatNodeUsageType } from '@fastgpt/global/support/wallet/bill/type';
import { ConcatUsageProps, CreateUsageProps } from '@fastgpt/global/support/wallet/usage/api';
import { i18nT } from '../../../../web/i18n/utils';
import { pushConcatBillTask, pushReduceTeamAiPointsTask } from './utils';
import { POST } from '../../../common/api/plusRequest';
import { isFastGPTMainService } from '../../../common/system/constants';
export async function createUsage(data: CreateUsageProps) {
try {
await global.createUsageHandler(data);
// In FastGPT server
if (isFastGPTMainService) {
await POST('/support/wallet/usage/createUsage', data);
} else if (global.reduceAiPointsQueue) {
// In FastGPT pro server
await MongoUsage.create(data);
pushReduceTeamAiPointsTask({ teamId: data.teamId, totalPoints: data.totalPoints });
if (data.totalPoints === 0) {
addLog.info('0 totalPoints', data);
}
}
} catch (error) {
addLog.error('createUsage error', error);
}
}
export async function concatUsage(data: ConcatUsageProps) {
try {
await global.concatUsageHandler(data);
// In FastGPT server
if (isFastGPTMainService) {
await POST('/support/wallet/usage/concatUsage', data);
} else if (global.reduceAiPointsQueue) {
const {
teamId,
billId,
totalPoints = 0,
listIndex,
inputTokens = 0,
outputTokens = 0
} = data;
// billId is required and valid
if (!billId || !Types.ObjectId.isValid(billId)) return;
// In FastGPT pro server
pushConcatBillTask([
{
billId,
listIndex,
inputTokens,
outputTokens,
totalPoints
}
]);
pushReduceTeamAiPointsTask({ teamId, totalPoints });
if (data.totalPoints === 0) {
addLog.info('0 totalPoints', data);
}
}
} catch (error) {
addLog.error('concatUsage error', error);
}

View File

@@ -1,5 +1,6 @@
import { findAIModel } from '../../../core/ai/model';
import { ModelTypeEnum } from '@fastgpt/global/core/ai/model';
import { ConcatBillQueueItemType } from './type';
export const formatModelChars2Points = ({
model,
@@ -34,3 +35,20 @@ export const formatModelChars2Points = ({
totalPoints
};
};
export const pushReduceTeamAiPointsTask = ({
teamId,
totalPoints
}: {
teamId: string;
totalPoints: number;
}) => {
global.reduceAiPointsQueue.push({
teamId: String(teamId),
totalPoints
});
};
export const pushConcatBillTask = (data: ConcatBillQueueItemType[]) => {
global.concatBillQueue.push(...data);
};

View File

@@ -97,7 +97,7 @@
"permission.des.read": "View knowledge base content",
"permission.des.write": "Ability to add and change knowledge base content",
"preview_chunk": "Preview chunks",
"preview_chunk_empty": "File content is empty",
"preview_chunk_empty": "Unable to read the contents of the file",
"preview_chunk_intro": "A total of {{total}} blocks, up to 10",
"preview_chunk_not_selected": "Click on the file on the left to preview",
"process.Auto_Index": "Automatic index generation",
@@ -124,7 +124,6 @@
"split_sign_question": "question mark",
"split_sign_semicolon": "semicolon",
"start_sync_website_tip": "Confirm to start synchronizing data? \nThe old data will be deleted and retrieved again, please confirm!",
"status_error": "Running exception",
"sync_collection_failed": "Synchronization collection error, please check whether the source file can be accessed normally",
"sync_schedule": "Timing synchronization",
"sync_schedule_tip": "Only existing collections will be synchronized. \nIncludes linked collections and all collections in the API knowledge base. \nThe system will poll for updates every day, and the specific update time cannot be determined.",

View File

@@ -97,7 +97,7 @@
"permission.des.read": "可查看知识库内容",
"permission.des.write": "可增加和变更知识库内容",
"preview_chunk": "分块预览",
"preview_chunk_empty": "文件内容为空",
"preview_chunk_empty": "无法读取该文件内容",
"preview_chunk_intro": "共 {{total}} 个分块,最多展示 10 个",
"preview_chunk_not_selected": "点击左侧文件后进行预览",
"process.Auto_Index": "自动索引生成",
@@ -125,7 +125,6 @@
"split_sign_question": "问号",
"split_sign_semicolon": "分号",
"start_sync_website_tip": "确认开始同步数据?将会删除旧数据后重新获取,请确认!",
"status_error": "运行异常",
"sync_collection_failed": "同步集合错误,请检查是否能正常访问源文件",
"sync_schedule": "定时同步",
"sync_schedule_tip": "仅会同步已存在的集合。包括链接集合以及 API 知识库里所有集合。系统会每天进行轮询更新,无法确定具体的更新时间。",

View File

@@ -97,7 +97,7 @@
"permission.des.read": "可檢視資料集內容",
"permission.des.write": "可新增和變更資料集內容",
"preview_chunk": "分塊預覽",
"preview_chunk_empty": "文件內容為空",
"preview_chunk_empty": "無法讀取該文件內容",
"preview_chunk_intro": "共 {{total}} 個分塊,最多展示 10 個",
"preview_chunk_not_selected": "點擊左側文件後進行預覽",
"process.Auto_Index": "自動索引生成",
@@ -124,7 +124,6 @@
"split_sign_question": "問號",
"split_sign_semicolon": "分號",
"start_sync_website_tip": "確認開始同步資料?\n將會刪除舊資料後重新獲取請確認",
"status_error": "運行異常",
"sync_collection_failed": "同步集合錯誤,請檢查是否能正常存取來源文件",
"sync_schedule": "定時同步",
"sync_schedule_tip": "只會同步已存在的集合。\n包括連結集合以及 API 知識庫裡所有集合。\n系統會每天進行輪詢更新無法確定特定的更新時間。",

View File

@@ -1,79 +0,0 @@
diff --git a/lib/index.js b/lib/index.js
index c5ca771c24dd914e342f791716a822431ee32b3a..457d9f8c4625f7d9c7ea1e9ffc13616db1fc6fef 100644
--- a/lib/index.js
+++ b/lib/index.js
@@ -126,8 +126,37 @@ function exitLiteralAutolink(token) {
this.exit(token)
}
-/** @type {FromMarkdownTransform} */
+// Regex support detector, for backward compatibility
+// Ref: https://github.com/syntax-tree/mdast-util-gfm-autolink-literal/pull/14
+const regexSupport = {
+ lookbehind: (() => {
+ try {
+ // Using regex literal instead of RegExp constructor
+ ;/(?<=x)/.test('x')
+ return true
+ } catch {
+ return false
+ }
+ })()
+}
+
+/**
+ * Main transform function that uses the appropriate version based on regex support
+ * @type {FromMarkdownTransform}
+ */
function transformGfmAutolinkLiterals(tree) {
+ if (regexSupport.lookbehind) {
+ modernAutolinkTransform(tree)
+ } else {
+ legacyAutolinkTransform(tree)
+ }
+}
+
+/**
+ * Modern version of autolink transform using lookbehind
+ * @type {FromMarkdownTransform}
+ */
+function modernAutolinkTransform(tree) {
findAndReplace(
tree,
[
@@ -138,6 +167,35 @@ function transformGfmAutolinkLiterals(tree) {
)
}
+
+/**
+ * Legacy version of autolink transform for older Node.js versions
+ * @type {FromMarkdownTransform}
+ */
+function legacyAutolinkTransform(tree) {
+ findAndReplace(
+ tree,
+ [
+ [/(https?:\/\/|www(?=\.))([-.\w]+)([^ \t\r\n]*)/gi, findUrl],
+ // [/([-.\w+]+)@([-\w]+(?:\.[-\w]+)+)/g, findEmail] # NOTE: original regex in 2.0.0
+ [/(^|\s|\p{P}|\p{S})([-.\w+]+)@([-\w]+(?:\.[-\w]+)+)/gu, findEmailLegacy]
+ ],
+ {ignore: ['link', 'linkReference']}
+ )
+}
+
+/**
+ * Helper function for legacy email matching
+ * @param {string} _ - Unused parameter
+ * @param {string} prefix - Email prefix
+ * @param {string} name - Email name
+ * @param {string} domain - Email domain
+ * @returns {*} The processed email
+ */
+function findEmailLegacy(_, prefix, name, domain) {
+ return findEmail(name + '@' + domain)
+}
+
/**
* @type {ReplaceFunction}
* @param {string} _

349
pnpm-lock.yaml generated
View File

@@ -4,11 +4,6 @@ settings:
autoInstallPeers: true
excludeLinksFromLockfile: false
patchedDependencies:
mdast-util-gfm-autolink-literal@2.0.1:
hash: f63d515781110436299ab612306211a9621c6dfaec1ce1a19e2f27454dc70251
path: patches/mdast-util-gfm-autolink-literal@2.0.1.patch
importers:
.:
@@ -30,7 +25,7 @@ importers:
version: 13.3.0
next-i18next:
specifier: 15.4.2
version: 15.4.2(i18next@23.16.8)(next@14.2.26(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.85.1))(react-i18next@14.1.2(i18next@23.16.8)(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react@18.3.1)
version: 15.4.2(i18next@23.16.8)(next@14.2.26(@babel/core@7.26.10)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.85.1))(react-i18next@14.1.2(i18next@23.16.8)(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react@18.3.1)
prettier:
specifier: 3.2.4
version: 3.2.4
@@ -256,8 +251,8 @@ importers:
specifier: 5.4.1
version: 5.4.1
pdfjs-dist:
specifier: 4.10.38
version: 4.10.38
specifier: 4.4.168
version: 4.4.168(encoding@0.1.13)
pg:
specifier: ^8.10.0
version: 8.14.0
@@ -327,7 +322,7 @@ importers:
version: 2.1.1(@chakra-ui/system@2.6.1(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@emotion/styled@11.11.0(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@types/react@18.3.1)(react@18.3.1))(react@18.3.1))(react@18.3.1)
'@chakra-ui/next-js':
specifier: 2.4.2
version: 2.4.2(@chakra-ui/react@2.10.7(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@emotion/styled@11.11.0(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@types/react@18.3.1)(react@18.3.1))(@types/react@18.3.1)(framer-motion@9.1.7(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(next@14.2.26(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.85.1))(react@18.3.1)
version: 2.4.2(@chakra-ui/react@2.10.7(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@emotion/styled@11.11.0(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@types/react@18.3.1)(react@18.3.1))(@types/react@18.3.1)(framer-motion@9.1.7(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(next@14.2.26(@babel/core@7.26.10)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.85.1))(react@18.3.1)
'@chakra-ui/react':
specifier: 2.10.7
version: 2.10.7(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@emotion/styled@11.11.0(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@types/react@18.3.1)(react@18.3.1))(@types/react@18.3.1)(framer-motion@9.1.7(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react-dom@18.3.1(react@18.3.1))(react@18.3.1)
@@ -390,7 +385,7 @@ importers:
version: 4.17.21
next-i18next:
specifier: 15.4.2
version: 15.4.2(i18next@23.16.8)(next@14.2.26(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.85.1))(react-i18next@14.1.2(i18next@23.16.8)(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react@18.3.1)
version: 15.4.2(i18next@23.16.8)(next@14.2.26(@babel/core@7.26.10)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.85.1))(react-i18next@14.1.2(i18next@23.16.8)(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react@18.3.1)
papaparse:
specifier: ^5.4.1
version: 5.4.1
@@ -451,7 +446,7 @@ importers:
version: 2.1.1(@chakra-ui/system@2.6.1(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@emotion/styled@11.11.0(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@types/react@18.3.1)(react@18.3.1))(react@18.3.1))(react@18.3.1)
'@chakra-ui/next-js':
specifier: 2.4.2
version: 2.4.2(@chakra-ui/react@2.10.7(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@emotion/styled@11.11.0(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@types/react@18.3.1)(react@18.3.1))(@types/react@18.3.1)(framer-motion@9.1.7(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(next@14.2.26(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.85.1))(react@18.3.1)
version: 2.4.2(@chakra-ui/react@2.10.7(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@emotion/styled@11.11.0(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@types/react@18.3.1)(react@18.3.1))(@types/react@18.3.1)(framer-motion@9.1.7(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(next@14.2.26(@babel/core@7.26.10)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.85.1))(react@18.3.1)
'@chakra-ui/react':
specifier: 2.10.7
version: 2.10.7(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@emotion/styled@11.11.0(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@types/react@18.3.1)(react@18.3.1))(@types/react@18.3.1)(framer-motion@9.1.7(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react-dom@18.3.1(react@18.3.1))(react@18.3.1)
@@ -550,7 +545,7 @@ importers:
version: 14.2.26(@babel/core@7.26.10)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.85.1)
next-i18next:
specifier: 15.4.2
version: 15.4.2(i18next@23.16.8)(next@14.2.26(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.85.1))(react-i18next@14.1.2(i18next@23.16.8)(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react@18.3.1)
version: 15.4.2(i18next@23.16.8)(next@14.2.26(@babel/core@7.26.10)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.85.1))(react-i18next@14.1.2(i18next@23.16.8)(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react@18.3.1)
nprogress:
specifier: ^0.2.0
version: 0.2.0
@@ -2305,6 +2300,10 @@ packages:
resolution: {integrity: sha512-9I2Zn6+NJLfaGoz9jN3lpwDgAYvfGeNYdbAIjJOqzs4Tpc+VU3Jqq4IofSUBKajiDS8k9fZIg18/z13mpk1bsA==}
engines: {node: '>=8'}
'@mapbox/node-pre-gyp@1.0.11':
resolution: {integrity: sha512-Yhlar6v9WQgUp/He7BdgzOz8lqMQ8sU+jkCq7Wx8Myc5YFJLbEe7lgui/V7G1qB1DJykHSGwreceSaD60Y0PUQ==}
hasBin: true
'@microsoft/tsdoc@0.15.1':
resolution: {integrity: sha512-4aErSrCR/On/e5G2hDP0wjooqDdauzEbIq8hIkIe5pXV0rtWJZvdCEKL0ykZxex+IxIwBp0eGeV48hQN07dXtw==}
@@ -2354,70 +2353,6 @@ packages:
cpu: [x64]
os: [win32]
'@napi-rs/canvas-android-arm64@0.1.69':
resolution: {integrity: sha512-4icWTByY8zPvM9SelfQKf3I6kwXw0aI5drBOVrwfER5kjwXJd78FPSDSZkxDHjvIo9Q86ljl18Yr963ehA4sHQ==}
engines: {node: '>= 10'}
cpu: [arm64]
os: [android]
'@napi-rs/canvas-darwin-arm64@0.1.69':
resolution: {integrity: sha512-HOanhhYlHdukA+unjelT4Dg3ta7e820x87/AG2dKUMsUzH19jaeZs9bcYjzEy2vYi/dFWKz7cSv2yaIOudB8Yg==}
engines: {node: '>= 10'}
cpu: [arm64]
os: [darwin]
'@napi-rs/canvas-darwin-x64@0.1.69':
resolution: {integrity: sha512-SIp7WfhxAPnSVK9bkFfJp+84rbATCIq9jMUzDwpCLhQ+v+OqtXe4pggX1oeV+62/HK6BT1t18qRmJfyqwJ9f3g==}
engines: {node: '>= 10'}
cpu: [x64]
os: [darwin]
'@napi-rs/canvas-linux-arm-gnueabihf@0.1.69':
resolution: {integrity: sha512-Ls+KujCp6TGpkuMVFvrlx+CxtL+casdkrprFjqIuOAnB30Mct6bCEr+I83Tu29s3nNq4EzIGjdmA3fFAZG/Dtw==}
engines: {node: '>= 10'}
cpu: [arm]
os: [linux]
'@napi-rs/canvas-linux-arm64-gnu@0.1.69':
resolution: {integrity: sha512-m8VcGmeSBNRbHZBd1srvdM1aq/ScS2y8KqGqmCCEgJlytYK4jdULzAo2K/BPKE1v3xvn8oUPZDLI/NBJbJkEoA==}
engines: {node: '>= 10'}
cpu: [arm64]
os: [linux]
'@napi-rs/canvas-linux-arm64-musl@0.1.69':
resolution: {integrity: sha512-a3xjNRIeK2m2ZORGv2moBvv3vbkaFZG1QKMeiEv/BKij+rkztuEhTJGMar+buICFgS0fLgphXXsKNkUSJb7eRQ==}
engines: {node: '>= 10'}
cpu: [arm64]
os: [linux]
'@napi-rs/canvas-linux-riscv64-gnu@0.1.69':
resolution: {integrity: sha512-pClUoJF5wdC9AvD0mc15G9JffL1Q85nuH1rLSQPRkGmGmQOtRjw5E9xNbanz7oFUiPbjH7xcAXUjVAcf7tdgPQ==}
engines: {node: '>= 10'}
cpu: [riscv64]
os: [linux]
'@napi-rs/canvas-linux-x64-gnu@0.1.69':
resolution: {integrity: sha512-96X3bFAmzemfw84Ts6Jg/omL86uuynvK06MWGR/mp3JYNumY9RXofA14eF/kJIYelbYFWXcwpbcBR71lJ6G/YQ==}
engines: {node: '>= 10'}
cpu: [x64]
os: [linux]
'@napi-rs/canvas-linux-x64-musl@0.1.69':
resolution: {integrity: sha512-2QTsEFO72Kwkj53W9hc5y1FAUvdGx0V+pjJB+9oQF6Ys9+y989GyPIl5wZDzeh8nIJW6koZZ1eFa8pD+pA5BFQ==}
engines: {node: '>= 10'}
cpu: [x64]
os: [linux]
'@napi-rs/canvas-win32-x64-msvc@0.1.69':
resolution: {integrity: sha512-Q4YA8kVnKarApBVLu7F8icGlIfSll5Glswo5hY6gPS4Is2dCI8+ig9OeDM8RlwYevUIxKq8lZBypN8Q1iLAQ7w==}
engines: {node: '>= 10'}
cpu: [x64]
os: [win32]
'@napi-rs/canvas@0.1.69':
resolution: {integrity: sha512-ydvNeJMRm+l3T14yCoUKqjYQiEdXDq1isznI93LEBGYssXKfSaLNLHOkeM4z9Fnw9Pkt2EKOCAtW9cS4b00Zcg==}
engines: {node: '>= 10'}
'@napi-rs/wasm-runtime@0.2.7':
resolution: {integrity: sha512-5yximcFK5FNompXfJFoWanu5l8v1hNGqNHh9du1xETp9HWk/B/PzvchX55WYOPaIeNglG8++68AAiauBAtbnzw==}
@@ -3714,6 +3649,9 @@ packages:
'@zilliz/milvus2-sdk-node@2.4.2':
resolution: {integrity: sha512-fkPu7XXzfUvHoCnSPVOjqQpWuSnnn9x2NMmmCcIOyRzMeXIsrz4Mf/+M7LUzmT8J9F0Khx65B0rJgCu27YzWQw==}
abbrev@1.1.1:
resolution: {integrity: sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==}
abbrev@2.0.0:
resolution: {integrity: sha512-6/mh1E2u2YgEsCHdY0Yx5oW+61gZU+1vXaoiHHrpKeuRNNgFvS+/jrwHiQhB5apAf5oB7UB7E19ol2R2LKH8hQ==}
engines: {node: ^14.17.0 || ^16.13.0 || >=18.0.0}
@@ -3743,6 +3681,10 @@ packages:
engines: {node: '>=0.4.0'}
hasBin: true
agent-base@6.0.2:
resolution: {integrity: sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==}
engines: {node: '>= 6.0.0'}
agent-base@7.1.3:
resolution: {integrity: sha512-jRR5wdylq8CkOe6hei19GGZnxM6rBGwFl3Bg0YItGDimvjGtAvdZk4Pu6Cl4u4Igsws4a1fd1Vq3ezrhn4KmFw==}
engines: {node: '>= 14'}
@@ -3850,6 +3792,14 @@ packages:
append-field@1.0.0:
resolution: {integrity: sha512-klpgFSWLW1ZEs8svjfb7g4qWY0YS5imI82dTg+QahUvJ8YqAY0P10Uk8tTyh9ZGuYEZEMaeJYCF5BFuX552hsw==}
aproba@2.0.0:
resolution: {integrity: sha512-lYe4Gx7QT+MKGbDsA+Z+he/Wtef0BiwDOlK/XkBrdfsh9J/jPPXbX0tE9x9cl27Tmu5gg3QUbUrQYa/y+KOHPQ==}
are-we-there-yet@2.0.0:
resolution: {integrity: sha512-Ci/qENmwHnsYo9xKIcUJN5LeDKdJ6R1Z1j9V/J5wyq8nh/mYPEpIKJbBZXtZjG04HiK7zV/p6Vs9952MrMeUIw==}
engines: {node: '>=10'}
deprecated: This package is no longer supported.
arg@4.1.3:
resolution: {integrity: sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA==}
@@ -4175,6 +4125,10 @@ packages:
caniuse-lite@1.0.30001704:
resolution: {integrity: sha512-+L2IgBbV6gXB4ETf0keSvLr7JUrRVbIaB/lrQ1+z8mRcQiisG5k+lG6O4n6Y5q6f5EuNfaYXKgymucphlEXQew==}
canvas@2.11.2:
resolution: {integrity: sha512-ItanGBMrmRV7Py2Z+Xhs7cT+FNt5K0vPL4p9EZ/UX/Mu7hFbkxSjKF2KVtPwX7UYWp7dRKnrTvReflgrItJbdw==}
engines: {node: '>=6'}
ccount@2.0.1:
resolution: {integrity: sha512-eyrF0jiFpY+3drT6383f1qhkbGsLSifNAjA61IUjZjmLCWjItY6LB9ft9YhoDgwfmclB2zhu51Lc7+95b8NRAg==}
@@ -4366,6 +4320,10 @@ packages:
color-string@1.9.1:
resolution: {integrity: sha512-shrVawQFojnZv6xM40anx4CkoDP+fZsw/ZerEMsW/pyzsRbElpsL/DBVW7q3ExxwusdNXI3lXpuhEZkzs8p5Eg==}
color-support@1.1.3:
resolution: {integrity: sha512-qiBjkpbMLO/HL68y+lh4q0/O1MZFj2RX6X/KmMa3+gJD3z+WwI1ZzDHysvqHGS3mP6mznPckpXmw1nI9cJjyRg==}
hasBin: true
color2k@2.0.3:
resolution: {integrity: sha512-zW190nQTIoXcGCaU08DvVNFTmQhUpnJfVuAKfWqUQkflXKpaDdpaYoM0iluLS9lgJNHyBF58KKA2FBEwkD7wog==}
@@ -4441,6 +4399,9 @@ packages:
consola@2.15.3:
resolution: {integrity: sha512-9vAdYbHj6x2fLKC4+oPH0kFzY/orMZyG2Aj+kNylHxKGJ/Ed4dpNyAQYwJOdqO4zdM7XpVHmyejQDcQHrnuXbw==}
console-control-strings@1.1.0:
resolution: {integrity: sha512-ty/fTekppD2fIwRvnZAVdeOiGd1c7YXEixbgJTNzqcxJWKQnjJ/V1bNEEE6hygpM3WjwHFUVK6HTjWSzV4a8sQ==}
content-disposition@0.5.4:
resolution: {integrity: sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==}
engines: {node: '>= 0.6'}
@@ -4769,6 +4730,10 @@ packages:
decode-named-character-reference@1.1.0:
resolution: {integrity: sha512-Wy+JTSbFThEOXQIR2L6mxJvEs+veIzpmqD7ynWxMXGpnk3smkHQOp6forLdHsKpAMW9iJpaBBIxz285t1n1C3w==}
decompress-response@4.2.1:
resolution: {integrity: sha512-jOSne2qbyE+/r8G1VU+G/82LBs2Fs4LAsTiLSHOCOMZQl2OKZ6i8i4IyHemTe+/yIXOtTcRQMzPcgyhoFlqPkw==}
engines: {node: '>=8'}
decompress-response@6.0.0:
resolution: {integrity: sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==}
engines: {node: '>=10'}
@@ -4854,6 +4819,9 @@ packages:
resolution: {integrity: sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==}
engines: {node: '>=0.4.0'}
delegates@1.0.0:
resolution: {integrity: sha512-bd2L678uiWATM6m5Z1VzNCErI3jiGzt6HGY8OVICs40JQq/HALfbyNJmp0UDakEY4pMMaN0Ly5om/B1VI/+xfQ==}
denque@2.1.0:
resolution: {integrity: sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==}
engines: {node: '>=0.10'}
@@ -5563,6 +5531,11 @@ packages:
functions-have-names@1.2.3:
resolution: {integrity: sha512-xckBUXyTIqT97tq2x2AMb+g163b5JFysYk0x4qxNFwbfQkmNZoiRHb6sPzI9/QV33WeuvVYBUIiD4NzNIyqaRQ==}
gauge@3.0.2:
resolution: {integrity: sha512-+5J6MS/5XksCuXq++uFRsnUd7Ovu1XenbeuIuNRJxYWjgQbPuFhT14lAvsWfqfAmnwluf1OwMjz39HjfLPci0Q==}
engines: {node: '>=10'}
deprecated: This package is no longer supported.
generate-function@2.3.1:
resolution: {integrity: sha512-eeB5GfMNeevm/GRYq20ShmsaGcmI81kIX2K9XQx5miC8KdHaC6Jm0qQ8ZNeGOi7wYB8OsdxKs+Y2oVuTFuVwKQ==}
@@ -5715,6 +5688,9 @@ packages:
resolution: {integrity: sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==}
engines: {node: '>= 0.4'}
has-unicode@2.0.1:
resolution: {integrity: sha512-8Rf9Y83NBReMnx0gFzA8JImQACstCYWUplepDa9xprwwtmgEZUF0h/i5xSA625zB/I37EtrswSST6OXxwaaIJQ==}
has-yarn@3.0.0:
resolution: {integrity: sha512-IrsVwUHhEULx3R8f/aA8AHuEzAorplsab/v8HBzEiIukwq5i/EC+xmOW+HfP1OaDP+2JkgT1yILHN2O3UFIbcA==}
engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0}
@@ -5802,6 +5778,10 @@ packages:
resolution: {integrity: sha512-V5nVw1PAOgfI3Lmeaj2Exmeg7fenjhRUgz1lPSezy1CuhPYbgQtbQj4jZfEAEMlaL+vupsvhjqCyjzob0yxsmQ==}
engines: {node: '>=10.19.0'}
https-proxy-agent@5.0.1:
resolution: {integrity: sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==}
engines: {node: '>= 6'}
https-proxy-agent@7.0.6:
resolution: {integrity: sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==}
engines: {node: '>= 14'}
@@ -6564,8 +6544,8 @@ packages:
lexical@0.14.5:
resolution: {integrity: sha512-ouV7Gyr9+3WT3WTrCgRAD3iZnlJWfs2/kBl2x3J2Q3X9uCWJn/zn21fQ8G1EUHlu0dvXPBmdk9hXb/FjTClt6Q==}
lib0@0.2.102:
resolution: {integrity: sha512-g70kydI0I1sZU0ChO8mBbhw0oUW/8U0GHzygpvEIx8k+jgOpqnTSb/E+70toYVqHxBhrERD21TwD5QcZJQ40ZQ==}
lib0@0.2.99:
resolution: {integrity: sha512-vwztYuUf1uf/1zQxfzRfO5yzfNKhTtgOByCruuiQQxWQXnPb8Itaube5ylofcV0oM0aKal9Mv+S1s1Ky0UYP1w==}
engines: {node: '>=16'}
hasBin: true
@@ -7054,6 +7034,10 @@ packages:
resolution: {integrity: sha512-vqiC06CuhBTUdZH+RYl8sFrL096vA45Ok5ISO6sE/Mr1jRbGH4Csnhi8f3wKVl7x8mO4Au7Ir9D3Oyv1VYMFJw==}
engines: {node: '>=12'}
mimic-response@2.1.0:
resolution: {integrity: sha512-wXqjST+SLt7R009ySCglWBCFpjUygmCIfD790/kVbiGmUgfYGuB14PiTd5DwVxSV4NcYHjzMkoj5LjQZwTQLEA==}
engines: {node: '>=8'}
mimic-response@3.1.0:
resolution: {integrity: sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==}
engines: {node: '>=10'}
@@ -7231,6 +7215,9 @@ packages:
resolution: {integrity: sha512-eLoBxg6wE/rZkJPhU/xRX1WTpkFEwDJEN96oxFrTsqBdbT5ec295Q+CoHrL9IT0DipqKhmGcaZmwOt8OON5x1w==}
engines: {node: '>=12.0.0'}
nan@2.22.2:
resolution: {integrity: sha512-DANghxFkS1plDdRsX0X9pm0Z6SJNN6gBdtXfanwoZ8hooC5gosGFSBGRYHUVPz1asKA/kMRqDRdHrluZ61SpBQ==}
nanoid@3.3.9:
resolution: {integrity: sha512-SppoicMGpZvbF1l3z4x7No3OlIjP7QJvC9XR7AhZr1kL133KHnKPztkKDc+Ir4aJ/1VhTySrtKhrsycmrMQfvg==}
engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1}
@@ -7363,6 +7350,11 @@ packages:
non-layered-tidy-tree-layout@2.0.2:
resolution: {integrity: sha512-gkXMxRzUH+PB0ax9dUN0yYF0S25BqeAYqhgMaLUFmpXLEk7Fcu8f4emJuOAY0V8kjDICxROIKsTAKsV/v355xw==}
nopt@5.0.0:
resolution: {integrity: sha512-Tbj67rffqceeLpcRXrT7vKAN8CwfPeIBgM7E6iBkmKLV7bEMwpGgYLGv0jACUsECaa/vuxP0IjEont6umdMgtQ==}
engines: {node: '>=6'}
hasBin: true
nopt@7.2.1:
resolution: {integrity: sha512-taM24ViiimT/XntxbPyJQzCG+p4EKOpgD3mxFwW38mGjVUrfERQOeY4EDHjdnptttfHuHQXFx+lTP08Q+mLa/w==}
engines: {node: ^14.17.0 || ^16.13.0 || >=18.0.0}
@@ -7384,6 +7376,10 @@ packages:
resolution: {integrity: sha512-ppwTtiJZq0O/ai0z7yfudtBpWIoxM8yE6nHi1X47eFR2EWORqfbu6CnPlNsjeN683eT0qG6H/Pyf9fCcvjnnnQ==}
engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0}
npmlog@5.0.1:
resolution: {integrity: sha512-AqZtDUWOMKs1G/8lwylVjrdYgqA4d9nu8hc+0gzRxlDb1I10+FHBGMXs6aiQHFdCUUlqH99MUMuLfzWDNDtfxw==}
deprecated: This package is no longer supported.
nprogress@0.2.0:
resolution: {integrity: sha512-I19aIingLgR1fmhftnbWWO3dXc0hSxqHQHQb3H8m+K3TnEn/iSeTZZOyvKXWqQESMwuUVnatlCnZdLBZZt2VSA==}
@@ -7612,6 +7608,10 @@ packages:
resolution: {integrity: sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==}
engines: {node: '>=8'}
path2d@0.2.2:
resolution: {integrity: sha512-+vnG6S4dYcYxZd+CZxzXCNKdELYZSKfohrk98yajCo1PtRoDgCTrrwOvK1GT0UoAdVszagDVllQc0U1vaX4NUQ==}
engines: {node: '>=6'}
pathe@1.1.2:
resolution: {integrity: sha512-whLdWMYL2TwI08hn8/ZqAbrVemu0LNaNNJZX73O6qaIdCTfXutsLhMkjdENX0qhsQ9uIimo4/aQOmXkoon2nDQ==}
@@ -7625,9 +7625,9 @@ packages:
resolution: {integrity: sha512-vE7JKRyES09KiunauX7nd2Q9/L7lhok4smP9RZTDeD4MVs72Dp2qNFVz39Nz5a0FVEW0BJR6C0DYrq6unoziZA==}
engines: {node: '>= 14.16'}
pdfjs-dist@4.10.38:
resolution: {integrity: sha512-/Y3fcFrXEAsMjJXeL9J8+ZG9U01LbuWaYypvDW2ycW1jL269L3js3DVBjDJ0Up9Np1uqDXsDrRihHANhZOlwdQ==}
engines: {node: '>=20'}
pdfjs-dist@4.4.168:
resolution: {integrity: sha512-MbkAjpwka/dMHaCfQ75RY1FXX3IewBVu6NGZOcxerRFlaBiIkZmUoR0jotX5VUzYZEXAGzSFtknWs5xRKliXPA==}
engines: {node: '>=18'}
peek-readable@5.4.2:
resolution: {integrity: sha512-peBp3qZyuS6cNIJ2akRNG1uo1WJ1d0wTxg/fxMdZ0BqCVhx242bSFHM9eNqflfJVS9SsgkzgT/1UgnsurBOTMg==}
@@ -8464,6 +8464,9 @@ packages:
simple-concat@1.0.1:
resolution: {integrity: sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==}
simple-get@3.1.1:
resolution: {integrity: sha512-CQ5LTKGfCpvE1K0n2us+kuMPbk/q0EKl82s4aheV9oXjFEz6W/Y7oQFVJuU6QG77hRT4Ghb5RURteF5vnWjupA==}
simple-get@4.0.1:
resolution: {integrity: sha512-brv7p5WgH0jmQJr1ZDDfKDOSeWWg+OVypG99A/5vYGPqJ6pxiaHLy8nxtFjBA7oMa01ebA9gfh1uMCFqOuXxvA==}
@@ -9555,6 +9558,9 @@ packages:
engines: {node: '>=8'}
hasBin: true
wide-align@1.1.5:
resolution: {integrity: sha512-eDMORYaPNZ4sQIuuYPDHdQvf4gyCF9rEEV/yPxGfwPkRodwEgiMUUXTx/dex+Me0wxx53S+NgUHaP7y3MGlDmg==}
widest-line@4.0.1:
resolution: {integrity: sha512-o0cyEG0e8GPzT4iGHphIOh0cJOV8fivsXxddQasHPHfoZf1ZexrfeA21w2NaEN1RHE+fXlfISmOE8R9N3u3Qig==}
engines: {node: '>=12'}
@@ -10740,7 +10746,7 @@ snapshots:
'@chakra-ui/system': 2.6.1(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@emotion/styled@11.11.0(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@types/react@18.3.1)(react@18.3.1))(react@18.3.1)
react: 18.3.1
'@chakra-ui/next-js@2.4.2(@chakra-ui/react@2.10.7(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@emotion/styled@11.11.0(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@types/react@18.3.1)(react@18.3.1))(@types/react@18.3.1)(framer-motion@9.1.7(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(next@14.2.26(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.85.1))(react@18.3.1)':
'@chakra-ui/next-js@2.4.2(@chakra-ui/react@2.10.7(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@emotion/styled@11.11.0(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@types/react@18.3.1)(react@18.3.1))(@types/react@18.3.1)(framer-motion@9.1.7(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(next@14.2.26(@babel/core@7.26.10)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.85.1))(react@18.3.1)':
dependencies:
'@chakra-ui/react': 2.10.7(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@emotion/styled@11.11.0(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@types/react@18.3.1)(react@18.3.1))(@types/react@18.3.1)(framer-motion@9.1.7(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react-dom@18.3.1(react@18.3.1))(react@18.3.1)
'@emotion/cache': 11.14.0
@@ -11604,6 +11610,22 @@ snapshots:
'@lukeed/ms@2.0.2': {}
'@mapbox/node-pre-gyp@1.0.11(encoding@0.1.13)':
dependencies:
detect-libc: 2.0.3
https-proxy-agent: 5.0.1
make-dir: 3.1.0
node-fetch: 2.7.0(encoding@0.1.13)
nopt: 5.0.0
npmlog: 5.0.1
rimraf: 3.0.2
semver: 7.7.1
tar: 6.2.1
transitivePeerDependencies:
- encoding
- supports-color
optional: true
'@microsoft/tsdoc@0.15.1': {}
'@mixmark-io/domino@2.2.0': {}
@@ -11641,50 +11663,6 @@ snapshots:
'@msgpackr-extract/msgpackr-extract-win32-x64@3.0.3':
optional: true
'@napi-rs/canvas-android-arm64@0.1.69':
optional: true
'@napi-rs/canvas-darwin-arm64@0.1.69':
optional: true
'@napi-rs/canvas-darwin-x64@0.1.69':
optional: true
'@napi-rs/canvas-linux-arm-gnueabihf@0.1.69':
optional: true
'@napi-rs/canvas-linux-arm64-gnu@0.1.69':
optional: true
'@napi-rs/canvas-linux-arm64-musl@0.1.69':
optional: true
'@napi-rs/canvas-linux-riscv64-gnu@0.1.69':
optional: true
'@napi-rs/canvas-linux-x64-gnu@0.1.69':
optional: true
'@napi-rs/canvas-linux-x64-musl@0.1.69':
optional: true
'@napi-rs/canvas-win32-x64-msvc@0.1.69':
optional: true
'@napi-rs/canvas@0.1.69':
optionalDependencies:
'@napi-rs/canvas-android-arm64': 0.1.69
'@napi-rs/canvas-darwin-arm64': 0.1.69
'@napi-rs/canvas-darwin-x64': 0.1.69
'@napi-rs/canvas-linux-arm-gnueabihf': 0.1.69
'@napi-rs/canvas-linux-arm64-gnu': 0.1.69
'@napi-rs/canvas-linux-arm64-musl': 0.1.69
'@napi-rs/canvas-linux-riscv64-gnu': 0.1.69
'@napi-rs/canvas-linux-x64-gnu': 0.1.69
'@napi-rs/canvas-linux-x64-musl': 0.1.69
'@napi-rs/canvas-win32-x64-msvc': 0.1.69
optional: true
'@napi-rs/wasm-runtime@0.2.7':
dependencies:
'@emnapi/core': 1.3.1
@@ -13088,6 +13066,9 @@ snapshots:
protobufjs: 7.4.0
winston: 3.17.0
abbrev@1.1.1:
optional: true
abbrev@2.0.0: {}
abort-controller@3.0.0:
@@ -13111,6 +13092,13 @@ snapshots:
acorn@8.14.1: {}
agent-base@6.0.2:
dependencies:
debug: 4.4.0
transitivePeerDependencies:
- supports-color
optional: true
agent-base@7.1.3: {}
agentkeepalive@4.6.0:
@@ -13218,6 +13206,15 @@ snapshots:
append-field@1.0.0: {}
aproba@2.0.0:
optional: true
are-we-there-yet@2.0.0:
dependencies:
delegates: 1.0.0
readable-stream: 3.6.2
optional: true
arg@4.1.3: {}
argparse@1.0.10:
@@ -13644,6 +13641,16 @@ snapshots:
caniuse-lite@1.0.30001704: {}
canvas@2.11.2(encoding@0.1.13):
dependencies:
'@mapbox/node-pre-gyp': 1.0.11(encoding@0.1.13)
nan: 2.22.2
simple-get: 3.1.1
transitivePeerDependencies:
- encoding
- supports-color
optional: true
ccount@2.0.1: {}
chai@4.5.0:
@@ -13841,6 +13848,9 @@ snapshots:
color-name: 1.1.4
simple-swizzle: 0.2.2
color-support@1.1.3:
optional: true
color2k@2.0.3: {}
color@3.2.1:
@@ -13913,6 +13923,9 @@ snapshots:
consola@2.15.3: {}
console-control-strings@1.1.0:
optional: true
content-disposition@0.5.4:
dependencies:
safe-buffer: 5.2.1
@@ -14270,6 +14283,11 @@ snapshots:
dependencies:
character-entities: 2.0.2
decompress-response@4.2.1:
dependencies:
mimic-response: 2.1.0
optional: true
decompress-response@6.0.0:
dependencies:
mimic-response: 3.1.0
@@ -14361,6 +14379,9 @@ snapshots:
delayed-stream@1.0.0: {}
delegates@1.0.0:
optional: true
denque@2.1.0: {}
depd@2.0.0: {}
@@ -15322,6 +15343,19 @@ snapshots:
functions-have-names@1.2.3: {}
gauge@3.0.2:
dependencies:
aproba: 2.0.0
color-support: 1.1.3
console-control-strings: 1.1.0
has-unicode: 2.0.1
object-assign: 4.1.1
signal-exit: 3.0.7
string-width: 4.2.3
strip-ansi: 6.0.1
wide-align: 1.1.5
optional: true
generate-function@2.3.1:
dependencies:
is-property: 1.0.2
@@ -15486,6 +15520,9 @@ snapshots:
dependencies:
has-symbols: 1.1.0
has-unicode@2.0.1:
optional: true
has-yarn@3.0.0: {}
hasown@2.0.2:
@@ -15631,6 +15668,14 @@ snapshots:
quick-lru: 5.1.1
resolve-alpn: 1.2.1
https-proxy-agent@5.0.1:
dependencies:
agent-base: 6.0.2
debug: 4.4.0
transitivePeerDependencies:
- supports-color
optional: true
https-proxy-agent@7.0.6:
dependencies:
agent-base: 7.1.3
@@ -16569,7 +16614,7 @@ snapshots:
lexical@0.14.5: {}
lib0@0.2.102:
lib0@0.2.99:
dependencies:
isomorphic.js: 0.2.5
@@ -16842,7 +16887,7 @@ snapshots:
transitivePeerDependencies:
- supports-color
mdast-util-gfm-autolink-literal@2.0.1(patch_hash=f63d515781110436299ab612306211a9621c6dfaec1ce1a19e2f27454dc70251):
mdast-util-gfm-autolink-literal@2.0.1:
dependencies:
'@types/mdast': 4.0.4
ccount: 2.0.1
@@ -16890,7 +16935,7 @@ snapshots:
mdast-util-gfm@3.1.0:
dependencies:
mdast-util-from-markdown: 2.0.2
mdast-util-gfm-autolink-literal: 2.0.1(patch_hash=f63d515781110436299ab612306211a9621c6dfaec1ce1a19e2f27454dc70251)
mdast-util-gfm-autolink-literal: 2.0.1
mdast-util-gfm-footnote: 2.1.0
mdast-util-gfm-strikethrough: 2.0.0
mdast-util-gfm-table: 2.0.0
@@ -17397,6 +17442,9 @@ snapshots:
mimic-fn@4.0.0: {}
mimic-response@2.1.0:
optional: true
mimic-response@3.1.0: {}
mimic-response@4.0.0: {}
@@ -17620,6 +17668,9 @@ snapshots:
dependencies:
lru-cache: 7.18.3
nan@2.22.2:
optional: true
nanoid@3.3.9: {}
nanoid@5.1.3: {}
@@ -17647,7 +17698,7 @@ snapshots:
transitivePeerDependencies:
- supports-color
next-i18next@15.4.2(i18next@23.16.8)(next@14.2.26(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.85.1))(react-i18next@14.1.2(i18next@23.16.8)(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react@18.3.1):
next-i18next@15.4.2(i18next@23.16.8)(next@14.2.26(@babel/core@7.26.10)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.85.1))(react-i18next@14.1.2(i18next@23.16.8)(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react@18.3.1):
dependencies:
'@babel/runtime': 7.26.10
'@types/hoist-non-react-statics': 3.3.6
@@ -17751,6 +17802,11 @@ snapshots:
non-layered-tidy-tree-layout@2.0.2: {}
nopt@5.0.0:
dependencies:
abbrev: 1.1.1
optional: true
nopt@7.2.1:
dependencies:
abbrev: 2.0.0
@@ -17767,6 +17823,14 @@ snapshots:
dependencies:
path-key: 4.0.0
npmlog@5.0.1:
dependencies:
are-we-there-yet: 2.0.0
console-control-strings: 1.1.0
gauge: 3.0.2
set-blocking: 2.0.0
optional: true
nprogress@0.2.0: {}
nth-check@2.1.1:
@@ -18051,6 +18115,9 @@ snapshots:
path-type@4.0.0: {}
path2d@0.2.2:
optional: true
pathe@1.1.2: {}
pathe@2.0.3: {}
@@ -18059,9 +18126,13 @@ snapshots:
pathval@2.0.0: {}
pdfjs-dist@4.10.38:
pdfjs-dist@4.4.168(encoding@0.1.13):
optionalDependencies:
'@napi-rs/canvas': 0.1.69
canvas: 2.11.2(encoding@0.1.13)
path2d: 0.2.2
transitivePeerDependencies:
- encoding
- supports-color
peek-readable@5.4.2: {}
@@ -19039,6 +19110,13 @@ snapshots:
simple-concat@1.0.1: {}
simple-get@3.1.1:
dependencies:
decompress-response: 4.2.1
once: 1.4.0
simple-concat: 1.0.1
optional: true
simple-get@4.0.1:
dependencies:
decompress-response: 6.0.0
@@ -20228,6 +20306,11 @@ snapshots:
siginfo: 2.0.0
stackback: 0.0.2
wide-align@1.1.5:
dependencies:
string-width: 4.2.3
optional: true
widest-line@4.0.1:
dependencies:
string-width: 5.1.2
@@ -20357,7 +20440,7 @@ snapshots:
yjs@13.6.24:
dependencies:
lib0: 0.2.102
lib0: 0.2.99
yn@3.1.1: {}

View File

@@ -1,7 +1,4 @@
packages:
- packages/*
- projects/*
- scripts/icon
patchedDependencies:
mdast-util-gfm-autolink-literal@2.0.1: patches/mdast-util-gfm-autolink-literal@2.0.1.patch
- 'packages/*'
- 'projects/*'
- 'scripts/icon'

View File

@@ -83,7 +83,6 @@ const nextConfig = {
serverComponentsExternalPackages: [
'mongoose',
'pg',
'bullmq',
'@zilliz/milvus2-sdk-node',
"tiktoken",
],

View File

@@ -96,8 +96,6 @@ const MobileVoiceInput = ({
const startYRef = useRef(0);
const [isCancel, setIsCancel] = useState(false);
const canvasPosition = canvasRef.current?.getBoundingClientRect();
const maskBottom = canvasPosition ? `${window.innerHeight - canvasPosition.top}px` : '50px';
const handleTouchStart = useCallback(
(e: React.TouchEvent<HTMLDivElement>) => {
@@ -197,7 +195,7 @@ const MobileVoiceInput = ({
position="fixed"
left={0}
right={0}
bottom={maskBottom}
bottom={'50px'}
h={'200px'}
bg="linear-gradient(to top, white, rgba(255, 255, 255, 0.7), rgba(255, 255, 255, 0))"
>
@@ -297,7 +295,7 @@ const VoiceInput = forwardRef<VoiceInputComponentRef, VoiceInputProps>(
}
};
startSpeak(finishWhisperTranscription);
}, [autoTTSResponse, onSendMessage, resetInputVal, startSpeak, whisperConfig?.autoSend]);
}, []);
const onSpeach = useCallback(() => {
if (isPc) {
@@ -305,7 +303,7 @@ const VoiceInput = forwardRef<VoiceInputComponentRef, VoiceInputProps>(
} else {
setMobilePreSpeak(true);
}
}, [isPc, onStartSpeak]);
}, []);
useImperativeHandle(ref, () => ({
onSpeak: onSpeach
}));

View File

@@ -76,9 +76,8 @@ const CollectionPageContextProvider = ({ children }: { children: ReactNode }) =>
});
const syncWebsite = async () => {
await checkTeamWebSyncLimit();
postWebsiteSync({ datasetId: datasetId }).then(() => {
loadDatasetDetail(datasetId);
});
await postWebsiteSync({ datasetId: datasetId });
await loadDatasetDetail(datasetId);
};
const {
isOpen: isOpenWebsiteModal,

View File

@@ -33,11 +33,10 @@ import HeaderTagPopOver from './HeaderTagPopOver';
import MyBox from '@fastgpt/web/components/common/MyBox';
import Icon from '@fastgpt/web/components/common/Icon';
import MyTag from '@fastgpt/web/components/common/Tag/index';
import QuestionTip from '@fastgpt/web/components/common/MyTooltip/QuestionTip';
const FileSourceSelector = dynamic(() => import('../Import/components/FileSourceSelector'));
const Header = ({ hasTrainingData }: { hasTrainingData: boolean }) => {
const Header = ({}: {}) => {
const { t } = useTranslation();
const { feConfigs } = useSystemStore();
const { isPc } = useSystem();
@@ -283,15 +282,13 @@ const Header = ({ hasTrainingData }: { hasTrainingData: boolean }) => {
>
{t('dataset:params_config')}
</Button>
{!hasTrainingData && (
<Button
variant={'whitePrimary'}
onClick={openWebSyncConfirm}
leftIcon={<Icon name="common/confirm/restoreTip" w={'1rem'} />}
>
{t('dataset:immediate_sync')}
</Button>
)}
<Button
variant={'whitePrimary'}
onClick={openWebSyncConfirm}
leftIcon={<Icon name="common/confirm/restoreTip" w={'1rem'} />}
>
{t('dataset:immediate_sync')}
</Button>
</HStack>
)}
{datasetDetail.status === DatasetStatusEnum.syncing && (
@@ -324,14 +321,6 @@ const Header = ({ hasTrainingData }: { hasTrainingData: boolean }) => {
{t('common:core.dataset.status.waiting')}
</MyTag>
)}
{datasetDetail.status === DatasetStatusEnum.error && (
<MyTag colorSchema="red" showDot px={3} h={'36px'}>
<HStack spacing={1}>
<Box>{t('dataset:status_error')}</Box>
<QuestionTip color={'red.500'} label={datasetDetail.errorMsg} />
</HStack>
</MyTag>
)}
</>
) : (
<Button

View File

@@ -196,7 +196,7 @@ const CollectionCard = () => {
<MyBox isLoading={isLoading} h={'100%'} py={[2, 4]}>
<Flex ref={BoxRef} flexDirection={'column'} py={[1, 0]} h={'100%'} px={[2, 6]}>
{/* header */}
<Header hasTrainingData={hasTrainingData} />
<Header />
{/* collection table */}
<TableContainer mt={3} overflowY={'auto'} fontSize={'sm'}>

View File

@@ -115,7 +115,6 @@ const CollectionChunkForm = ({ form }: { form: UseFormReturn<CollectionChunkForm
const chunkSplitMode = watch('chunkSplitMode');
const autoIndexes = watch('autoIndexes');
const indexSize = watch('indexSize');
const imageIndex = watch('imageIndex');
const trainingModeList = useMemo(() => {
const list = Object.entries(DatasetCollectionDataProcessModeMap);
@@ -226,11 +225,7 @@ const CollectionChunkForm = ({ form }: { form: UseFormReturn<CollectionChunkForm
<HStack gap={[3, 7]}>
<HStack flex={'1'} spacing={1}>
<MyTooltip label={!feConfigs?.isPlus ? t('common:commercial_function_tip') : ''}>
<Checkbox
isDisabled={!feConfigs?.isPlus}
isChecked={autoIndexes}
{...register('autoIndexes')}
>
<Checkbox isDisabled={!feConfigs?.isPlus} {...register('autoIndexes')}>
<FormLabel>{t('dataset:auto_indexes')}</FormLabel>
</Checkbox>
</MyTooltip>
@@ -248,7 +243,6 @@ const CollectionChunkForm = ({ form }: { form: UseFormReturn<CollectionChunkForm
>
<Checkbox
isDisabled={!feConfigs?.isPlus || !datasetDetail?.vlmModel}
isChecked={imageIndex}
{...register('imageIndex')}
>
<FormLabel>{t('dataset:image_auto_parse')}</FormLabel>

View File

@@ -20,14 +20,17 @@ import FormLabel from '@fastgpt/web/components/common/MyBox/FormLabel';
import QuestionTip from '@fastgpt/web/components/common/MyTooltip/QuestionTip';
import { shadowLight } from '@fastgpt/web/styles/theme';
import CollectionChunkForm from '../../Form/CollectionChunkForm';
import { DatasetCollectionDataProcessModeEnum } from '@fastgpt/global/core/dataset/constants';
function DataProcess() {
const { t } = useTranslation();
const { feConfigs } = useSystemStore();
const { goToNext, processParamsForm } = useContextSelector(DatasetImportContext, (v) => v);
const { register, watch } = processParamsForm;
const customPdfParseValue = watch('customPdfParse');
const { goToNext, processParamsForm, chunkSize } = useContextSelector(
DatasetImportContext,
(v) => v
);
const { register } = processParamsForm;
const Title = useCallback(({ title }: { title: string }) => {
return (
@@ -63,7 +66,7 @@ function DataProcess() {
>
{feConfigs.showCustomPdfParse && (
<HStack spacing={1}>
<Checkbox isChecked={customPdfParseValue} {...register('customPdfParse')}>
<Checkbox {...register('customPdfParse')}>
<FormLabel>{t('dataset:pdf_enhance_parse')}</FormLabel>
</Checkbox>
<QuestionTip label={t('dataset:pdf_enhance_parse_tips')} />

View File

@@ -1,6 +1,8 @@
import { getFeishuAndYuqueDatasetFileList } from '@/service/core/dataset/apiDataset/controller';
import { NextAPI } from '@/service/middleware/entry';
import { DatasetErrEnum } from '@fastgpt/global/common/error/code/dataset';
import { ParentIdType } from '@fastgpt/global/common/parentFolder/type';
import { APIFileItem } from '@fastgpt/global/core/dataset/apiDataset';
import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant';
import { useApiDatasetRequest } from '@fastgpt/service/core/dataset/apiDataset/api';
import { authDataset } from '@fastgpt/service/support/permission/dataset/auth';
@@ -12,6 +14,8 @@ export type GetApiDatasetFileListProps = {
datasetId: string;
};
export type GetApiDatasetFileListResponse = APIFileItem[];
async function handler(req: NextApiRequest) {
let { searchKey = '', parentId = null, datasetId } = req.body;
@@ -31,7 +35,7 @@ async function handler(req: NextApiRequest) {
return useApiDatasetRequest({ apiServer }).listFiles({ searchKey, parentId });
}
if (feishuServer || yuqueServer) {
return global.getProApiDatasetFileList({
return getFeishuAndYuqueDatasetFileList({
feishuServer,
yuqueServer,
parentId

View File

@@ -10,6 +10,7 @@ import { DatasetErrEnum } from '@fastgpt/global/common/error/code/dataset';
import { authChatCrud, authCollectionInChat } from '@/service/support/permission/auth/chat';
import { getCollectionWithDataset } from '@fastgpt/service/core/dataset/controller';
import { useApiDatasetRequest } from '@fastgpt/service/core/dataset/apiDataset/api';
import { POST } from '@fastgpt/service/common/api/plusRequest';
export type readCollectionSourceQuery = {};
@@ -105,7 +106,8 @@ async function handler(
}
if (feishuServer || yuqueServer) {
return global.getProApiDatasetFilePreviewUrl({
return POST<string>(`/core/dataset/systemApiDataset`, {
type: 'read',
apiFileId: collection.apiFileId,
feishuServer,
yuqueServer

View File

@@ -30,21 +30,17 @@ async function handler(req: ApiRequestProps<Query>): Promise<DatasetItemType> {
per: ReadPermissionVal
});
const { status, errorMsg } = await (async () => {
const status = await (async () => {
if (dataset.type === DatasetTypeEnum.websiteDataset) {
return await getWebsiteSyncDatasetStatus(datasetId);
}
return {
status: DatasetStatusEnum.active,
errorMsg: undefined
};
return DatasetStatusEnum.active;
})();
return {
...dataset,
status,
errorMsg,
apiServer: dataset.apiServer
? {
baseUrl: dataset.apiServer.baseUrl,

View File

@@ -11,18 +11,6 @@ import { defaultGroup, defaultTemplateTypes } from '@fastgpt/web/core/workflow/c
import { MongoPluginGroups } from '@fastgpt/service/core/app/plugin/pluginGroupSchema';
import { MongoTemplateTypes } from '@fastgpt/service/core/app/templates/templateTypeSchema';
import { loadSystemModels } from '@fastgpt/service/core/ai/config/utils';
import { POST } from '@fastgpt/service/common/api/plusRequest';
import {
DeepRagSearchProps,
SearchDatasetDataResponse
} from '@fastgpt/service/core/dataset/search/controller';
import { AuthOpenApiLimitProps } from '@fastgpt/service/support/openapi/auth';
import { ConcatUsageProps, CreateUsageProps } from '@fastgpt/global/support/wallet/usage/api';
import {
getProApiDatasetFileContentRequest,
getProApiDatasetFileListRequest,
getProApiDatasetFilePreviewUrlRequest
} from '@/service/core/dataset/apiDataset/controller';
export const readConfigData = async (name: string) => {
const splitName = name.split('.');
@@ -48,37 +36,12 @@ export const readConfigData = async (name: string) => {
/* Init global variables */
export function initGlobalVariables() {
function initPlusRequest() {
global.textCensorHandler = function textCensorHandler({ text }: { text: string }) {
return POST<{ code: number; message?: string }>('/common/censor/check', { text });
};
global.deepRagHandler = function deepRagHandler(data: DeepRagSearchProps) {
return POST<SearchDatasetDataResponse>('/core/dataset/deepRag', data);
};
global.authOpenApiHandler = function authOpenApiHandler(data: AuthOpenApiLimitProps) {
return POST<AuthOpenApiLimitProps>('/support/openapi/authLimit', data);
};
global.createUsageHandler = function createUsageHandler(data: CreateUsageProps) {
return POST('/support/wallet/usage/createUsage', data);
};
global.concatUsageHandler = function concatUsageHandler(data: ConcatUsageProps) {
return POST('/support/wallet/usage/concatUsage', data);
};
global.getProApiDatasetFileList = getProApiDatasetFileListRequest;
global.getProApiDatasetFileContent = getProApiDatasetFileContentRequest;
global.getProApiDatasetFilePreviewUrl = getProApiDatasetFilePreviewUrlRequest;
}
if (global.communityPlugins) return;
global.communityPlugins = [];
global.qaQueueLen = global.qaQueueLen ?? 0;
global.vectorQueueLen = global.vectorQueueLen ?? 0;
initHttpAgent();
initPlusRequest();
}
/* Init system data(Need to connected db). It only needs to run once */

View File

@@ -1,35 +1,14 @@
import { APIFileItem, ApiFileReadContentResponse } from '@fastgpt/global/core/dataset/apiDataset';
import { GetApiDatasetFileListResponse } from '@/pages/api/core/dataset/apiDataset/list';
import { FeishuServer, YuqueServer } from '@fastgpt/global/core/dataset/apiDataset';
import { POST } from '@fastgpt/service/common/api/plusRequest';
import {
GetProApiDatasetFileContentParams,
GetProApiDatasetFileListParams,
GetProApiDatasetFilePreviewUrlParams,
ProApiDatasetOperationTypeEnum
} from '@fastgpt/service/core/dataset/apiDataset/proApi';
export const getProApiDatasetFileListRequest = async (data: GetProApiDatasetFileListParams) => {
const res = await POST<APIFileItem[]>('/core/dataset/systemApiDataset', {
type: ProApiDatasetOperationTypeEnum.LIST,
...data
});
return res;
};
export const getProApiDatasetFileContentRequest = async (
data: GetProApiDatasetFileContentParams
) => {
const res = await POST<ApiFileReadContentResponse>('/core/dataset/systemApiDataset', {
type: ProApiDatasetOperationTypeEnum.CONTENT,
...data
});
return res;
};
export const getProApiDatasetFilePreviewUrlRequest = async (
data: GetProApiDatasetFilePreviewUrlParams
) => {
const res = await POST<string>('/core/dataset/systemApiDataset', {
type: ProApiDatasetOperationTypeEnum.READ,
export const getFeishuAndYuqueDatasetFileList = async (data: {
feishuServer?: FeishuServer;
yuqueServer?: YuqueServer;
parentId?: string;
}) => {
const res = await POST<GetApiDatasetFileListResponse>('/core/dataset/systemApiDataset', {
type: 'list',
...data
});
return res;

View File

@@ -17,7 +17,7 @@ import { splitText2Chunks } from '@fastgpt/global/common/string/textSplitter';
import { countPromptTokens } from '@fastgpt/service/common/string/tiktoken';
const formatIndexes = async ({
indexes = [],
indexes,
q,
a = '',
indexSize,
@@ -66,6 +66,7 @@ const formatIndexes = async ({
];
};
indexes = indexes || [];
// If index not type, set it to custom
indexes = indexes
.map((item) => ({
@@ -92,7 +93,7 @@ const formatIndexes = async ({
indexes = indexes.filter((item) => item.type !== DatasetDataIndexTypeEnum.default);
indexes.push(...concatDefaultIndexes);
// Remove same text
// Filter same text
indexes = indexes.filter(
(item, index, self) => index === self.findIndex((t) => t.text === item.text)
);
@@ -100,16 +101,12 @@ const formatIndexes = async ({
const chekcIndexes = (
await Promise.all(
indexes.map(async (item) => {
if (item.type === DatasetDataIndexTypeEnum.default) {
return item;
}
// If oversize tokens, split it
const tokens = await countPromptTokens(item.text);
if (tokens > maxIndexSize) {
if (tokens > indexSize) {
const splitText = splitText2Chunks({
text: item.text,
chunkSize: indexSize,
chunkSize: 512,
maxSize: maxIndexSize
}).chunks;
return splitText.map((text) => ({
@@ -117,7 +114,6 @@ const formatIndexes = async ({
type: item.type
}));
}
return item;
})
)
@@ -168,30 +164,24 @@ export async function insertData2Dataset({
});
// insert to vector store
const results: {
tokens: number;
index: {
dataId: string;
type: `${DatasetDataIndexTypeEnum}`;
text: string;
};
}[] = [];
for await (const item of newIndexes) {
const result = await insertDatasetDataVector({
query: item.text,
model: embModel,
teamId,
datasetId,
collectionId
});
results.push({
tokens: result.tokens,
index: {
...item,
dataId: result.insertId
}
});
}
const result = await Promise.all(
newIndexes.map(async (item) => {
const result = await insertDatasetDataVector({
query: item.text,
model: embModel,
teamId,
datasetId,
collectionId
});
return {
tokens: result.tokens,
index: {
...item,
dataId: result.insertId
}
};
})
);
// 2. Create mongo data
const [{ _id }] = await MongoDatasetData.create(
@@ -204,7 +194,7 @@ export async function insertData2Dataset({
q,
a,
chunkIndex,
indexes: results.map((item) => item.index)
indexes: result.map((item) => item.index)
}
],
{ session, ordered: true }
@@ -226,7 +216,7 @@ export async function insertData2Dataset({
return {
insertId: _id,
tokens: results.reduce((acc, cur) => acc + cur.tokens, 0)
tokens: result.reduce((acc, cur) => acc + cur.tokens, 0)
};
}
@@ -313,27 +303,25 @@ export async function updateData2Dataset({
await mongoData.save();
// 5. insert vector
const insertResults: {
tokens: number;
}[] = [];
for await (const item of patchResult) {
if (item.type === 'delete' || item.type === 'unChange') continue;
// insert new vector and update dateId
const result = await insertDatasetDataVector({
query: item.index.text,
model: getEmbeddingModel(model),
teamId: mongoData.teamId,
datasetId: mongoData.datasetId,
collectionId: mongoData.collectionId
});
item.index.dataId = result.insertId;
insertResults.push({
tokens: result.tokens
});
}
const tokens = insertResults.reduce((acc, cur) => acc + cur.tokens, 0);
const insertResult = await Promise.all(
patchResult
.filter((item) => item.type === 'create' || item.type === 'update')
.map(async (item) => {
// insert new vector and update dateId
const result = await insertDatasetDataVector({
query: item.index.text,
model: getEmbeddingModel(model),
teamId: mongoData.teamId,
datasetId: mongoData.datasetId,
collectionId: mongoData.collectionId
});
item.index.dataId = result.insertId;
return {
tokens: result.tokens
};
})
);
const tokens = insertResult.reduce((acc, cur) => acc + cur.tokens, 0);
const newIndexes = patchResult
.filter((item) => item.type !== 'delete')

View File

@@ -200,24 +200,19 @@ const rebuildData = async ({
// update vector, update dataset_data rebuilding status, delete data from training
// 1. Insert new vector to dataset_data
const updateResult: {
tokens: number;
insertId: string;
}[] = [];
let i = 0;
for await (const index of mongoData.indexes) {
const result = await insertDatasetDataVector({
query: index.text,
model: getEmbeddingModel(trainingData.model),
teamId: mongoData.teamId,
datasetId: mongoData.datasetId,
collectionId: mongoData.collectionId
});
mongoData.indexes[i].dataId = result.insertId;
updateResult.push(result);
i++;
}
const updateResult = await Promise.all(
mongoData.indexes.map(async (index, i) => {
const result = await insertDatasetDataVector({
query: index.text,
model: getEmbeddingModel(trainingData.model),
teamId: mongoData.teamId,
datasetId: mongoData.datasetId,
collectionId: mongoData.collectionId
});
mongoData.indexes[i].dataId = result.insertId;
return result;
})
);
const { tokens } = await mongoSessionRun(async (session) => {
// 2. Ensure that the training data is deleted after the Mongo update is successful
await mongoData.save({ session });

View File

@@ -52,7 +52,10 @@ import type {
import type { UpdateDatasetDataProps } from '@fastgpt/global/core/dataset/controller';
import type { DatasetFolderCreateBody } from '@/pages/api/core/dataset/folder/create';
import type { PaginationProps, PaginationResponse } from '@fastgpt/web/common/fetch/type';
import type { GetApiDatasetFileListProps } from '@/pages/api/core/dataset/apiDataset/list';
import type {
GetApiDatasetFileListProps,
GetApiDatasetFileListResponse
} from '@/pages/api/core/dataset/apiDataset/list';
import type {
listExistIdQuery,
listExistIdResponse
@@ -71,7 +74,6 @@ import type {
getTrainingErrorBody,
getTrainingErrorResponse
} from '@/pages/api/core/dataset/training/getTrainingError';
import type { APIFileItem } from '@fastgpt/global/core/dataset/apiDataset';
/* ======================== dataset ======================= */
export const getDatasets = (data: GetDatasetListBody) =>
@@ -252,6 +254,6 @@ export const getCollectionSource = (data: readCollectionSourceBody) =>
/* ================== apiDataset ======================== */
export const getApiDatasetFileList = (data: GetApiDatasetFileListProps) =>
POST<APIFileItem[]>('/core/dataset/apiDataset/list', data);
POST<GetApiDatasetFileListResponse>('/core/dataset/apiDataset/list', data);
export const getApiDatasetFileListExistId = (data: listExistIdQuery) =>
GET<listExistIdResponse>('/core/dataset/apiDataset/listExistId', data);