Compare commits
25 Commits
v4.8.23-al
...
v4.8.23-fi
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
6d4776b3aa | ||
|
|
2d351c3654 | ||
|
|
662a4a4671 | ||
|
|
3fadabd28b | ||
|
|
dbf25cef88 | ||
|
|
b2e2fa6b76 | ||
|
|
576c60bd55 | ||
|
|
33617ab5dc | ||
|
|
b4dda6a41b | ||
|
|
e860c56b77 | ||
|
|
efac5312b4 | ||
|
|
4bc7f21182 | ||
|
|
113e8f711f | ||
|
|
abc6dffb41 | ||
|
|
f7b2a57ca3 | ||
|
|
cf0aaa1091 | ||
|
|
ac4255ea0c | ||
|
|
df4d6f86ce | ||
|
|
e697fda82f | ||
|
|
1aa319e7aa | ||
|
|
fc9e614f88 | ||
|
|
1121ea33bd | ||
|
|
9bbee60cde | ||
|
|
9f57ad0017 | ||
|
|
c3d3b30d7e |
2
.github/workflows/docs-deploy-kubeconfig.yml
vendored
2
.github/workflows/docs-deploy-kubeconfig.yml
vendored
@@ -6,8 +6,6 @@ on:
|
||||
- 'docSite/**'
|
||||
branches:
|
||||
- 'main'
|
||||
tags:
|
||||
- 'v*.*.*'
|
||||
|
||||
jobs:
|
||||
build-fastgpt-docs-images:
|
||||
|
||||
2
.github/workflows/docs-deploy-vercel.yml
vendored
2
.github/workflows/docs-deploy-vercel.yml
vendored
@@ -7,8 +7,6 @@ on:
|
||||
- 'docSite/**'
|
||||
branches:
|
||||
- 'main'
|
||||
tags:
|
||||
- 'v*.*.*'
|
||||
|
||||
# A workflow run is made up of one or more jobs that can run sequentially or in parallel
|
||||
jobs:
|
||||
|
||||
2
.github/workflows/docs-preview.yml
vendored
2
.github/workflows/docs-preview.yml
vendored
@@ -4,8 +4,6 @@ on:
|
||||
pull_request_target:
|
||||
paths:
|
||||
- 'docSite/**'
|
||||
branches:
|
||||
- 'main'
|
||||
workflow_dispatch:
|
||||
|
||||
# A workflow run is made up of one or more jobs that can run sequentially or in parallel
|
||||
|
||||
@@ -26,7 +26,7 @@ jobs:
|
||||
with:
|
||||
driver-opts: network=host
|
||||
- name: Cache Docker layers
|
||||
uses: actions/cache@v2
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: /tmp/.buildx-cache
|
||||
key: ${{ runner.os }}-buildx-${{ github.sha }}
|
||||
@@ -108,7 +108,7 @@ jobs:
|
||||
with:
|
||||
driver-opts: network=host
|
||||
- name: Cache Docker layers
|
||||
uses: actions/cache@v2
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: /tmp/.buildx-cache
|
||||
key: ${{ runner.os }}-buildx-${{ github.sha }}
|
||||
@@ -191,7 +191,7 @@ jobs:
|
||||
with:
|
||||
driver-opts: network=host
|
||||
- name: Cache Docker layers
|
||||
uses: actions/cache@v2
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: /tmp/.buildx-cache
|
||||
key: ${{ runner.os }}-buildx-${{ github.sha }}
|
||||
@@ -68,14 +68,3 @@ jobs:
|
||||
SEALOS_TYPE: 'pr_comment'
|
||||
SEALOS_FILENAME: 'report.md'
|
||||
SEALOS_REPLACE_TAG: 'DEFAULT_REPLACE_DEPLOY'
|
||||
|
||||
helm-check:
|
||||
runs-on: ubuntu-20.04
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v3
|
||||
- name: Helm Check
|
||||
run: |
|
||||
helm dependency update files/helm/fastgpt
|
||||
helm lint files/helm/fastgpt
|
||||
helm package files/helm/fastgpt
|
||||
4
.github/workflows/helm-release.yaml
vendored
4
.github/workflows/helm-release.yaml
vendored
@@ -24,6 +24,6 @@ jobs:
|
||||
export APP_VERSION=${{ steps.vars.outputs.tag }}
|
||||
export HELM_VERSION=${{ steps.vars.outputs.tag }}
|
||||
export HELM_REPO=ghcr.io/${{ github.repository_owner }}
|
||||
helm dependency update files/helm/fastgpt
|
||||
helm package files/helm/fastgpt --version ${HELM_VERSION}-helm --app-version ${APP_VERSION} -d bin
|
||||
helm dependency update deploy/helm/fastgpt
|
||||
helm package deploy/helm/fastgpt --version ${HELM_VERSION}-helm --app-version ${APP_VERSION} -d bin
|
||||
helm push bin/fastgpt-${HELM_VERSION}-helm.tgz oci://${HELM_REPO}
|
||||
|
||||
@@ -25,7 +25,7 @@ jobs:
|
||||
with:
|
||||
driver-opts: network=host
|
||||
- name: Cache Docker layers
|
||||
uses: actions/cache@v2
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: /tmp/.buildx-cache
|
||||
key: ${{ runner.os }}-buildx-${{ github.sha }}
|
||||
2
.vscode/nextapi.code-snippets
vendored
2
.vscode/nextapi.code-snippets
vendored
@@ -58,7 +58,7 @@
|
||||
"body": [
|
||||
"import '@/pages/api/__mocks__/base';",
|
||||
"import { root } from '@/pages/api/__mocks__/db/init';",
|
||||
"import { getTestRequest } from '@/test/utils';",
|
||||
"import { getTestRequest } from '@fastgpt/service/test/utils'; ;",
|
||||
"import { AppErrEnum } from '@fastgpt/global/common/error/code/app';",
|
||||
"import handler from './demo';",
|
||||
"",
|
||||
|
||||
26
SECURITY.md
Normal file
26
SECURITY.md
Normal file
@@ -0,0 +1,26 @@
|
||||
# 安全策略
|
||||
|
||||
## 漏洞报告
|
||||
|
||||
如果您发现了 FastGPT 的安全漏洞,请按照以下步骤进行报告:
|
||||
|
||||
1. **报告方式**
|
||||
发送邮件至:yujinlong@sealos.io
|
||||
请备注版本以及您的 GitHub 账号
|
||||
|
||||
3. **响应时间**
|
||||
- 我们会在 48 小时内确认收到您的报告
|
||||
- 一般在 3 个工作日内给出初步评估结果
|
||||
|
||||
4. **漏洞处理流程**
|
||||
- 确认漏洞:我们会验证漏洞的存在性和影响范围
|
||||
- 修复开发:针对已确认的漏洞进行修复
|
||||
- 版本发布:在下一个版本更新中发布安全补丁
|
||||
- 公开披露:在修复完成后,我们会在更新日志中公布相关信息
|
||||
|
||||
5. **注意事项**
|
||||
- 在漏洞未修复前,请勿公开披露漏洞详情
|
||||
- 我们欢迎负责任的漏洞披露
|
||||
- 对于重大贡献者,我们会在项目致谢名单中提及
|
||||
|
||||
感谢您为 FastGPT 的安全性做出贡献!
|
||||
@@ -114,15 +114,15 @@ services:
|
||||
# fastgpt
|
||||
sandbox:
|
||||
container_name: sandbox
|
||||
image: ghcr.io/labring/fastgpt-sandbox:v4.8.22 # git
|
||||
# image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-sandbox:v4.8.22 # 阿里云
|
||||
image: ghcr.io/labring/fastgpt-sandbox:v4.8.23-fix # git
|
||||
# image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-sandbox:v4.8.23-fix # 阿里云
|
||||
networks:
|
||||
- fastgpt
|
||||
restart: always
|
||||
fastgpt:
|
||||
container_name: fastgpt
|
||||
image: ghcr.io/labring/fastgpt:v4.8.22 # git
|
||||
# image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt:v4.8.22 # 阿里云
|
||||
image: ghcr.io/labring/fastgpt:v4.8.23-fix # git
|
||||
# image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt:v4.8.23-fix # 阿里云
|
||||
ports:
|
||||
- 3000:3000
|
||||
networks:
|
||||
@@ -72,15 +72,15 @@ services:
|
||||
# fastgpt
|
||||
sandbox:
|
||||
container_name: sandbox
|
||||
image: ghcr.io/labring/fastgpt-sandbox:v4.8.22 # git
|
||||
# image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-sandbox:v4.8.22 # 阿里云
|
||||
image: ghcr.io/labring/fastgpt-sandbox:v4.8.23-fix # git
|
||||
# image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-sandbox:v4.8.23-fix # 阿里云
|
||||
networks:
|
||||
- fastgpt
|
||||
restart: always
|
||||
fastgpt:
|
||||
container_name: fastgpt
|
||||
image: ghcr.io/labring/fastgpt:v4.8.22 # git
|
||||
# image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt:v4.8.22 # 阿里云
|
||||
image: ghcr.io/labring/fastgpt:v4.8.23-fix # git
|
||||
# image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt:v4.8.23-fix # 阿里云
|
||||
ports:
|
||||
- 3000:3000
|
||||
networks:
|
||||
@@ -53,15 +53,15 @@ services:
|
||||
wait $$!
|
||||
sandbox:
|
||||
container_name: sandbox
|
||||
image: ghcr.io/labring/fastgpt-sandbox:v4.8.22 # git
|
||||
# image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-sandbox:v4.8.22 # 阿里云
|
||||
image: ghcr.io/labring/fastgpt-sandbox:v4.8.23-fix # git
|
||||
# image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-sandbox:v4.8.23-fix # 阿里云
|
||||
networks:
|
||||
- fastgpt
|
||||
restart: always
|
||||
fastgpt:
|
||||
container_name: fastgpt
|
||||
image: ghcr.io/labring/fastgpt:v4.8.22 # git
|
||||
# image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt:v4.8.22 # 阿里云
|
||||
image: ghcr.io/labring/fastgpt:v4.8.23-fix # git
|
||||
# image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt:v4.8.23-fix # 阿里云
|
||||
ports:
|
||||
- 3000:3000
|
||||
networks:
|
||||
@@ -31,9 +31,9 @@ weight: 920
|
||||
|
||||
3 个模型代码分别为:
|
||||
|
||||
1. [https://github.com/labring/FastGPT/tree/main/python/bge-rerank/bge-reranker-base](https://github.com/labring/FastGPT/tree/main/python/bge-rerank/bge-reranker-base)
|
||||
2. [https://github.com/labring/FastGPT/tree/main/python/bge-rerank/bge-reranker-large](https://github.com/labring/FastGPT/tree/main/python/bge-rerank/bge-reranker-large)
|
||||
3. [https://github.com/labring/FastGPT/tree/main/python/bge-rerank/bge-reranker-v2-m3](https://github.com/labring/FastGPT/tree/main/python/bge-rerank/bge-reranker-v2-m3)
|
||||
1. [https://github.com/labring/FastGPT/tree/main/plugins/rerank-bge/bge-reranker-base](https://github.com/labring/FastGPT/tree/main/plugins/rerank-bge/bge-reranker-base)
|
||||
2. [https://github.com/labring/FastGPT/tree/main/plugins/rerank-bge/bge-reranker-large](https://github.com/labring/FastGPT/tree/main/plugins/rerank-bge/bge-reranker-large)
|
||||
3. [https://github.com/labring/FastGPT/tree/main/plugins/rerank-bge/bge-reranker-v2-m3](https://github.com/labring/FastGPT/tree/main/plugins/rerank-bge/bge-reranker-v2-m3)
|
||||
|
||||
### 3. 安装依赖
|
||||
|
||||
|
||||
@@ -19,7 +19,7 @@ PDF 是一个相对复杂的文件格式,在 FastGPT 内置的 pdf 解析器
|
||||
|
||||
### 1. 按照 Marker
|
||||
|
||||
参考文档 [Marker 安装教程](https://github.com/labring/FastGPT/tree/main/python/pdf-marker),安装 Marker 模型。封装的 API 已经适配了 FastGPT 自定义解析服务。
|
||||
参考文档 [Marker 安装教程](https://github.com/labring/FastGPT/tree/main/plugins/model/pdf-marker),安装 Marker 模型。封装的 API 已经适配了 FastGPT 自定义解析服务。
|
||||
|
||||
这里介绍快速 Docker 安装的方法:
|
||||
|
||||
|
||||
@@ -118,7 +118,7 @@ brew install orbstack
|
||||
非 Linux 环境或无法访问外网环境,可手动创建一个目录,并下载配置文件和对应版本的`docker-compose.yml`,在这个文件夹中依据下载的配置文件运行docker,若作为本地开发使用推荐`docker-compose-pgvector`版本,并且自行拉取并运行`sandbox`和`fastgpt`,并在docker配置文件中注释掉`sandbox`和`fastgpt`的部分
|
||||
|
||||
- [config.json](https://raw.githubusercontent.com/labring/FastGPT/refs/heads/main/projects/app/data/config.json)
|
||||
- [docker-compose.yml](https://github.com/labring/FastGPT/blob/main/files/docker) (注意,不同向量库版本的文件不一样)
|
||||
- [docker-compose.yml](https://github.com/labring/FastGPT/blob/main/deploy/docker) (注意,不同向量库版本的文件不一样)
|
||||
|
||||
{{% alert icon="🤖" context="success" %}}
|
||||
|
||||
@@ -134,11 +134,11 @@ cd fastgpt
|
||||
curl -O https://raw.githubusercontent.com/labring/FastGPT/main/projects/app/data/config.json
|
||||
|
||||
# pgvector 版本(测试推荐,简单快捷)
|
||||
curl -o docker-compose.yml https://raw.githubusercontent.com/labring/FastGPT/main/files/docker/docker-compose-pgvector.yml
|
||||
curl -o docker-compose.yml https://raw.githubusercontent.com/labring/FastGPT/main/deploy/docker/docker-compose-pgvector.yml
|
||||
# milvus 版本
|
||||
# curl -o docker-compose.yml https://raw.githubusercontent.com/labring/FastGPT/main/files/docker/docker-compose-milvus.yml
|
||||
# curl -o docker-compose.yml https://raw.githubusercontent.com/labring/FastGPT/main/deploy/docker/docker-compose-milvus.yml
|
||||
# zilliz 版本
|
||||
# curl -o docker-compose.yml https://raw.githubusercontent.com/labring/FastGPT/main/files/docker/docker-compose-zilliz.yml
|
||||
# curl -o docker-compose.yml https://raw.githubusercontent.com/labring/FastGPT/main/deploy/docker/docker-compose-zilliz.yml
|
||||
```
|
||||
|
||||
### 2. 修改环境变量
|
||||
@@ -201,6 +201,8 @@ docker restart oneapi
|
||||
|
||||
在OneApi中添加合适的AI模型渠道。[点击查看相关教程](/docs/development/modelconfig/one-api/)
|
||||
|
||||
只需要添加模型即可,模板已经配置好了oneapi的连接地址和令牌,无需变更。
|
||||
|
||||
### 5. 访问 FastGPT
|
||||
|
||||
目前可以通过 `ip:3000` 直接访问(注意防火墙)。登录用户名为 `root`,密码为`docker-compose.yml`环境变量里设置的 `DEFAULT_ROOT_PSW`。
|
||||
@@ -211,7 +213,7 @@ docker restart oneapi
|
||||
|
||||
### 6. 配置模型
|
||||
|
||||
务必先配置至少一组模型,否则系统无法正常使用。
|
||||
登录FastGPT后,进入模型配置页面,务必先配置至少一个语言模型和一个向量模型,否则系统无法正常使用。
|
||||
|
||||
[点击查看模型配置教程](/docs/development/modelConfig/intro/)
|
||||
|
||||
|
||||
@@ -20,7 +20,7 @@ SANDBOX_URL=内网地址
|
||||
|
||||
## Docker 部署
|
||||
|
||||
可以拉取最新 [docker-compose.yml](https://github.com/labring/FastGPT/blob/main/files/docker/docker-compose.yml) 文件参考
|
||||
可以拉取最新 [docker-compose.yml](https://github.com/labring/FastGPT/blob/main/deploy/docker/docker-compose.yml) 文件参考
|
||||
|
||||
1. 新增一个容器 `sandbox`
|
||||
2. fastgpt 和 fastgpt-pro(商业版) 容器新增环境变量: `SANDBOX_URL`
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
---
|
||||
title: 'V4.8.23(进行中)'
|
||||
title: 'V4.8.23'
|
||||
description: 'FastGPT V4.8.23 更新说明'
|
||||
icon: 'upgrade'
|
||||
draft: false
|
||||
@@ -7,11 +7,33 @@ toc: true
|
||||
weight: 802
|
||||
---
|
||||
|
||||
## 更新指南
|
||||
|
||||
### 1. 做好数据库备份
|
||||
|
||||
### 2. 更新镜像:
|
||||
|
||||
- 更新 fastgpt 镜像 tag: v4.8.23-fix
|
||||
- 更新 fastgpt-pro 商业版镜像 tag: v4.8.23-fix
|
||||
- Sandbox 镜像无需更新
|
||||
|
||||
### 3. 运行升级脚本
|
||||
|
||||
从任意终端,发起 1 个 HTTP 请求。其中 {{rootkey}} 替换成环境变量里的 `rootkey`;{{host}} 替换成**FastGPT 域名**。
|
||||
|
||||
```bash
|
||||
curl --location --request POST 'https://{{host}}/api/admin/initv4823' \
|
||||
--header 'rootkey: {{rootkey}}' \
|
||||
--header 'Content-Type: application/json'
|
||||
```
|
||||
|
||||
脚本会清理一些知识库脏数据,主要是多余的全文索引。
|
||||
|
||||
## 🚀 新增内容
|
||||
|
||||
1. 增加默认“知识库文本理解模型”配置
|
||||
2. AI proxy V1版,可替换 OneAPI使用,同时提供完整模型调用日志,便于排查问题。
|
||||
3. 增加工单入口支持。
|
||||
|
||||
## ⚙️ 优化
|
||||
|
||||
@@ -27,4 +49,6 @@ weight: 802
|
||||
1. 标签过滤时,子文件夹未成功过滤。
|
||||
2. 暂时移除 md 阅读优化,避免链接分割错误。
|
||||
3. 离开团队时,未刷新成员列表。
|
||||
4. PPTX 编码错误,导致解析失败。
|
||||
4. PPTX 编码错误,导致解析失败。
|
||||
5. 删除知识库单条数据时,全文索引未跟随删除。
|
||||
6. 修复 Mongo Dataset text 索引在查询数据时未生效。
|
||||
@@ -15,7 +15,7 @@ weight: 821
|
||||
|
||||
## V4.8.3 更新说明
|
||||
|
||||
1. 新增 - 支持 Milvus 数据库, 可参考最新的 [docker-compose-milvus.yml](https://github.com/labring/FastGPT/blob/main/files/docker/docker-compose-milvus.yml).
|
||||
1. 新增 - 支持 Milvus 数据库, 可参考最新的 [docker-compose-milvus.yml](https://github.com/labring/FastGPT/blob/main/deploy/docker/docker-compose-milvus.yml).
|
||||
2. 新增 - 给 chat 接口 empty answer 增加 log,便于排查模型问题。
|
||||
3. 新增 - ifelse判断器,字符串支持正则。
|
||||
4. 新增 - 代码运行支持 console.log 输出调试。
|
||||
|
||||
3
packages/README.md
Normal file
3
packages/README.md
Normal file
@@ -0,0 +1,3 @@
|
||||
# 目录说明
|
||||
|
||||
该目录为 FastGPT 的依赖包,多端复用。
|
||||
@@ -4,6 +4,7 @@ import { ErrType } from '../errorCode';
|
||||
/* dataset: 501000 */
|
||||
export enum DatasetErrEnum {
|
||||
unExist = 'unExistDataset',
|
||||
unExistCollection = 'unExistCollection',
|
||||
unAuthDataset = 'unAuthDataset',
|
||||
unCreateCollection = 'unCreateCollection',
|
||||
unAuthDatasetCollection = 'unAuthDatasetCollection',
|
||||
@@ -28,6 +29,10 @@ const datasetErr = [
|
||||
statusText: DatasetErrEnum.unExist,
|
||||
message: 'core.dataset.error.unExistDataset'
|
||||
},
|
||||
{
|
||||
statusText: DatasetErrEnum.unExistCollection,
|
||||
message: i18nT('common:error_collection_not_exist')
|
||||
},
|
||||
{
|
||||
statusText: DatasetErrEnum.unAuthDataset,
|
||||
message: 'core.dataset.error.unAuthDataset'
|
||||
|
||||
@@ -420,137 +420,3 @@ export function rewriteNodeOutputByHistories(
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
// Parse <think></think> tags to think and answer - unstream response
|
||||
export const parseReasoningContent = (text: string): [string, string] => {
|
||||
const regex = /<think>([\s\S]*?)<\/think>/;
|
||||
const match = text.match(regex);
|
||||
|
||||
if (!match) {
|
||||
return ['', text];
|
||||
}
|
||||
|
||||
const thinkContent = match[1].trim();
|
||||
|
||||
// Add answer (remaining text after think tag)
|
||||
const answerContent = text.slice(match.index! + match[0].length);
|
||||
|
||||
return [thinkContent, answerContent];
|
||||
};
|
||||
|
||||
// Parse <think></think> tags to think and answer - stream response
|
||||
export const parseReasoningStreamContent = () => {
|
||||
let isInThinkTag: boolean | undefined;
|
||||
|
||||
const startTag = '<think>';
|
||||
let startTagBuffer = '';
|
||||
|
||||
const endTag = '</think>';
|
||||
let endTagBuffer = '';
|
||||
|
||||
/*
|
||||
parseReasoning - 只控制是否主动解析 <think></think>,如果接口已经解析了,仍然会返回 think 内容。
|
||||
*/
|
||||
const parsePart = (
|
||||
part: {
|
||||
choices: {
|
||||
delta: {
|
||||
content?: string;
|
||||
reasoning_content?: string;
|
||||
};
|
||||
}[];
|
||||
},
|
||||
parseReasoning = false
|
||||
): [string, string] => {
|
||||
const content = part.choices?.[0]?.delta?.content || '';
|
||||
|
||||
// @ts-ignore
|
||||
const reasoningContent = part.choices?.[0]?.delta?.reasoning_content || '';
|
||||
if (reasoningContent || !parseReasoning) {
|
||||
isInThinkTag = false;
|
||||
return [reasoningContent, content];
|
||||
}
|
||||
|
||||
if (!content) {
|
||||
return ['', ''];
|
||||
}
|
||||
|
||||
// 如果不在 think 标签中,或者有 reasoningContent(接口已解析),则返回 reasoningContent 和 content
|
||||
if (isInThinkTag === false) {
|
||||
return ['', content];
|
||||
}
|
||||
|
||||
// 检测是否为 think 标签开头的数据
|
||||
if (isInThinkTag === undefined) {
|
||||
// Parse content think and answer
|
||||
startTagBuffer += content;
|
||||
// 太少内容时候,暂时不解析
|
||||
if (startTagBuffer.length < startTag.length) {
|
||||
return ['', ''];
|
||||
}
|
||||
|
||||
if (startTagBuffer.startsWith(startTag)) {
|
||||
isInThinkTag = true;
|
||||
return [startTagBuffer.slice(startTag.length), ''];
|
||||
}
|
||||
|
||||
// 如果未命中 think 标签,则认为不在 think 标签中,返回 buffer 内容作为 content
|
||||
isInThinkTag = false;
|
||||
return ['', startTagBuffer];
|
||||
}
|
||||
|
||||
// 确认是 think 标签内容,开始返回 think 内容,并实时检测 </think>
|
||||
/*
|
||||
检测 </think> 方案。
|
||||
存储所有疑似 </think> 的内容,直到检测到完整的 </think> 标签或超出 </think> 长度。
|
||||
content 返回值包含以下几种情况:
|
||||
abc - 完全未命中尾标签
|
||||
abc<th - 命中一部分尾标签
|
||||
abc</think> - 完全命中尾标签
|
||||
abc</think>abc - 完全命中尾标签
|
||||
</think>abc - 完全命中尾标签
|
||||
k>abc - 命中一部分尾标签
|
||||
*/
|
||||
// endTagBuffer 专门用来记录疑似尾标签的内容
|
||||
if (endTagBuffer) {
|
||||
endTagBuffer += content;
|
||||
if (endTagBuffer.includes(endTag)) {
|
||||
isInThinkTag = false;
|
||||
const answer = endTagBuffer.slice(endTag.length);
|
||||
return ['', answer];
|
||||
} else if (endTagBuffer.length >= endTag.length) {
|
||||
// 缓存内容超出尾标签长度,且仍未命中 </think>,则认为本次猜测 </think> 失败,仍处于 think 阶段。
|
||||
const tmp = endTagBuffer;
|
||||
endTagBuffer = '';
|
||||
return [tmp, ''];
|
||||
}
|
||||
return ['', ''];
|
||||
} else if (content.includes(endTag)) {
|
||||
// 返回内容,完整命中</think>,直接结束
|
||||
isInThinkTag = false;
|
||||
const [think, answer] = content.split(endTag);
|
||||
return [think, answer];
|
||||
} else {
|
||||
// 无 buffer,且未命中 </think>,开始疑似 </think> 检测。
|
||||
for (let i = 1; i < endTag.length; i++) {
|
||||
const partialEndTag = endTag.slice(0, i);
|
||||
// 命中一部分尾标签
|
||||
if (content.endsWith(partialEndTag)) {
|
||||
const think = content.slice(0, -partialEndTag.length);
|
||||
endTagBuffer += partialEndTag;
|
||||
return [think, ''];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 完全未命中尾标签,还是 think 阶段。
|
||||
return [content, ''];
|
||||
};
|
||||
|
||||
const getStartTagBuffer = () => startTagBuffer;
|
||||
|
||||
return {
|
||||
parsePart,
|
||||
getStartTagBuffer
|
||||
};
|
||||
};
|
||||
|
||||
@@ -10,7 +10,6 @@ export type AuthTeamRoleProps = {
|
||||
export type CreateTeamProps = {
|
||||
name: string;
|
||||
avatar?: string;
|
||||
defaultTeam?: boolean;
|
||||
memberName?: string;
|
||||
memberAvatar?: string;
|
||||
notificationAccount?: string;
|
||||
|
||||
2
packages/global/support/user/team/type.d.ts
vendored
2
packages/global/support/user/team/type.d.ts
vendored
@@ -47,7 +47,6 @@ export type TeamMemberSchema = {
|
||||
role: `${TeamMemberRoleEnum}`;
|
||||
status: `${TeamMemberStatusEnum}`;
|
||||
avatar: string;
|
||||
defaultTeam: boolean;
|
||||
};
|
||||
|
||||
export type TeamMemberWithTeamAndUserSchema = TeamMemberSchema & {
|
||||
@@ -65,7 +64,6 @@ export type TeamTmbItemType = {
|
||||
balance?: number;
|
||||
tmbId: string;
|
||||
teamDomain: string;
|
||||
defaultTeam: boolean;
|
||||
role: `${TeamMemberRoleEnum}`;
|
||||
status: `${TeamMemberStatusEnum}`;
|
||||
notificationAccount?: string;
|
||||
|
||||
@@ -16,7 +16,7 @@
|
||||
"nodeId": "lmpb9v2lo2lk",
|
||||
"name": "插件开始",
|
||||
"intro": "自定义配置外部输入,使用插件时,仅暴露自定义配置的输入",
|
||||
"avatar": "/imgs/workflow/input.png",
|
||||
"avatar": "core/workflow/template/workflowStart",
|
||||
"flowNodeType": "pluginInput",
|
||||
"showStatus": false,
|
||||
"position": {
|
||||
@@ -26,14 +26,16 @@
|
||||
"version": "481",
|
||||
"inputs": [
|
||||
{
|
||||
"renderTypeList": ["reference"],
|
||||
"renderTypeList": ["input", "reference"],
|
||||
"selectedTypeIndex": 0,
|
||||
"valueType": "string",
|
||||
"key": "url",
|
||||
"label": "url",
|
||||
"description": "需要读取的网页链接",
|
||||
"required": true,
|
||||
"toolDescription": "需要读取的网页链接"
|
||||
"toolDescription": "需要读取的网页链接",
|
||||
"list": [],
|
||||
"defaultValue": ""
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
@@ -50,12 +52,12 @@
|
||||
"nodeId": "i7uow4wj2wdp",
|
||||
"name": "插件输出",
|
||||
"intro": "自定义配置外部输出,使用插件时,仅暴露自定义配置的输出",
|
||||
"avatar": "/imgs/workflow/output.png",
|
||||
"avatar": "core/workflow/template/pluginOutput",
|
||||
"flowNodeType": "pluginOutput",
|
||||
"showStatus": false,
|
||||
"position": {
|
||||
"x": 1607.7142331269129,
|
||||
"y": -150.8808596935447
|
||||
"x": 1853.935047606551,
|
||||
"y": -154.13661665265613
|
||||
},
|
||||
"version": "481",
|
||||
"inputs": [
|
||||
@@ -81,12 +83,12 @@
|
||||
"nodeId": "ebLCxU43hHuZ",
|
||||
"name": "HTTP 请求",
|
||||
"intro": "可以发出一个 HTTP 请求,实现更为复杂的操作(联网搜索、数据库查询等)",
|
||||
"avatar": "/imgs/workflow/http.png",
|
||||
"avatar": "core/workflow/template/httpRequest",
|
||||
"flowNodeType": "httpRequest468",
|
||||
"showStatus": true,
|
||||
"position": {
|
||||
"x": 1050.9890727421412,
|
||||
"y": -415.2085119990912
|
||||
"x": 1054.2940501177068,
|
||||
"y": -503.13661665265613
|
||||
},
|
||||
"version": "481",
|
||||
"inputs": [
|
||||
@@ -96,7 +98,7 @@
|
||||
"valueType": "dynamic",
|
||||
"label": "",
|
||||
"required": false,
|
||||
"description": "core.module.input.description.HTTP Dynamic Input",
|
||||
"description": "common:core.module.input.description.HTTP Dynamic Input",
|
||||
"customInputConfig": {
|
||||
"selectValueTypeList": [
|
||||
"string",
|
||||
@@ -107,16 +109,19 @@
|
||||
"arrayNumber",
|
||||
"arrayBoolean",
|
||||
"arrayObject",
|
||||
"arrayAny",
|
||||
"any",
|
||||
"chatHistory",
|
||||
"datasetQuote",
|
||||
"dynamic",
|
||||
"selectApp",
|
||||
"selectDataset"
|
||||
"selectDataset",
|
||||
"selectApp"
|
||||
],
|
||||
"showDescription": false,
|
||||
"showDefaultValue": true
|
||||
}
|
||||
},
|
||||
"debugLabel": "",
|
||||
"toolDescription": ""
|
||||
},
|
||||
{
|
||||
"key": "system_httpMethod",
|
||||
@@ -124,17 +129,33 @@
|
||||
"valueType": "string",
|
||||
"label": "",
|
||||
"value": "POST",
|
||||
"required": true
|
||||
"required": true,
|
||||
"debugLabel": "",
|
||||
"toolDescription": ""
|
||||
},
|
||||
{
|
||||
"key": "system_httpTimeout",
|
||||
"renderTypeList": ["custom"],
|
||||
"valueType": "number",
|
||||
"label": "",
|
||||
"value": 30,
|
||||
"min": 5,
|
||||
"max": 600,
|
||||
"required": true,
|
||||
"debugLabel": "",
|
||||
"toolDescription": ""
|
||||
},
|
||||
{
|
||||
"key": "system_httpReqUrl",
|
||||
"renderTypeList": ["hidden"],
|
||||
"valueType": "string",
|
||||
"label": "",
|
||||
"description": "core.module.input.description.Http Request Url",
|
||||
"description": "common:core.module.input.description.Http Request Url",
|
||||
"placeholder": "https://api.ai.com/getInventory",
|
||||
"required": false,
|
||||
"value": "fetchUrl"
|
||||
"value": "fetchUrl",
|
||||
"debugLabel": "",
|
||||
"toolDescription": ""
|
||||
},
|
||||
{
|
||||
"key": "system_httpHeader",
|
||||
@@ -142,9 +163,11 @@
|
||||
"valueType": "any",
|
||||
"value": [],
|
||||
"label": "",
|
||||
"description": "core.module.input.description.Http Request Header",
|
||||
"placeholder": "core.module.input.description.Http Request Header",
|
||||
"required": false
|
||||
"description": "common:core.module.input.description.Http Request Header",
|
||||
"placeholder": "common:core.module.input.description.Http Request Header",
|
||||
"required": false,
|
||||
"debugLabel": "",
|
||||
"toolDescription": ""
|
||||
},
|
||||
{
|
||||
"key": "system_httpParams",
|
||||
@@ -152,7 +175,9 @@
|
||||
"valueType": "any",
|
||||
"value": [],
|
||||
"label": "",
|
||||
"required": false
|
||||
"required": false,
|
||||
"debugLabel": "",
|
||||
"toolDescription": ""
|
||||
},
|
||||
{
|
||||
"key": "system_httpJsonBody",
|
||||
@@ -160,7 +185,29 @@
|
||||
"valueType": "any",
|
||||
"value": "{\n \"url\": \"{{url}}\"\n}",
|
||||
"label": "",
|
||||
"required": false
|
||||
"required": false,
|
||||
"debugLabel": "",
|
||||
"toolDescription": ""
|
||||
},
|
||||
{
|
||||
"key": "system_httpFormBody",
|
||||
"renderTypeList": ["hidden"],
|
||||
"valueType": "any",
|
||||
"value": [],
|
||||
"label": "",
|
||||
"required": false,
|
||||
"debugLabel": "",
|
||||
"toolDescription": ""
|
||||
},
|
||||
{
|
||||
"key": "system_httpContentType",
|
||||
"renderTypeList": ["hidden"],
|
||||
"valueType": "string",
|
||||
"value": "json",
|
||||
"label": "",
|
||||
"required": false,
|
||||
"debugLabel": "",
|
||||
"toolDescription": ""
|
||||
},
|
||||
{
|
||||
"renderTypeList": ["reference"],
|
||||
@@ -178,12 +225,13 @@
|
||||
"arrayNumber",
|
||||
"arrayBoolean",
|
||||
"arrayObject",
|
||||
"arrayAny",
|
||||
"any",
|
||||
"chatHistory",
|
||||
"datasetQuote",
|
||||
"dynamic",
|
||||
"selectApp",
|
||||
"selectDataset"
|
||||
"selectDataset",
|
||||
"selectApp"
|
||||
],
|
||||
"showDescription": false,
|
||||
"showDefaultValue": true
|
||||
@@ -193,6 +241,23 @@
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"id": "error",
|
||||
"key": "error",
|
||||
"label": "workflow:request_error",
|
||||
"description": "HTTP请求错误信息,成功时返回空",
|
||||
"valueType": "object",
|
||||
"type": "static"
|
||||
},
|
||||
{
|
||||
"id": "httpRawResponse",
|
||||
"key": "httpRawResponse",
|
||||
"required": true,
|
||||
"label": "workflow:raw_response",
|
||||
"description": "HTTP请求的原始响应。只能接受字符串或JSON类型响应数据。",
|
||||
"valueType": "any",
|
||||
"type": "static"
|
||||
},
|
||||
{
|
||||
"id": "system_addOutputParam",
|
||||
"key": "system_addOutputParam",
|
||||
@@ -220,23 +285,6 @@
|
||||
"showDefaultValue": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "error",
|
||||
"key": "error",
|
||||
"label": "请求错误",
|
||||
"description": "HTTP请求错误信息,成功时返回空",
|
||||
"valueType": "object",
|
||||
"type": "static"
|
||||
},
|
||||
{
|
||||
"id": "httpRawResponse",
|
||||
"key": "httpRawResponse",
|
||||
"label": "原始响应",
|
||||
"required": true,
|
||||
"description": "HTTP请求的原始响应。只能接受字符串或JSON类型响应数据。",
|
||||
"valueType": "any",
|
||||
"type": "static"
|
||||
},
|
||||
{
|
||||
"id": "rH4tMV02robs",
|
||||
"valueType": "string",
|
||||
@@ -260,6 +308,34 @@
|
||||
"sourceHandle": "ebLCxU43hHuZ-source-right",
|
||||
"targetHandle": "i7uow4wj2wdp-target-left"
|
||||
}
|
||||
]
|
||||
],
|
||||
"chatConfig": {
|
||||
"welcomeText": "",
|
||||
"variables": [],
|
||||
"questionGuide": {
|
||||
"open": false,
|
||||
"model": "gpt-4o-mini",
|
||||
"customPrompt": "You are an AI assistant tasked with predicting the user's next question based on the conversation history. Your goal is to generate 3 potential questions that will guide the user to continue the conversation. When generating these questions, adhere to the following rules:\n\n1. Use the same language as the user's last question in the conversation history.\n2. Keep each question under 20 characters in length.\n\nAnalyze the conversation history provided to you and use it as context to generate relevant and engaging follow-up questions. Your predictions should be logical extensions of the current topic or related areas that the user might be interested in exploring further.\n\nRemember to maintain consistency in tone and style with the existing conversation while providing diverse options for the user to choose from. Your goal is to keep the conversation flowing naturally and help the user delve deeper into the subject matter or explore related topics."
|
||||
},
|
||||
"ttsConfig": {
|
||||
"type": "web"
|
||||
},
|
||||
"whisperConfig": {
|
||||
"open": false,
|
||||
"autoSend": false,
|
||||
"autoTTSResponse": false
|
||||
},
|
||||
"chatInputGuide": {
|
||||
"open": false,
|
||||
"textList": [],
|
||||
"customUrl": ""
|
||||
},
|
||||
"instruction": "",
|
||||
"autoExecute": {
|
||||
"open": false,
|
||||
"defaultPrompt": ""
|
||||
},
|
||||
"_id": "677b59849d672185a5671b45"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,13 +3,16 @@ import { PassThrough } from 'stream';
|
||||
|
||||
export const gridFsStream2Buffer = (stream: NodeJS.ReadableStream) => {
|
||||
return new Promise<Buffer>((resolve, reject) => {
|
||||
let tmpBuffer: Buffer = Buffer.from([]);
|
||||
const chunks: Buffer[] = [];
|
||||
let totalLength = 0;
|
||||
|
||||
stream.on('data', (chunk) => {
|
||||
tmpBuffer = Buffer.concat([tmpBuffer, chunk]);
|
||||
chunks.push(chunk);
|
||||
totalLength += chunk.length;
|
||||
});
|
||||
stream.on('end', () => {
|
||||
resolve(tmpBuffer);
|
||||
const resultBuffer = Buffer.concat(chunks, totalLength); // 一次性拼接
|
||||
resolve(resultBuffer);
|
||||
});
|
||||
stream.on('error', (err) => {
|
||||
reject(err);
|
||||
|
||||
@@ -118,7 +118,7 @@ export async function delImgByRelatedId({
|
||||
}: {
|
||||
teamId: string;
|
||||
relateIds: string[];
|
||||
session: ClientSession;
|
||||
session?: ClientSession;
|
||||
}) {
|
||||
if (relateIds.length === 0) return;
|
||||
|
||||
|
||||
@@ -38,10 +38,12 @@ const addCommonMiddleware = (schema: mongoose.Schema) => {
|
||||
schema.post(op, function (this: any, result: any, next) {
|
||||
if (this._startTime) {
|
||||
const duration = Date.now() - this._startTime;
|
||||
|
||||
const warnLogData = {
|
||||
query: this._query,
|
||||
op,
|
||||
collectionName: this.collection?.name,
|
||||
op: this.op,
|
||||
...(this._query && { query: this._query }),
|
||||
...(this._update && { update: this._update }),
|
||||
...(this._delete && { delete: this._delete }),
|
||||
duration
|
||||
};
|
||||
|
||||
|
||||
@@ -16,16 +16,30 @@ export async function connectMongo(): Promise<Mongoose> {
|
||||
|
||||
console.log('mongo start connect');
|
||||
try {
|
||||
connectionMongo.set('strictQuery', true);
|
||||
// Remove existing listeners to prevent duplicates
|
||||
connectionMongo.connection.removeAllListeners('error');
|
||||
connectionMongo.connection.removeAllListeners('disconnected');
|
||||
connectionMongo.set('strictQuery', false);
|
||||
|
||||
connectionMongo.connection.on('error', async (error) => {
|
||||
console.log('mongo error', error);
|
||||
await connectionMongo.disconnect();
|
||||
await delay(1000);
|
||||
connectMongo();
|
||||
try {
|
||||
if (connectionMongo.connection.readyState !== 0) {
|
||||
await connectionMongo.disconnect();
|
||||
await delay(1000);
|
||||
await connectMongo();
|
||||
}
|
||||
} catch (error) {}
|
||||
});
|
||||
connectionMongo.connection.on('disconnected', () => {
|
||||
connectionMongo.connection.on('disconnected', async () => {
|
||||
console.log('mongo disconnected');
|
||||
try {
|
||||
if (connectionMongo.connection.readyState !== 0) {
|
||||
await connectionMongo.disconnect();
|
||||
await delay(1000);
|
||||
await connectMongo();
|
||||
}
|
||||
} catch (error) {}
|
||||
});
|
||||
|
||||
await connectionMongo.connect(process.env.MONGODB_URI as string, {
|
||||
|
||||
@@ -2,6 +2,7 @@ import { UrlFetchParams, UrlFetchResponse } from '@fastgpt/global/common/file/ap
|
||||
import * as cheerio from 'cheerio';
|
||||
import axios from 'axios';
|
||||
import { htmlToMarkdown } from './utils';
|
||||
import { isInternalAddress } from '../system/utils';
|
||||
|
||||
export const cheerioToHtml = ({
|
||||
fetchUrl,
|
||||
@@ -75,6 +76,16 @@ export const urlsFetch = async ({
|
||||
|
||||
const response = await Promise.all(
|
||||
urlList.map(async (url) => {
|
||||
const isInternal = isInternalAddress(url);
|
||||
if (isInternal) {
|
||||
return {
|
||||
url,
|
||||
title: '',
|
||||
content: 'Cannot fetch internal url',
|
||||
selector: ''
|
||||
};
|
||||
}
|
||||
|
||||
try {
|
||||
const fetchRes = await axios.get(url, {
|
||||
timeout: 30000
|
||||
|
||||
63
packages/service/common/system/utils.ts
Normal file
63
packages/service/common/system/utils.ts
Normal file
@@ -0,0 +1,63 @@
|
||||
import { SERVICE_LOCAL_HOST } from './tools';
|
||||
|
||||
export const isInternalAddress = (url: string): boolean => {
|
||||
try {
|
||||
const parsedUrl = new URL(url);
|
||||
const hostname = parsedUrl.hostname;
|
||||
const fullUrl = parsedUrl.toString();
|
||||
|
||||
// Check for localhost and common internal domains
|
||||
if (hostname === SERVICE_LOCAL_HOST) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// Metadata endpoints whitelist
|
||||
const metadataEndpoints = [
|
||||
// AWS
|
||||
'http://169.254.169.254/latest/meta-data/',
|
||||
// Azure
|
||||
'http://169.254.169.254/metadata/instance?api-version=2021-02-01',
|
||||
// GCP
|
||||
'http://metadata.google.internal/computeMetadata/v1/',
|
||||
// Alibaba Cloud
|
||||
'http://100.100.100.200/latest/meta-data/',
|
||||
// Tencent Cloud
|
||||
'http://metadata.tencentyun.com/latest/meta-data/',
|
||||
// Huawei Cloud
|
||||
'http://169.254.169.254/latest/meta-data/'
|
||||
];
|
||||
if (metadataEndpoints.some((endpoint) => fullUrl.startsWith(endpoint))) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// For non-metadata URLs, check if it's a domain name
|
||||
const ipv4Pattern = /^(\d{1,3}\.){3}\d{1,3}$/;
|
||||
if (!ipv4Pattern.test(hostname)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// ... existing IP validation code ...
|
||||
const parts = hostname.split('.').map(Number);
|
||||
|
||||
if (parts.length !== 4 || parts.some((part) => part < 0 || part > 255)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Only allow public IP ranges
|
||||
return (
|
||||
parts[0] !== 0 &&
|
||||
parts[0] !== 10 &&
|
||||
parts[0] !== 127 &&
|
||||
!(parts[0] === 169 && parts[1] === 254) &&
|
||||
!(parts[0] === 172 && parts[1] >= 16 && parts[1] <= 31) &&
|
||||
!(parts[0] === 192 && parts[1] === 168) &&
|
||||
!(parts[0] >= 224 && parts[0] <= 239) &&
|
||||
!(parts[0] >= 240 && parts[0] <= 255) &&
|
||||
!(parts[0] === 100 && parts[1] >= 64 && parts[1] <= 127) &&
|
||||
!(parts[0] === 9 && parts[1] === 0) &&
|
||||
!(parts[0] === 11 && parts[1] === 0)
|
||||
);
|
||||
} catch {
|
||||
return false; // If URL parsing fails, reject it as potentially unsafe
|
||||
}
|
||||
};
|
||||
@@ -35,7 +35,7 @@ export const getAxiosConfig = (props?: { userKey?: OpenaiAccountType }) => {
|
||||
const { userKey } = props || {};
|
||||
|
||||
const baseUrl = userKey?.baseUrl || global?.systemEnv?.oneapiUrl || openaiBaseUrl;
|
||||
const apiKey = userKey?.key || global?.systemEnv?.chatApiKey || process.env.CHAT_API_KEY || '';
|
||||
const apiKey = userKey?.key || global?.systemEnv?.chatApiKey || openaiBaseKey;
|
||||
|
||||
return {
|
||||
baseUrl,
|
||||
|
||||
@@ -1,4 +1,10 @@
|
||||
{
|
||||
"provider": "AliCloud",
|
||||
"list": []
|
||||
}
|
||||
"list": [
|
||||
{
|
||||
"model": "SenseVoiceSmall",
|
||||
"name": "SenseVoiceSmall",
|
||||
"type": "stt"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -95,11 +95,145 @@ export const llmCompletionsBodyFormat = <T extends CompletionsBodyType>(
|
||||
return requestBody as unknown as InferCompletionsBody<T>;
|
||||
};
|
||||
|
||||
export const llmStreamResponseToText = async (response: StreamChatType) => {
|
||||
export const llmStreamResponseToAnswerText = async (response: StreamChatType) => {
|
||||
let answer = '';
|
||||
for await (const part of response) {
|
||||
const content = part.choices?.[0]?.delta?.content || '';
|
||||
answer += content;
|
||||
}
|
||||
return answer;
|
||||
return parseReasoningContent(answer)[1];
|
||||
};
|
||||
|
||||
// Parse <think></think> tags to think and answer - unstream response
|
||||
export const parseReasoningContent = (text: string): [string, string] => {
|
||||
const regex = /<think>([\s\S]*?)<\/think>/;
|
||||
const match = text.match(regex);
|
||||
|
||||
if (!match) {
|
||||
return ['', text];
|
||||
}
|
||||
|
||||
const thinkContent = match[1].trim();
|
||||
|
||||
// Add answer (remaining text after think tag)
|
||||
const answerContent = text.slice(match.index! + match[0].length);
|
||||
|
||||
return [thinkContent, answerContent];
|
||||
};
|
||||
|
||||
// Parse <think></think> tags to think and answer - stream response
|
||||
export const parseReasoningStreamContent = () => {
|
||||
let isInThinkTag: boolean | undefined;
|
||||
|
||||
const startTag = '<think>';
|
||||
let startTagBuffer = '';
|
||||
|
||||
const endTag = '</think>';
|
||||
let endTagBuffer = '';
|
||||
|
||||
/*
|
||||
parseReasoning - 只控制是否主动解析 <think></think>,如果接口已经解析了,仍然会返回 think 内容。
|
||||
*/
|
||||
const parsePart = (
|
||||
part: {
|
||||
choices: {
|
||||
delta: {
|
||||
content?: string;
|
||||
reasoning_content?: string;
|
||||
};
|
||||
}[];
|
||||
},
|
||||
parseReasoning = false
|
||||
): [string, string] => {
|
||||
const content = part.choices?.[0]?.delta?.content || '';
|
||||
|
||||
// @ts-ignore
|
||||
const reasoningContent = part.choices?.[0]?.delta?.reasoning_content || '';
|
||||
if (reasoningContent || !parseReasoning) {
|
||||
isInThinkTag = false;
|
||||
return [reasoningContent, content];
|
||||
}
|
||||
|
||||
if (!content) {
|
||||
return ['', ''];
|
||||
}
|
||||
|
||||
// 如果不在 think 标签中,或者有 reasoningContent(接口已解析),则返回 reasoningContent 和 content
|
||||
if (isInThinkTag === false) {
|
||||
return ['', content];
|
||||
}
|
||||
|
||||
// 检测是否为 think 标签开头的数据
|
||||
if (isInThinkTag === undefined) {
|
||||
// Parse content think and answer
|
||||
startTagBuffer += content;
|
||||
// 太少内容时候,暂时不解析
|
||||
if (startTagBuffer.length < startTag.length) {
|
||||
return ['', ''];
|
||||
}
|
||||
|
||||
if (startTagBuffer.startsWith(startTag)) {
|
||||
isInThinkTag = true;
|
||||
return [startTagBuffer.slice(startTag.length), ''];
|
||||
}
|
||||
|
||||
// 如果未命中 think 标签,则认为不在 think 标签中,返回 buffer 内容作为 content
|
||||
isInThinkTag = false;
|
||||
return ['', startTagBuffer];
|
||||
}
|
||||
|
||||
// 确认是 think 标签内容,开始返回 think 内容,并实时检测 </think>
|
||||
/*
|
||||
检测 </think> 方案。
|
||||
存储所有疑似 </think> 的内容,直到检测到完整的 </think> 标签或超出 </think> 长度。
|
||||
content 返回值包含以下几种情况:
|
||||
abc - 完全未命中尾标签
|
||||
abc<th - 命中一部分尾标签
|
||||
abc</think> - 完全命中尾标签
|
||||
abc</think>abc - 完全命中尾标签
|
||||
</think>abc - 完全命中尾标签
|
||||
k>abc - 命中一部分尾标签
|
||||
*/
|
||||
// endTagBuffer 专门用来记录疑似尾标签的内容
|
||||
if (endTagBuffer) {
|
||||
endTagBuffer += content;
|
||||
if (endTagBuffer.includes(endTag)) {
|
||||
isInThinkTag = false;
|
||||
const answer = endTagBuffer.slice(endTag.length);
|
||||
return ['', answer];
|
||||
} else if (endTagBuffer.length >= endTag.length) {
|
||||
// 缓存内容超出尾标签长度,且仍未命中 </think>,则认为本次猜测 </think> 失败,仍处于 think 阶段。
|
||||
const tmp = endTagBuffer;
|
||||
endTagBuffer = '';
|
||||
return [tmp, ''];
|
||||
}
|
||||
return ['', ''];
|
||||
} else if (content.includes(endTag)) {
|
||||
// 返回内容,完整命中</think>,直接结束
|
||||
isInThinkTag = false;
|
||||
const [think, answer] = content.split(endTag);
|
||||
return [think, answer];
|
||||
} else {
|
||||
// 无 buffer,且未命中 </think>,开始疑似 </think> 检测。
|
||||
for (let i = 1; i < endTag.length; i++) {
|
||||
const partialEndTag = endTag.slice(0, i);
|
||||
// 命中一部分尾标签
|
||||
if (content.endsWith(partialEndTag)) {
|
||||
const think = content.slice(0, -partialEndTag.length);
|
||||
endTagBuffer += partialEndTag;
|
||||
return [think, ''];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 完全未命中尾标签,还是 think 阶段。
|
||||
return [content, ''];
|
||||
};
|
||||
|
||||
const getStartTagBuffer = () => startTagBuffer;
|
||||
|
||||
return {
|
||||
parsePart,
|
||||
getStartTagBuffer
|
||||
};
|
||||
};
|
||||
|
||||
1
packages/service/core/app/plugin/type.d.ts
vendored
1
packages/service/core/app/plugin/type.d.ts
vendored
@@ -25,6 +25,7 @@ export type SystemPluginConfigSchemaType = {
|
||||
templateType: string;
|
||||
associatedPluginId: string;
|
||||
userGuide: string;
|
||||
author?: string;
|
||||
};
|
||||
};
|
||||
|
||||
|
||||
@@ -18,6 +18,9 @@ const AppTemplateSchema = new Schema({
|
||||
avatar: {
|
||||
type: String
|
||||
},
|
||||
author: {
|
||||
type: String
|
||||
},
|
||||
tags: {
|
||||
type: [String],
|
||||
default: undefined
|
||||
|
||||
@@ -25,6 +25,7 @@ import { MongoImage } from '../../../common/file/image/schema';
|
||||
import { hashStr } from '@fastgpt/global/common/string/tools';
|
||||
import { addDays } from 'date-fns';
|
||||
import { MongoDatasetDataText } from '../data/dataTextSchema';
|
||||
import { delay, retryFn } from '@fastgpt/global/common/system/utils';
|
||||
|
||||
export const createCollectionAndInsertData = async ({
|
||||
dataset,
|
||||
@@ -216,7 +217,7 @@ export async function createOneCollection({
|
||||
nextSyncTime
|
||||
}
|
||||
],
|
||||
{ session }
|
||||
{ session, ordered: true }
|
||||
);
|
||||
|
||||
return collection;
|
||||
@@ -227,8 +228,14 @@ export const delCollectionRelatedSource = async ({
|
||||
collections,
|
||||
session
|
||||
}: {
|
||||
collections: DatasetCollectionSchemaType[];
|
||||
session: ClientSession;
|
||||
collections: {
|
||||
teamId: string;
|
||||
fileId?: string;
|
||||
metadata?: {
|
||||
relatedImgId?: string;
|
||||
};
|
||||
}[];
|
||||
session?: ClientSession;
|
||||
}) => {
|
||||
if (collections.length === 0) return;
|
||||
|
||||
@@ -259,11 +266,13 @@ export const delCollectionRelatedSource = async ({
|
||||
export async function delCollection({
|
||||
collections,
|
||||
session,
|
||||
delRelatedSource
|
||||
delImg = true,
|
||||
delFile = true
|
||||
}: {
|
||||
collections: DatasetCollectionSchemaType[];
|
||||
session: ClientSession;
|
||||
delRelatedSource: boolean;
|
||||
delImg: boolean;
|
||||
delFile: boolean;
|
||||
}) {
|
||||
if (collections.length === 0) return;
|
||||
|
||||
@@ -274,83 +283,55 @@ export async function delCollection({
|
||||
const datasetIds = Array.from(new Set(collections.map((item) => String(item.datasetId))));
|
||||
const collectionIds = collections.map((item) => String(item._id));
|
||||
|
||||
// Delete training data
|
||||
await MongoDatasetTraining.deleteMany({
|
||||
teamId,
|
||||
datasetIds: { $in: datasetIds },
|
||||
collectionId: { $in: collectionIds }
|
||||
await retryFn(async () => {
|
||||
await Promise.all([
|
||||
// Delete training data
|
||||
MongoDatasetTraining.deleteMany({
|
||||
teamId,
|
||||
datasetId: { $in: datasetIds },
|
||||
collectionId: { $in: collectionIds }
|
||||
}),
|
||||
// Delete dataset_data_texts
|
||||
MongoDatasetDataText.deleteMany({
|
||||
teamId,
|
||||
datasetId: { $in: datasetIds },
|
||||
collectionId: { $in: collectionIds }
|
||||
}),
|
||||
// Delete dataset_datas
|
||||
MongoDatasetData.deleteMany({
|
||||
teamId,
|
||||
datasetId: { $in: datasetIds },
|
||||
collectionId: { $in: collectionIds }
|
||||
}),
|
||||
...(delImg
|
||||
? [
|
||||
delImgByRelatedId({
|
||||
teamId,
|
||||
relateIds: collections
|
||||
.map((item) => item?.metadata?.relatedImgId || '')
|
||||
.filter(Boolean)
|
||||
})
|
||||
]
|
||||
: []),
|
||||
...(delFile
|
||||
? [
|
||||
delFileByFileIdList({
|
||||
bucketName: BucketNameEnum.dataset,
|
||||
fileIdList: collections.map((item) => item?.fileId || '').filter(Boolean)
|
||||
})
|
||||
]
|
||||
: []),
|
||||
// Delete vector data
|
||||
deleteDatasetDataVector({ teamId, datasetIds, collectionIds })
|
||||
]);
|
||||
|
||||
// delete collections
|
||||
await MongoDatasetCollection.deleteMany(
|
||||
{
|
||||
teamId,
|
||||
_id: { $in: collectionIds }
|
||||
},
|
||||
{ session }
|
||||
);
|
||||
});
|
||||
|
||||
/* file and imgs */
|
||||
if (delRelatedSource) {
|
||||
await delCollectionRelatedSource({ collections, session });
|
||||
}
|
||||
|
||||
// Delete dataset_datas
|
||||
await MongoDatasetData.deleteMany(
|
||||
{ teamId, datasetIds: { $in: datasetIds }, collectionId: { $in: collectionIds } },
|
||||
{ session }
|
||||
);
|
||||
// Delete dataset_data_texts
|
||||
await MongoDatasetDataText.deleteMany(
|
||||
{ teamId, datasetIds: { $in: datasetIds }, collectionId: { $in: collectionIds } },
|
||||
{ session }
|
||||
);
|
||||
|
||||
// delete collections
|
||||
await MongoDatasetCollection.deleteMany(
|
||||
{
|
||||
teamId,
|
||||
_id: { $in: collectionIds }
|
||||
},
|
||||
{ session }
|
||||
);
|
||||
|
||||
// no session delete: delete files, vector data
|
||||
await deleteDatasetDataVector({ teamId, datasetIds, collectionIds });
|
||||
}
|
||||
|
||||
/**
|
||||
* delete delOnlyCollection
|
||||
*/
|
||||
export async function delOnlyCollection({
|
||||
collections,
|
||||
session
|
||||
}: {
|
||||
collections: DatasetCollectionSchemaType[];
|
||||
session: ClientSession;
|
||||
}) {
|
||||
if (collections.length === 0) return;
|
||||
|
||||
const teamId = collections[0].teamId;
|
||||
|
||||
if (!teamId) return Promise.reject('teamId is not exist');
|
||||
|
||||
const datasetIds = Array.from(new Set(collections.map((item) => String(item.datasetId))));
|
||||
const collectionIds = collections.map((item) => String(item._id));
|
||||
|
||||
// delete training data
|
||||
await MongoDatasetTraining.deleteMany({
|
||||
teamId,
|
||||
datasetIds: { $in: datasetIds },
|
||||
collectionId: { $in: collectionIds }
|
||||
});
|
||||
|
||||
// delete dataset.datas
|
||||
await MongoDatasetData.deleteMany(
|
||||
{ teamId, datasetIds: { $in: datasetIds }, collectionId: { $in: collectionIds } },
|
||||
{ session }
|
||||
);
|
||||
|
||||
// delete collections
|
||||
await MongoDatasetCollection.deleteMany(
|
||||
{
|
||||
teamId,
|
||||
_id: { $in: collectionIds }
|
||||
},
|
||||
{ session }
|
||||
);
|
||||
|
||||
// no session delete: delete files, vector data
|
||||
await deleteDatasetDataVector({ teamId, datasetIds, collectionIds });
|
||||
}
|
||||
|
||||
@@ -97,7 +97,7 @@ export const createOrGetCollectionTags = async ({
|
||||
datasetId,
|
||||
tag: tagContent
|
||||
})),
|
||||
{ session }
|
||||
{ session, ordered: true }
|
||||
);
|
||||
|
||||
return [...existingTags.map((tag) => tag._id), ...newTags.map((tag) => tag._id)];
|
||||
@@ -174,6 +174,14 @@ export const syncCollection = async (collection: CollectionWithDatasetType) => {
|
||||
}
|
||||
|
||||
await mongoSessionRun(async (session) => {
|
||||
// Delete old collection
|
||||
await delCollection({
|
||||
collections: [collection],
|
||||
delImg: false,
|
||||
delFile: false,
|
||||
session
|
||||
});
|
||||
|
||||
// Create new collection
|
||||
await createCollectionAndInsertData({
|
||||
session,
|
||||
@@ -208,13 +216,6 @@ export const syncCollection = async (collection: CollectionWithDatasetType) => {
|
||||
updateTime: new Date()
|
||||
}
|
||||
});
|
||||
|
||||
// Delete old collection
|
||||
await delCollection({
|
||||
collections: [collection],
|
||||
delRelatedSource: false,
|
||||
session
|
||||
});
|
||||
});
|
||||
|
||||
return DatasetCollectionSyncResultEnum.success;
|
||||
|
||||
@@ -7,6 +7,8 @@ import { MongoDatasetTraining } from './training/schema';
|
||||
import { MongoDatasetData } from './data/schema';
|
||||
import { deleteDatasetDataVector } from '../../common/vectorStore/controller';
|
||||
import { MongoDatasetDataText } from './data/dataTextSchema';
|
||||
import { DatasetErrEnum } from '@fastgpt/global/common/error/code/dataset';
|
||||
import { retryFn } from '@fastgpt/global/common/system/utils';
|
||||
|
||||
/* ============= dataset ========== */
|
||||
/* find all datasetId by top datasetId */
|
||||
@@ -54,7 +56,7 @@ export async function getCollectionWithDataset(collectionId: string) {
|
||||
.populate<{ dataset: DatasetSchemaType }>('dataset')
|
||||
.lean();
|
||||
if (!data) {
|
||||
return Promise.reject('Collection is not exist');
|
||||
return Promise.reject(DatasetErrEnum.unExistCollection);
|
||||
}
|
||||
return data;
|
||||
}
|
||||
@@ -77,40 +79,39 @@ export async function delDatasetRelevantData({
|
||||
|
||||
const datasetIds = datasets.map((item) => item._id);
|
||||
|
||||
// delete training data
|
||||
await MongoDatasetTraining.deleteMany({
|
||||
teamId,
|
||||
datasetId: { $in: datasetIds }
|
||||
});
|
||||
|
||||
// Get _id, teamId, fileId, metadata.relatedImgId for all collections
|
||||
const collections = await MongoDatasetCollection.find(
|
||||
{
|
||||
teamId,
|
||||
datasetId: { $in: datasetIds }
|
||||
},
|
||||
'_id teamId datasetId fileId metadata',
|
||||
{ session }
|
||||
'_id teamId datasetId fileId metadata'
|
||||
).lean();
|
||||
|
||||
// Delete Image and file
|
||||
await delCollectionRelatedSource({ collections, session });
|
||||
await retryFn(async () => {
|
||||
await Promise.all([
|
||||
// delete training data
|
||||
MongoDatasetTraining.deleteMany({
|
||||
teamId,
|
||||
datasetId: { $in: datasetIds }
|
||||
}),
|
||||
//Delete dataset_data_texts
|
||||
MongoDatasetDataText.deleteMany({
|
||||
teamId,
|
||||
datasetId: { $in: datasetIds }
|
||||
}),
|
||||
//delete dataset_datas
|
||||
MongoDatasetData.deleteMany({ teamId, datasetId: { $in: datasetIds } }),
|
||||
// Delete Image and file
|
||||
delCollectionRelatedSource({ collections }),
|
||||
// Delete vector data
|
||||
deleteDatasetDataVector({ teamId, datasetIds })
|
||||
]);
|
||||
});
|
||||
|
||||
// delete collections
|
||||
await MongoDatasetCollection.deleteMany({
|
||||
teamId,
|
||||
datasetId: { $in: datasetIds }
|
||||
}).session(session);
|
||||
|
||||
// No session delete:
|
||||
// Delete dataset_data_texts
|
||||
await MongoDatasetDataText.deleteMany({
|
||||
teamId,
|
||||
datasetId: { $in: datasetIds }
|
||||
});
|
||||
// delete dataset_datas
|
||||
await MongoDatasetData.deleteMany({ teamId, datasetId: { $in: datasetIds } });
|
||||
|
||||
// Delete vector data
|
||||
await deleteDatasetDataVector({ teamId, datasetIds });
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { connectionMongo, getMongoModel } from '../../../common/mongo';
|
||||
const { Schema } = connectionMongo;
|
||||
import { DatasetDataSchemaType } from '@fastgpt/global/core/dataset/type.d';
|
||||
import { DatasetDataTextSchemaType } from '@fastgpt/global/core/dataset/type.d';
|
||||
import { TeamCollectionName } from '@fastgpt/global/support/user/team/constant';
|
||||
import { DatasetCollectionName } from '../schema';
|
||||
import { DatasetColCollectionName } from '../collection/schema';
|
||||
@@ -40,12 +40,13 @@ try {
|
||||
default_language: 'none'
|
||||
}
|
||||
);
|
||||
DatasetDataTextSchema.index({ teamId: 1, datasetId: 1, collectionId: 1 });
|
||||
DatasetDataTextSchema.index({ dataId: 1 }, { unique: true });
|
||||
} catch (error) {
|
||||
console.log(error);
|
||||
}
|
||||
|
||||
export const MongoDatasetDataText = getMongoModel<DatasetDataSchemaType>(
|
||||
export const MongoDatasetDataText = getMongoModel<DatasetDataTextSchemaType>(
|
||||
DatasetDataTextCollectionName,
|
||||
DatasetDataTextSchema
|
||||
);
|
||||
|
||||
@@ -3,11 +3,8 @@ import { filterGPTMessageByMaxContext, loadRequestMessages } from '../../../chat
|
||||
import type { ChatItemType, UserChatItemValueItemType } from '@fastgpt/global/core/chat/type.d';
|
||||
import { ChatRoleEnum } from '@fastgpt/global/core/chat/constants';
|
||||
import { SseResponseEventEnum } from '@fastgpt/global/core/workflow/runtime/constants';
|
||||
import {
|
||||
parseReasoningContent,
|
||||
parseReasoningStreamContent,
|
||||
textAdaptGptResponse
|
||||
} from '@fastgpt/global/core/workflow/runtime/utils';
|
||||
import { textAdaptGptResponse } from '@fastgpt/global/core/workflow/runtime/utils';
|
||||
import { parseReasoningContent, parseReasoningStreamContent } from '../../../ai/utils';
|
||||
import { createChatCompletion } from '../../../ai/config';
|
||||
import type { ChatCompletionMessageParam, StreamChatType } from '@fastgpt/global/core/ai/type.d';
|
||||
import { formatModelChars2Points } from '../../../../support/wallet/usage/utils';
|
||||
|
||||
@@ -62,6 +62,7 @@ export const dispatchLoop = async (props: Props): Promise<Response> => {
|
||||
|
||||
const response = await dispatchWorkFlow({
|
||||
...props,
|
||||
variables: newVariables,
|
||||
runtimeEdges: cloneDeep(runtimeEdges)
|
||||
});
|
||||
|
||||
|
||||
@@ -120,27 +120,144 @@ export const dispatchHttp468Request = async (props: HttpRequestProps): Promise<H
|
||||
2. Replace newline strings
|
||||
*/
|
||||
const replaceJsonBodyString = (text: string) => {
|
||||
const valToStr = (val: any) => {
|
||||
// Check if the variable is in quotes
|
||||
const isVariableInQuotes = (text: string, variable: string) => {
|
||||
const index = text.indexOf(variable);
|
||||
if (index === -1) return false;
|
||||
|
||||
// 计算变量前面的引号数量
|
||||
const textBeforeVar = text.substring(0, index);
|
||||
const matches = textBeforeVar.match(/"/g) || [];
|
||||
|
||||
// 如果引号数量为奇数,则变量在引号内
|
||||
return matches.length % 2 === 1;
|
||||
};
|
||||
const valToStr = (val: any, isQuoted = false) => {
|
||||
if (val === undefined) return 'null';
|
||||
if (val === null) return 'null';
|
||||
|
||||
if (typeof val === 'object') return JSON.stringify(val);
|
||||
|
||||
if (typeof val === 'string') {
|
||||
if (isQuoted) {
|
||||
return val.replace(/(?<!\\)"/g, '\\"');
|
||||
}
|
||||
try {
|
||||
const parsed = JSON.parse(val);
|
||||
if (typeof parsed === 'object') {
|
||||
return JSON.stringify(parsed);
|
||||
}
|
||||
JSON.parse(val);
|
||||
return val;
|
||||
} catch (error) {
|
||||
const str = JSON.stringify(val);
|
||||
|
||||
return str.startsWith('"') && str.endsWith('"') ? str.slice(1, -1) : str;
|
||||
}
|
||||
}
|
||||
|
||||
return String(val);
|
||||
};
|
||||
// Test cases for variable replacement in JSON body
|
||||
// const bodyTest = () => {
|
||||
// const testData = [
|
||||
// // 基本字符串替换
|
||||
// {
|
||||
// body: `{"name":"{{name}}","age":"18"}`,
|
||||
// variables: [{ key: '{{name}}', value: '测试' }],
|
||||
// result: `{"name":"测试","age":"18"}`
|
||||
// },
|
||||
// // 特殊字符处理
|
||||
// {
|
||||
// body: `{"text":"{{text}}"}`,
|
||||
// variables: [{ key: '{{text}}', value: '包含"引号"和\\反斜杠' }],
|
||||
// result: `{"text":"包含\\"引号\\"和\\反斜杠"}`
|
||||
// },
|
||||
// // 数字类型处理
|
||||
// {
|
||||
// body: `{"count":{{count}},"price":{{price}}}`,
|
||||
// variables: [
|
||||
// { key: '{{count}}', value: '42' },
|
||||
// { key: '{{price}}', value: '99.99' }
|
||||
// ],
|
||||
// result: `{"count":42,"price":99.99}`
|
||||
// },
|
||||
// // 布尔值处理
|
||||
// {
|
||||
// body: `{"isActive":{{isActive}},"hasData":{{hasData}}}`,
|
||||
// variables: [
|
||||
// { key: '{{isActive}}', value: 'true' },
|
||||
// { key: '{{hasData}}', value: 'false' }
|
||||
// ],
|
||||
// result: `{"isActive":true,"hasData":false}`
|
||||
// },
|
||||
// // 对象类型处理
|
||||
// {
|
||||
// body: `{"user":{{user}},"user2":"{{user2}}"}`,
|
||||
// variables: [
|
||||
// { key: '{{user}}', value: `{"id":1,"name":"张三"}` },
|
||||
// { key: '{{user2}}', value: `{"id":1,"name":"张三"}` }
|
||||
// ],
|
||||
// result: `{"user":{"id":1,"name":"张三"},"user2":"{\\"id\\":1,\\"name\\":\\"张三\\"}"}`
|
||||
// },
|
||||
// // 数组类型处理
|
||||
// {
|
||||
// body: `{"items":{{items}}}`,
|
||||
// variables: [{ key: '{{items}}', value: '[1, 2, 3]' }],
|
||||
// result: `{"items":[1,2,3]}`
|
||||
// },
|
||||
// // null 和 undefined 处理
|
||||
// {
|
||||
// body: `{"nullValue":{{nullValue}},"undefinedValue":{{undefinedValue}}}`,
|
||||
// variables: [
|
||||
// { key: '{{nullValue}}', value: 'null' },
|
||||
// { key: '{{undefinedValue}}', value: 'undefined' }
|
||||
// ],
|
||||
// result: `{"nullValue":null,"undefinedValue":null}`
|
||||
// },
|
||||
// // 嵌套JSON结构
|
||||
// {
|
||||
// body: `{"data":{"nested":{"value":"{{nestedValue}}"}}}`,
|
||||
// variables: [{ key: '{{nestedValue}}', value: '嵌套值' }],
|
||||
// result: `{"data":{"nested":{"value":"嵌套值"}}}`
|
||||
// },
|
||||
// // 多变量替换
|
||||
// {
|
||||
// body: `{"first":"{{first}}","second":"{{second}}","third":{{third}}}`,
|
||||
// variables: [
|
||||
// { key: '{{first}}', value: '第一' },
|
||||
// { key: '{{second}}', value: '第二' },
|
||||
// { key: '{{third}}', value: '3' }
|
||||
// ],
|
||||
// result: `{"first":"第一","second":"第二","third":3}`
|
||||
// },
|
||||
// // JSON字符串作为变量值
|
||||
// {
|
||||
// body: `{"config":{{config}}}`,
|
||||
// variables: [{ key: '{{config}}', value: '{"setting":"enabled","mode":"advanced"}' }],
|
||||
// result: `{"config":{"setting":"enabled","mode":"advanced"}}`
|
||||
// }
|
||||
// ];
|
||||
|
||||
// for (let i = 0; i < testData.length; i++) {
|
||||
// const item = testData[i];
|
||||
// let bodyStr = item.body;
|
||||
// for (const variable of item.variables) {
|
||||
// const isQuote = isVariableInQuotes(bodyStr, variable.key);
|
||||
// bodyStr = bodyStr.replace(variable.key, valToStr(variable.value, isQuote));
|
||||
// }
|
||||
// bodyStr = bodyStr.replace(/(".*?")\s*:\s*undefined\b/g, '$1:null');
|
||||
|
||||
// console.log(bodyStr === item.result, i);
|
||||
// if (bodyStr !== item.result) {
|
||||
// console.log(bodyStr);
|
||||
// console.log(item.result);
|
||||
// } else {
|
||||
// try {
|
||||
// JSON.parse(item.result);
|
||||
// } catch (error) {
|
||||
// console.log('反序列化异常', i, item.result);
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// };
|
||||
// bodyTest();
|
||||
|
||||
// 1. Replace {{key.key}} variables
|
||||
const regex1 = /\{\{\$([^.]+)\.([^$]+)\$\}\}/g;
|
||||
@@ -148,6 +265,10 @@ export const dispatchHttp468Request = async (props: HttpRequestProps): Promise<H
|
||||
matches1.forEach((match) => {
|
||||
const nodeId = match[1];
|
||||
const id = match[2];
|
||||
const fullMatch = match[0];
|
||||
|
||||
// 检查变量是否在引号内
|
||||
const isInQuotes = isVariableInQuotes(text, fullMatch);
|
||||
|
||||
const variableVal = (() => {
|
||||
if (nodeId === VARIABLE_NODE_ID) {
|
||||
@@ -165,9 +286,9 @@ export const dispatchHttp468Request = async (props: HttpRequestProps): Promise<H
|
||||
return getReferenceVariableValue({ value: input.value, nodes: runtimeNodes, variables });
|
||||
})();
|
||||
|
||||
const formatVal = valToStr(variableVal);
|
||||
const formatVal = valToStr(variableVal, isInQuotes);
|
||||
|
||||
const regex = new RegExp(`\\{\\{\\$(${nodeId}\\.${id})\\$\\}\\}`, 'g');
|
||||
const regex = new RegExp(`\\{\\{\\$(${nodeId}\\.${id})\\$\\}\\}`, '');
|
||||
text = text.replace(regex, () => formatVal);
|
||||
});
|
||||
|
||||
@@ -176,10 +297,16 @@ export const dispatchHttp468Request = async (props: HttpRequestProps): Promise<H
|
||||
const matches2 = text.match(regex2) || [];
|
||||
const uniqueKeys2 = [...new Set(matches2.map((match) => match.slice(2, -2)))];
|
||||
for (const key of uniqueKeys2) {
|
||||
text = text.replace(new RegExp(`{{(${key})}}`, 'g'), () => valToStr(allVariables[key]));
|
||||
const fullMatch = `{{${key}}}`;
|
||||
// 检查变量是否在引号内
|
||||
const isInQuotes = isVariableInQuotes(text, fullMatch);
|
||||
|
||||
text = text.replace(new RegExp(`{{(${key})}}`, ''), () =>
|
||||
valToStr(allVariables[key], isInQuotes)
|
||||
);
|
||||
}
|
||||
|
||||
return text.replace(/(".*?")\s*:\s*undefined\b/g, '$1: null');
|
||||
return text.replace(/(".*?")\s*:\s*undefined\b/g, '$1:null');
|
||||
};
|
||||
|
||||
httpReqUrl = replaceStringVariables(httpReqUrl);
|
||||
|
||||
@@ -196,7 +196,8 @@ export async function syncCollaborators({
|
||||
permission: item.permission
|
||||
})),
|
||||
{
|
||||
session
|
||||
session,
|
||||
ordered: true
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { AppDetailType } from '@fastgpt/global/core/app/type';
|
||||
import { OutLinkSchema } from '@fastgpt/global/support/outLink/type';
|
||||
import { OutlinkAppType, OutLinkSchema } from '@fastgpt/global/support/outLink/type';
|
||||
import { parseHeaderCert } from '../controller';
|
||||
import { MongoOutLink } from '../../outLink/schema';
|
||||
import { OutLinkErrEnum } from '@fastgpt/global/common/error/code/outLink';
|
||||
@@ -54,11 +54,15 @@ export async function authOutLinkCrud({
|
||||
}
|
||||
|
||||
/* outLink exist and it app exist */
|
||||
export async function authOutLinkValid({ shareId }: { shareId?: string }) {
|
||||
export async function authOutLinkValid<T extends OutlinkAppType = any>({
|
||||
shareId
|
||||
}: {
|
||||
shareId?: string;
|
||||
}) {
|
||||
if (!shareId) {
|
||||
return Promise.reject(OutLinkErrEnum.linkUnInvalid);
|
||||
}
|
||||
const outLinkConfig = await MongoOutLink.findOne({ shareId }).lean();
|
||||
const outLinkConfig = await MongoOutLink.findOne({ shareId }).lean<OutLinkSchema<T>>();
|
||||
|
||||
if (!outLinkConfig) {
|
||||
return Promise.reject(OutLinkErrEnum.linkUnInvalid);
|
||||
@@ -66,6 +70,6 @@ export async function authOutLinkValid({ shareId }: { shareId?: string }) {
|
||||
|
||||
return {
|
||||
appId: outLinkConfig.appId,
|
||||
outLinkConfig
|
||||
outLinkConfig: outLinkConfig
|
||||
};
|
||||
}
|
||||
|
||||
@@ -43,7 +43,6 @@ async function getTeamMember(match: Record<string, any>): Promise<TeamTmbItemTyp
|
||||
teamDomain: tmb.team?.teamDomain,
|
||||
role: tmb.role,
|
||||
status: tmb.status,
|
||||
defaultTeam: tmb.defaultTeam,
|
||||
permission: new TeamPermission({
|
||||
per: Per ?? TeamDefaultPermissionVal,
|
||||
isOwner: tmb.role === TeamMemberRoleEnum.owner
|
||||
@@ -71,8 +70,7 @@ export async function getUserDefaultTeam({ userId }: { userId: string }) {
|
||||
return Promise.reject('tmbId or userId is required');
|
||||
}
|
||||
return getTeamMember({
|
||||
userId: new Types.ObjectId(userId),
|
||||
defaultTeam: true
|
||||
userId: new Types.ObjectId(userId)
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -39,14 +39,14 @@ const TeamMemberSchema = new Schema({
|
||||
updateTime: {
|
||||
type: Date
|
||||
},
|
||||
defaultTeam: {
|
||||
type: Boolean,
|
||||
default: false
|
||||
},
|
||||
|
||||
// Abandoned
|
||||
role: {
|
||||
type: String
|
||||
},
|
||||
// Abandoned
|
||||
defaultTeam: {
|
||||
type: Boolean
|
||||
}
|
||||
});
|
||||
|
||||
|
||||
@@ -100,7 +100,7 @@ export const initTeamFreePlan = async ({
|
||||
surplusPoints: freePoints
|
||||
}
|
||||
],
|
||||
{ session }
|
||||
{ session, ordered: true }
|
||||
);
|
||||
};
|
||||
|
||||
|
||||
@@ -160,7 +160,7 @@ export const createTrainingUsage = async ({
|
||||
]
|
||||
}
|
||||
],
|
||||
{ session }
|
||||
{ session, ordered: true }
|
||||
);
|
||||
|
||||
return { billId: String(_id) };
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import React, { useEffect } from 'react';
|
||||
import React, { useEffect, useState } from 'react';
|
||||
import type { IconProps } from '@chakra-ui/react';
|
||||
import { Box, Icon } from '@chakra-ui/react';
|
||||
import { iconPaths } from './constants';
|
||||
@@ -8,7 +8,7 @@ import { useRefresh } from '../../../hooks/useRefresh';
|
||||
const iconCache: Record<string, any> = {};
|
||||
|
||||
const MyIcon = ({ name, w = 'auto', h = 'auto', ...props }: { name: IconNameType } & IconProps) => {
|
||||
const { refresh } = useRefresh();
|
||||
const [, setUpdate] = useState(0);
|
||||
|
||||
useEffect(() => {
|
||||
if (iconCache[name]) {
|
||||
@@ -20,7 +20,7 @@ const MyIcon = ({ name, w = 'auto', h = 'auto', ...props }: { name: IconNameType
|
||||
const component = { as: icon.default };
|
||||
// Store in cache
|
||||
iconCache[name] = component;
|
||||
refresh();
|
||||
setUpdate((prev) => prev + 1); // force update
|
||||
})
|
||||
.catch((error) => console.log(error));
|
||||
}, [name]);
|
||||
|
||||
@@ -214,10 +214,15 @@ export function useScrollPagination<
|
||||
async (init = false, ScrollContainerRef?: RefObject<HTMLDivElement>) => {
|
||||
if (noMore && !init) return;
|
||||
|
||||
if (init) {
|
||||
setData([]);
|
||||
setTotal(0);
|
||||
}
|
||||
|
||||
const offset = init ? 0 : data.length;
|
||||
|
||||
setTrue();
|
||||
console.log(offset);
|
||||
|
||||
try {
|
||||
const res = await api({
|
||||
offset,
|
||||
@@ -288,7 +293,7 @@ export function useScrollPagination<
|
||||
// Watch scroll position
|
||||
useThrottleEffect(
|
||||
() => {
|
||||
if (!ref?.current || noMore) return;
|
||||
if (!ref?.current || noMore || isLoading || data.length === 0) return;
|
||||
const { scrollTop, scrollHeight, clientHeight } = ref.current;
|
||||
|
||||
if (
|
||||
|
||||
@@ -24,6 +24,7 @@
|
||||
"key_type": "API key format:",
|
||||
"log": "Call log",
|
||||
"log_detail": "Log details",
|
||||
"log_request_id_search": "Search by requestId",
|
||||
"log_status": "Status",
|
||||
"mapping": "Model Mapping",
|
||||
"mapping_tip": "A valid Json is required. \nThe model can be mapped when sending a request to the actual address. \nFor example:\n{\n \n \"gpt-4o\": \"gpt-4o-test\"\n\n}\n\nWhen FastGPT requests the gpt-4o model, the gpt-4o-test model is sent to the actual address, instead of gpt-4o.",
|
||||
|
||||
@@ -888,6 +888,7 @@
|
||||
"error.upload_file_error_filename": "{{name}} Upload Failed",
|
||||
"error.upload_image_error": "File upload failed",
|
||||
"error.username_empty": "Account cannot be empty",
|
||||
"error_collection_not_exist": "The collection does not exist",
|
||||
"extraction_results": "Extraction Results",
|
||||
"field_name": "Field Name",
|
||||
"free": "Free",
|
||||
|
||||
@@ -24,6 +24,7 @@
|
||||
"key_type": "API key 格式: ",
|
||||
"log": "调用日志",
|
||||
"log_detail": "日志详情",
|
||||
"log_request_id_search": "根据 requestId 搜索",
|
||||
"log_status": "状态",
|
||||
"mapping": "模型映射",
|
||||
"mapping_tip": "需填写一个有效 Json。可在向实际地址发送请求时,对模型进行映射。例如:\n{\n \"gpt-4o\": \"gpt-4o-test\"\n}\n当 FastGPT 请求 gpt-4o 模型时,会向实际地址发送 gpt-4o-test 的模型,而不是 gpt-4o。",
|
||||
|
||||
@@ -891,6 +891,7 @@
|
||||
"error.upload_file_error_filename": "{{name}} 上传失败",
|
||||
"error.upload_image_error": "上传文件失败",
|
||||
"error.username_empty": "账号不能为空",
|
||||
"error_collection_not_exist": "集合不存在",
|
||||
"extraction_results": "提取结果",
|
||||
"field_name": "字段名",
|
||||
"free": "免费",
|
||||
|
||||
@@ -22,6 +22,7 @@
|
||||
"key_type": "API key 格式:",
|
||||
"log": "調用日誌",
|
||||
"log_detail": "日誌詳情",
|
||||
"log_request_id_search": "根據 requestId 搜索",
|
||||
"log_status": "狀態",
|
||||
"mapping": "模型映射",
|
||||
"mapping_tip": "需填寫一個有效 Json。\n可在向實際地址發送請求時,對模型進行映射。\n例如:\n{\n \n \"gpt-4o\": \"gpt-4o-test\"\n\n}\n\n當 FastGPT 請求 gpt-4o 模型時,會向實際地址發送 gpt-4o-test 的模型,而不是 gpt-4o。",
|
||||
|
||||
@@ -888,6 +888,7 @@
|
||||
"error.upload_file_error_filename": "{{name}} 上傳失敗",
|
||||
"error.upload_image_error": "上傳文件失敗",
|
||||
"error.username_empty": "帳號不能為空",
|
||||
"error_collection_not_exist": "集合不存在",
|
||||
"extraction_results": "提取結果",
|
||||
"field_name": "欄位名稱",
|
||||
"free": "免費",
|
||||
|
||||
5
plugins/README.md
Normal file
5
plugins/README.md
Normal file
@@ -0,0 +1,5 @@
|
||||
# 目录说明
|
||||
|
||||
该目录为 FastGPT 辅助子项目,非必须。
|
||||
|
||||
- model 私有化模型
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user