diff --git a/.github/workflows/preview-fastgpt-image.yml b/.github/workflows/preview-fastgpt-image.yml index 4db6e43ad..03d426304 100644 --- a/.github/workflows/preview-fastgpt-image.yml +++ b/.github/workflows/preview-fastgpt-image.yml @@ -36,7 +36,7 @@ jobs: password: ${{ secrets.GH_PAT }} - name: Set DOCKER_REPO_TAGGED based on branch or tag run: | - echo "DOCKER_REPO_TAGGED=ghcr.io/${{ github.repository_owner }}/fastgpt-pr:${{ github.event.pull_request.number }}" >> $GITHUB_ENV + echo "DOCKER_REPO_TAGGED=ghcr.io/${{ github.repository_owner }}/fastgpt-pr:${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV - name: Build image for PR env: DOCKER_REPO_TAGGED: ${{ env.DOCKER_REPO_TAGGED }} @@ -44,13 +44,30 @@ jobs: docker buildx build \ -f projects/app/Dockerfile \ --label "org.opencontainers.image.source=https://github.com/${{ github.repository_owner }}/FastGPT" \ - --label "org.opencontainers.image.description=fastgpt-pr imae" \ + --label "org.opencontainers.image.description=fastgpt-pr image" \ --label "org.opencontainers.image.licenses=Apache" \ --push \ --cache-from=type=local,src=/tmp/.buildx-cache \ --cache-to=type=local,dest=/tmp/.buildx-cache \ -t ${DOCKER_REPO_TAGGED} \ . + # Add write md step after build + - name: Write md + run: | + echo "# 🤖 Generated by deploy action" > report.md + echo "📦 Preview Image: \`${DOCKER_REPO_TAGGED}\`" >> report.md + cat report.md + + - name: Gh Rebot for Sealos + uses: labring/gh-rebot@v0.0.6 + if: ${{ (github.event_name == 'pull_request_target') }} + with: + version: v0.0.6 + env: + GH_TOKEN: '${{ secrets.GH_PAT }}' + SEALOS_TYPE: 'pr_comment' + SEALOS_FILENAME: 'report.md' + SEALOS_REPLACE_TAG: 'DEFAULT_REPLACE_DEPLOY' helm-check: runs-on: ubuntu-20.04 diff --git a/README.md b/README.md index 520ffc8bd..007d4c3f3 100644 --- a/README.md +++ b/README.md @@ -58,9 +58,9 @@ https://github.com/labring/FastGPT/assets/15308462/7d3a38df-eb0e-4388-9250-2409b - [x] 多库复用，混用 - [x] chunk 记录修改和删除 - [x] 支持手动输入，直接分段，QA 拆分导入 - - [x] 支持 txt，md，html，pdf，docx，pptx，csv，xlsx (有需要更多可 PR file loader) - - [x] 支持 url 读取、CSV 批量导入 + - [x] 支持 txt，md，html，pdf，docx，pptx，csv，xlsx (有需要更多可 PR file loader)，支持 url 读取、CSV 批量导入 - [x] 混合检索 & 重排 + - [x] API 知识库 - [ ] 自定义文件读取服务 - [ ] 自定义分块服务 @@ -69,7 +69,7 @@ https://github.com/labring/FastGPT/assets/15308462/7d3a38df-eb0e-4388-9250-2409b - [x] 对话时反馈引用并可修改与删除 - [x] 完整上下文呈现 - [x] 完整模块中间值呈现 - - [x] 高级编排 DeBug 模式 + - [ ] 高级编排 DeBug 模式 `4` OpenAPI 接口 - [x] completions 接口 (chat 模式对齐 GPT 接口) @@ -104,7 +104,7 @@ https://github.com/labring/FastGPT/assets/15308462/7d3a38df-eb0e-4388-9250-2409b * [快速开始本地开发](https://doc.tryfastgpt.ai/docs/development/intro/) * [部署 FastGPT](https://doc.tryfastgpt.ai/docs/development/sealos/) * [系统配置文件说明](https://doc.tryfastgpt.ai/docs/development/configuration/) -* [多模型配置](https://doc.tryfastgpt.ai/docs/development/one-api/) +* [多模型配置方案](https://doc.tryfastgpt.ai/docs/development/modelconfig/one-api/) * [版本更新/升级介绍](https://doc.tryfastgpt.ai/docs/development/upgrading/) * [OpenAPI API 文档](https://doc.tryfastgpt.ai/docs/development/openapi/) * [知识库结构详解](https://doc.tryfastgpt.ai/docs/guide/knowledge_base/rag/) @@ -127,7 +127,6 @@ https://github.com/labring/FastGPT/assets/15308462/7d3a38df-eb0e-4388-9250-2409b 我们正在寻找志同道合的小伙伴，加速 FastGPT 的发展。你可以通过 [FastGPT 2025 招聘](https://fael3z0zfze.feishu.cn/wiki/P7FOwEmPziVcaYkvVaacnVX1nvg)了解 FastGPT 的招聘信息。 - ## 💪 相关项目 - [Laf：3 分钟快速接入三方应用](https://github.com/labring/laf) @@ -139,19 +138,21 @@ https://github.com/labring/FastGPT/assets/15308462/7d3a38df-eb0e-4388-9250-2409b # -## 👀 其他 -- [保姆级 FastGPT 教程](https://www.bilibili.com/video/BV1n34y1A7Bo/?spm_id_from=333.999.0.0) -- [接入飞书](https://www.bilibili.com/video/BV1Su4y1r7R3/?spm_id_from=333.999.0.0) -- [接入企微](https://www.bilibili.com/video/BV1Tp4y1n72T/?spm_id_from=333.999.0.0) +## 🌿 第三方生态 + +- [COW 个人微信/企微机器人](https://doc.tryfastgpt.ai/docs/use-cases/external-integration/onwechat/) +- [SiliconCloud (硅基流动) —— 开源模型在线体验平台](https://cloud.siliconflow.cn/i/TR9Ym0c4) # -## 🌿 第三方生态 +## 👀 其他 -- [COW 个人微信/企微机器人](https://doc.tryfastgpt.ai/docs/use-cases/external-integration/onwechat/) +- [保姆级 FastGPT 教程](https://www.bilibili.com/video/BV1n34y1A7Bo/?spm_id_from=333.999.0.0) +- [接入飞书](https://www.bilibili.com/video/BV1Su4y1r7R3/?spm_id_from=333.999.0.0) +- [接入企微](https://www.bilibili.com/video/BV1Tp4y1n72T/?spm_id_from=333.999.0.0) # @@ -214,4 +215,4 @@ https://github.com/labring/FastGPT/assets/15308462/7d3a38df-eb0e-4388-9250-2409b 1. 允许作为后台服务直接商用，但不允许提供 SaaS 服务。 2. 未经商业授权，任何形式的商用服务均需保留相关版权信息。 3. 完整请查看 [FastGPT Open Source License](./LICENSE) -4. 联系方式：Dennis@sealos.io，[点击查看商业版定价策略](https://doc.tryfastgpt.ai/docs/commercial) +4. 联系方式：Dennis@sealos.io，[点击查看商业版定价策略](https://doc.tryfastgpt.ai/docs/shopping_cart/intro/) diff --git a/dev.md b/dev.md index 373ac626b..2a58d8d30 100644 --- a/dev.md +++ b/dev.md @@ -1,6 +1,6 @@ ## Premise -Since FastGPT is managed in the same way as monorepo, it is recommended to install 'make' first during development. +Since FastGPT is managed in the same way as monorepo, it is recommended to install ‘make’ first during development. monorepo Project Name: diff --git a/docSite/assets/imgs/dataset1.png b/docSite/assets/imgs/dataset1.png new file mode 100644 index 000000000..c521ba681 Binary files /dev/null and b/docSite/assets/imgs/dataset1.png differ diff --git a/docSite/assets/imgs/dataset2.png b/docSite/assets/imgs/dataset2.png new file mode 100644 index 000000000..5bbe11676 Binary files /dev/null and b/docSite/assets/imgs/dataset2.png differ diff --git a/docSite/assets/imgs/dingtalk-bot-1.png b/docSite/assets/imgs/dingtalk-bot-1.png new file mode 100644 index 000000000..76fad7d93 Binary files /dev/null and b/docSite/assets/imgs/dingtalk-bot-1.png differ diff --git a/docSite/assets/imgs/dingtalk-bot-2.png b/docSite/assets/imgs/dingtalk-bot-2.png new file mode 100644 index 000000000..025df4375 Binary files /dev/null and b/docSite/assets/imgs/dingtalk-bot-2.png differ diff --git a/docSite/assets/imgs/dingtalk-bot-3.png b/docSite/assets/imgs/dingtalk-bot-3.png new file mode 100644 index 000000000..ff1d64c25 Binary files /dev/null and b/docSite/assets/imgs/dingtalk-bot-3.png differ diff --git a/docSite/assets/imgs/dingtalk-bot-4.png b/docSite/assets/imgs/dingtalk-bot-4.png new file mode 100644 index 000000000..80c8cff54 Binary files /dev/null and b/docSite/assets/imgs/dingtalk-bot-4.png differ diff --git a/docSite/assets/imgs/dingtalk-bot-5.png b/docSite/assets/imgs/dingtalk-bot-5.png new file mode 100644 index 000000000..aefbd33fe Binary files /dev/null and b/docSite/assets/imgs/dingtalk-bot-5.png differ diff --git a/docSite/assets/imgs/dingtalk-bot-6.png b/docSite/assets/imgs/dingtalk-bot-6.png new file mode 100644 index 000000000..4a93f374a Binary files /dev/null and b/docSite/assets/imgs/dingtalk-bot-6.png differ diff --git a/docSite/assets/imgs/dingtalk-bot-7.png b/docSite/assets/imgs/dingtalk-bot-7.png new file mode 100644 index 000000000..0fb545c75 Binary files /dev/null and b/docSite/assets/imgs/dingtalk-bot-7.png differ diff --git a/docSite/assets/imgs/dingtalk-bot-8.png b/docSite/assets/imgs/dingtalk-bot-8.png new file mode 100644 index 000000000..ec296fabf Binary files /dev/null and b/docSite/assets/imgs/dingtalk-bot-8.png differ diff --git a/docSite/assets/imgs/image-20.png b/docSite/assets/imgs/image-20.png index fb16dbba6..eb6634477 100644 Binary files a/docSite/assets/imgs/image-20.png and b/docSite/assets/imgs/image-20.png differ diff --git a/docSite/assets/imgs/image-21.png b/docSite/assets/imgs/image-21.png index abd790c59..408e69a07 100644 Binary files a/docSite/assets/imgs/image-21.png and b/docSite/assets/imgs/image-21.png differ diff --git a/docSite/assets/imgs/image-22.png b/docSite/assets/imgs/image-22.png index fa2ebb529..4de74eb48 100644 Binary files a/docSite/assets/imgs/image-22.png and b/docSite/assets/imgs/image-22.png differ diff --git a/docSite/assets/imgs/image-23.png b/docSite/assets/imgs/image-23.png index addfa17bb..3c6dd26a4 100644 Binary files a/docSite/assets/imgs/image-23.png and b/docSite/assets/imgs/image-23.png differ diff --git a/docSite/assets/imgs/image-24.png b/docSite/assets/imgs/image-24.png index e46ea65ba..486175194 100644 Binary files a/docSite/assets/imgs/image-24.png and b/docSite/assets/imgs/image-24.png differ diff --git a/docSite/assets/imgs/image-25.png b/docSite/assets/imgs/image-25.png new file mode 100644 index 000000000..8f988bef4 Binary files /dev/null and b/docSite/assets/imgs/image-25.png differ diff --git a/docSite/assets/imgs/image-26.png b/docSite/assets/imgs/image-26.png new file mode 100644 index 000000000..f0bd89f04 Binary files /dev/null and b/docSite/assets/imgs/image-26.png differ diff --git a/docSite/assets/imgs/image-27.png b/docSite/assets/imgs/image-27.png new file mode 100644 index 000000000..ab9d4ee25 Binary files /dev/null and b/docSite/assets/imgs/image-27.png differ diff --git a/docSite/assets/imgs/image-28.png b/docSite/assets/imgs/image-28.png new file mode 100644 index 000000000..5769d0053 Binary files /dev/null and b/docSite/assets/imgs/image-28.png differ diff --git a/docSite/assets/imgs/image-29.png b/docSite/assets/imgs/image-29.png new file mode 100644 index 000000000..18dccb895 Binary files /dev/null and b/docSite/assets/imgs/image-29.png differ diff --git a/docSite/assets/imgs/image-30.png b/docSite/assets/imgs/image-30.png new file mode 100644 index 000000000..deb7a8cac Binary files /dev/null and b/docSite/assets/imgs/image-30.png differ diff --git a/docSite/assets/imgs/image-31.png b/docSite/assets/imgs/image-31.png new file mode 100644 index 000000000..a6ba81457 Binary files /dev/null and b/docSite/assets/imgs/image-31.png differ diff --git a/docSite/assets/imgs/image-32.png b/docSite/assets/imgs/image-32.png new file mode 100644 index 000000000..09b23e655 Binary files /dev/null and b/docSite/assets/imgs/image-32.png differ diff --git a/docSite/assets/imgs/image-33.png b/docSite/assets/imgs/image-33.png new file mode 100644 index 000000000..ad1bf9800 Binary files /dev/null and b/docSite/assets/imgs/image-33.png differ diff --git a/docSite/assets/imgs/image-34.png b/docSite/assets/imgs/image-34.png new file mode 100644 index 000000000..ab90957b2 Binary files /dev/null and b/docSite/assets/imgs/image-34.png differ diff --git a/docSite/assets/imgs/image-35.png b/docSite/assets/imgs/image-35.png new file mode 100644 index 000000000..503b6bbd4 Binary files /dev/null and b/docSite/assets/imgs/image-35.png differ diff --git a/docSite/assets/imgs/image-36.png b/docSite/assets/imgs/image-36.png new file mode 100644 index 000000000..c83516229 Binary files /dev/null and b/docSite/assets/imgs/image-36.png differ diff --git a/docSite/assets/imgs/image-37.png b/docSite/assets/imgs/image-37.png new file mode 100644 index 000000000..2c2ecf85b Binary files /dev/null and b/docSite/assets/imgs/image-37.png differ diff --git a/docSite/assets/imgs/image-38.png b/docSite/assets/imgs/image-38.png new file mode 100644 index 000000000..bf5229703 Binary files /dev/null and b/docSite/assets/imgs/image-38.png differ diff --git a/docSite/assets/imgs/image-39.png b/docSite/assets/imgs/image-39.png new file mode 100644 index 000000000..3c46ea9a3 Binary files /dev/null and b/docSite/assets/imgs/image-39.png differ diff --git a/docSite/assets/imgs/image-40.png b/docSite/assets/imgs/image-40.png new file mode 100644 index 000000000..dc44b9a70 Binary files /dev/null and b/docSite/assets/imgs/image-40.png differ diff --git a/docSite/assets/imgs/image-41.png b/docSite/assets/imgs/image-41.png new file mode 100644 index 000000000..5dfbb8b51 Binary files /dev/null and b/docSite/assets/imgs/image-41.png differ diff --git a/docSite/assets/imgs/image-42.png b/docSite/assets/imgs/image-42.png new file mode 100644 index 000000000..7ee45b74b Binary files /dev/null and b/docSite/assets/imgs/image-42.png differ diff --git a/docSite/assets/imgs/image-43.png b/docSite/assets/imgs/image-43.png new file mode 100644 index 000000000..b4482c177 Binary files /dev/null and b/docSite/assets/imgs/image-43.png differ diff --git a/docSite/assets/imgs/image-44.png b/docSite/assets/imgs/image-44.png new file mode 100644 index 000000000..afe92a25a Binary files /dev/null and b/docSite/assets/imgs/image-44.png differ diff --git a/docSite/assets/imgs/image-45.png b/docSite/assets/imgs/image-45.png new file mode 100644 index 000000000..182c08217 Binary files /dev/null and b/docSite/assets/imgs/image-45.png differ diff --git a/docSite/assets/imgs/image-46.png b/docSite/assets/imgs/image-46.png new file mode 100644 index 000000000..17ec0638b Binary files /dev/null and b/docSite/assets/imgs/image-46.png differ diff --git a/docSite/assets/imgs/image-47.png b/docSite/assets/imgs/image-47.png new file mode 100644 index 000000000..1cbaa64ee Binary files /dev/null and b/docSite/assets/imgs/image-47.png differ diff --git a/docSite/assets/imgs/image-48.png b/docSite/assets/imgs/image-48.png new file mode 100644 index 000000000..cd221a313 Binary files /dev/null and b/docSite/assets/imgs/image-48.png differ diff --git a/docSite/assets/imgs/image-49.png b/docSite/assets/imgs/image-49.png new file mode 100644 index 000000000..02e5eef2e Binary files /dev/null and b/docSite/assets/imgs/image-49.png differ diff --git a/docSite/assets/imgs/image-50.png b/docSite/assets/imgs/image-50.png new file mode 100644 index 000000000..39c1423e1 Binary files /dev/null and b/docSite/assets/imgs/image-50.png differ diff --git a/docSite/assets/imgs/image-51.png b/docSite/assets/imgs/image-51.png new file mode 100644 index 000000000..7e363302f Binary files /dev/null and b/docSite/assets/imgs/image-51.png differ diff --git a/docSite/assets/imgs/image-52.png b/docSite/assets/imgs/image-52.png new file mode 100644 index 000000000..ecccb3e0e Binary files /dev/null and b/docSite/assets/imgs/image-52.png differ diff --git a/docSite/assets/imgs/image-53.png b/docSite/assets/imgs/image-53.png new file mode 100644 index 000000000..80f2248d4 Binary files /dev/null and b/docSite/assets/imgs/image-53.png differ diff --git a/docSite/assets/imgs/image-54.png b/docSite/assets/imgs/image-54.png new file mode 100644 index 000000000..686805149 Binary files /dev/null and b/docSite/assets/imgs/image-54.png differ diff --git a/docSite/assets/imgs/image-55.png b/docSite/assets/imgs/image-55.png new file mode 100644 index 000000000..537436f6e Binary files /dev/null and b/docSite/assets/imgs/image-55.png differ diff --git a/docSite/assets/imgs/image-56.png b/docSite/assets/imgs/image-56.png new file mode 100644 index 000000000..2b4fa299e Binary files /dev/null and b/docSite/assets/imgs/image-56.png differ diff --git a/docSite/assets/imgs/image-57.png b/docSite/assets/imgs/image-57.png new file mode 100644 index 000000000..9069593d6 Binary files /dev/null and b/docSite/assets/imgs/image-57.png differ diff --git a/docSite/assets/imgs/image-58.png b/docSite/assets/imgs/image-58.png new file mode 100644 index 000000000..3dbc8d803 Binary files /dev/null and b/docSite/assets/imgs/image-58.png differ diff --git a/docSite/assets/imgs/image-59.png b/docSite/assets/imgs/image-59.png new file mode 100644 index 000000000..1ae30747d Binary files /dev/null and b/docSite/assets/imgs/image-59.png differ diff --git a/docSite/assets/imgs/image-60.png b/docSite/assets/imgs/image-60.png new file mode 100644 index 000000000..31a0c4761 Binary files /dev/null and b/docSite/assets/imgs/image-60.png differ diff --git a/docSite/assets/imgs/image-61.png b/docSite/assets/imgs/image-61.png new file mode 100644 index 000000000..b4a3c5c7b Binary files /dev/null and b/docSite/assets/imgs/image-61.png differ diff --git a/docSite/assets/imgs/image-62.png b/docSite/assets/imgs/image-62.png new file mode 100644 index 000000000..1a9cd5be6 Binary files /dev/null and b/docSite/assets/imgs/image-62.png differ diff --git a/docSite/assets/imgs/image-63.png b/docSite/assets/imgs/image-63.png new file mode 100644 index 000000000..14829db62 Binary files /dev/null and b/docSite/assets/imgs/image-63.png differ diff --git a/docSite/assets/imgs/image-64.png b/docSite/assets/imgs/image-64.png new file mode 100644 index 000000000..fccce7595 Binary files /dev/null and b/docSite/assets/imgs/image-64.png differ diff --git a/docSite/assets/imgs/image-65.png b/docSite/assets/imgs/image-65.png new file mode 100644 index 000000000..e40ddd56a Binary files /dev/null and b/docSite/assets/imgs/image-65.png differ diff --git a/docSite/assets/imgs/image-66.png b/docSite/assets/imgs/image-66.png new file mode 100644 index 000000000..e23677bb2 Binary files /dev/null and b/docSite/assets/imgs/image-66.png differ diff --git a/docSite/assets/imgs/image-67.png b/docSite/assets/imgs/image-67.png new file mode 100644 index 000000000..cc0b9c611 Binary files /dev/null and b/docSite/assets/imgs/image-67.png differ diff --git a/docSite/assets/imgs/image-68.png b/docSite/assets/imgs/image-68.png new file mode 100644 index 000000000..7de230757 Binary files /dev/null and b/docSite/assets/imgs/image-68.png differ diff --git a/docSite/assets/imgs/image-69.png b/docSite/assets/imgs/image-69.png new file mode 100644 index 000000000..ac82def69 Binary files /dev/null and b/docSite/assets/imgs/image-69.png differ diff --git a/docSite/assets/imgs/image-70.png b/docSite/assets/imgs/image-70.png new file mode 100644 index 000000000..b54295e2d Binary files /dev/null and b/docSite/assets/imgs/image-70.png differ diff --git a/docSite/assets/imgs/image-71.png b/docSite/assets/imgs/image-71.png new file mode 100644 index 000000000..8560c40b2 Binary files /dev/null and b/docSite/assets/imgs/image-71.png differ diff --git a/docSite/assets/imgs/image-72.png b/docSite/assets/imgs/image-72.png new file mode 100644 index 000000000..e2fef0faa Binary files /dev/null and b/docSite/assets/imgs/image-72.png differ diff --git a/docSite/assets/imgs/image-73.png b/docSite/assets/imgs/image-73.png new file mode 100644 index 000000000..065b20f38 Binary files /dev/null and b/docSite/assets/imgs/image-73.png differ diff --git a/docSite/assets/imgs/image-74.png b/docSite/assets/imgs/image-74.png new file mode 100644 index 000000000..fe2f85bed Binary files /dev/null and b/docSite/assets/imgs/image-74.png differ diff --git a/docSite/assets/imgs/image-75.png b/docSite/assets/imgs/image-75.png new file mode 100644 index 000000000..a989e6c74 Binary files /dev/null and b/docSite/assets/imgs/image-75.png differ diff --git a/docSite/assets/imgs/image-76.png b/docSite/assets/imgs/image-76.png new file mode 100644 index 000000000..eb1ab62a6 Binary files /dev/null and b/docSite/assets/imgs/image-76.png differ diff --git a/docSite/assets/imgs/image-77.png b/docSite/assets/imgs/image-77.png new file mode 100644 index 000000000..9d21aee47 Binary files /dev/null and b/docSite/assets/imgs/image-77.png differ diff --git a/docSite/assets/imgs/image-78.png b/docSite/assets/imgs/image-78.png new file mode 100644 index 000000000..25082fe9a Binary files /dev/null and b/docSite/assets/imgs/image-78.png differ diff --git a/docSite/assets/imgs/image-79.png b/docSite/assets/imgs/image-79.png new file mode 100644 index 000000000..207b675c2 Binary files /dev/null and b/docSite/assets/imgs/image-79.png differ diff --git a/docSite/assets/imgs/image-80.png b/docSite/assets/imgs/image-80.png new file mode 100644 index 000000000..9f5d500d2 Binary files /dev/null and b/docSite/assets/imgs/image-80.png differ diff --git a/docSite/assets/imgs/image-81.png b/docSite/assets/imgs/image-81.png new file mode 100644 index 000000000..bf15d98f6 Binary files /dev/null and b/docSite/assets/imgs/image-81.png differ diff --git a/docSite/assets/imgs/image-82.png b/docSite/assets/imgs/image-82.png new file mode 100644 index 000000000..02ac7fd88 Binary files /dev/null and b/docSite/assets/imgs/image-82.png differ diff --git a/docSite/assets/imgs/image-83.png b/docSite/assets/imgs/image-83.png new file mode 100644 index 000000000..b6dbe4e64 Binary files /dev/null and b/docSite/assets/imgs/image-83.png differ diff --git a/docSite/assets/imgs/image-84.png b/docSite/assets/imgs/image-84.png new file mode 100644 index 000000000..a60a4c337 Binary files /dev/null and b/docSite/assets/imgs/image-84.png differ diff --git a/docSite/assets/imgs/image-85.png b/docSite/assets/imgs/image-85.png new file mode 100644 index 000000000..c2af997ec Binary files /dev/null and b/docSite/assets/imgs/image-85.png differ diff --git a/docSite/assets/imgs/image-86.png b/docSite/assets/imgs/image-86.png new file mode 100644 index 000000000..d35c930e1 Binary files /dev/null and b/docSite/assets/imgs/image-86.png differ diff --git a/docSite/assets/imgs/image-87.png b/docSite/assets/imgs/image-87.png new file mode 100644 index 000000000..74b545f01 Binary files /dev/null and b/docSite/assets/imgs/image-87.png differ diff --git a/docSite/assets/imgs/searxng_plugin_guide1.png b/docSite/assets/imgs/searxng_plugin_guide1.png new file mode 100644 index 000000000..4524363ed Binary files /dev/null and b/docSite/assets/imgs/searxng_plugin_guide1.png differ diff --git a/docSite/assets/imgs/template_submission2.png b/docSite/assets/imgs/template_submission2.png index 338d47b1a..14ca7b91f 100644 Binary files a/docSite/assets/imgs/template_submission2.png and b/docSite/assets/imgs/template_submission2.png differ diff --git a/docSite/content/zh-cn/docs/agreement/open-source.md b/docSite/content/zh-cn/docs/agreement/open-source.md index 8bd372353..eaa493ec6 100644 --- a/docSite/content/zh-cn/docs/agreement/open-source.md +++ b/docSite/content/zh-cn/docs/agreement/open-source.md @@ -7,7 +7,7 @@ toc: true weight: 1210 --- -FastGPT 项目在 Apache License 2.0 许可下开源，同时包含以下附加条件： +FastGPT 项目在 Apache License 2.0 许可下开源，但包含以下附加条件： + FastGPT 允许被用于商业化，例如作为其他应用的“后端即服务”使用，或者作为应用开发平台提供给企业。然而，当满足以下条件时，必须联系作者获得商业许可： diff --git a/docSite/content/zh-cn/docs/development/configuration.md b/docSite/content/zh-cn/docs/development/configuration.md index 06e4e2b79..530806c09 100644 --- a/docSite/content/zh-cn/docs/development/configuration.md +++ b/docSite/content/zh-cn/docs/development/configuration.md @@ -4,14 +4,14 @@ description: 'FastGPT 配置参数介绍' icon: 'settings' draft: false toc: true -weight: 708 +weight: 707 --- 由于环境变量不利于配置复杂的内容，新版 FastGPT 采用了 ConfigMap 的形式挂载配置文件，你可以在 `projects/app/data/config.json` 看到默认的配置文件。可以参考 [docker-compose 快速部署](/docs/development/docker/) 来挂载配置文件。 -**开发环境下**，你需要将示例配置文件 `config.json` 复制成 `config.local.json` 文件才会生效。 +**开发环境下**，你需要将示例配置文件 `config.json` 复制成 `config.local.json` 文件才会生效。 -这个配置文件中包含了系统参数和各个模型配置： +下面配置文件示例中包含了系统参数和各个模型配置： ## 4.6.8+ 版本新配置文件示例 @@ -21,16 +21,16 @@ weight: 708 "lafEnv": "https://laf.dev" // laf环境。 https://laf.run （杭州阿里云） ,或者私有化的laf环境。如果使用 Laf openapi 功能，需要最新版的 laf 。 }, "systemEnv": { - "vectorMaxProcess": 15, - "qaMaxProcess": 15, + "vectorMaxProcess": 15, // 向量处理线程数量 + "qaMaxProcess": 15, // 问答拆分线程数量 "tokenWorkers": 50, // Token 计算线程保持数，会持续占用内存，不能设置太大。 "pgHNSWEfSearch": 100 // 向量搜索参数。越大，搜索越精确，但是速度越慢。设置为100，有99%+精度。 }, "llmModels": [ { + "provider": "OpenAI", // 模型提供商，主要用于分类展示，目前已经内置提供商包括：https://github.com/labring/FastGPT/blob/main/packages/global/core/ai/provider.ts, 可 pr 提供新的提供商，或直接填写 Other "model": "gpt-4o-mini", // 模型名(对应OneAPI中渠道的模型名) "name": "gpt-4o-mini", // 模型别名 - "avatar": "/imgs/model/openai.svg", // 模型的logo "maxContext": 125000, // 最大上下文 "maxResponse": 16000, // 最大回复 "quoteMaxToken": 120000, // 最大引用内容 @@ -38,7 +38,7 @@ weight: 708 "charsPointsPrice": 0, // n积分/1k token（商业版） "censor": false, // 是否开启敏感校验（商业版） "vision": true, // 是否支持图片输入 - "datasetProcess": true, // 是否设置为知识库处理模型（QA），务必保证至少有一个为true，否则知识库会报错 + "datasetProcess": true, // 是否设置为文本理解模型（QA），务必保证至少有一个为true，否则知识库会报错 "usedInClassify": true, // 是否用于问题分类（务必保证至少有一个为true） "usedInExtractFields": true, // 是否用于内容提取（务必保证至少有一个为true） "usedInToolCall": true, // 是否用于工具调用（务必保证至少有一个为true） @@ -48,12 +48,13 @@ weight: 708 "customCQPrompt": "", // 自定义文本分类提示词（不支持工具和函数调用的模型 "customExtractPrompt": "", // 自定义内容提取提示词 "defaultSystemChatPrompt": "", // 对话默认携带的系统提示词 - "defaultConfig": {} // 请求API时，挟带一些默认配置（比如 GLM4 的 top_p） + "defaultConfig": {}, // 请求API时，挟带一些默认配置（比如 GLM4 的 top_p） + "fieldMap": {} // 字段映射（o1 模型需要把 max_tokens 映射为 max_completion_tokens） }, { + "provider": "OpenAI", "model": "gpt-4o", "name": "gpt-4o", - "avatar": "/imgs/model/openai.svg", "maxContext": 125000, "maxResponse": 4000, "quoteMaxToken": 120000, @@ -71,14 +72,15 @@ weight: 708 "customCQPrompt": "", "customExtractPrompt": "", "defaultSystemChatPrompt": "", - "defaultConfig": {} + "defaultConfig": {}, + "fieldMap": {} }, { + "provider": "OpenAI", "model": "o1-mini", "name": "o1-mini", - "avatar": "/imgs/model/openai.svg", "maxContext": 125000, - "maxResponse": 4000, + "maxResponse": 65000, "quoteMaxToken": 120000, "maxTemperature": 1.2, "charsPointsPrice": 0, @@ -95,15 +97,17 @@ weight: 708 "customExtractPrompt": "", "defaultSystemChatPrompt": "", "defaultConfig": { - "temperature": 1 + "temperature": 1, + "max_tokens": null, + "stream": false } }, { + "provider": "OpenAI", "model": "o1-preview", "name": "o1-preview", - "avatar": "/imgs/model/openai.svg", "maxContext": 125000, - "maxResponse": 4000, + "maxResponse": 32000, "quoteMaxToken": 120000, "maxTemperature": 1.2, "charsPointsPrice": 0, @@ -120,27 +124,26 @@ weight: 708 "customExtractPrompt": "", "defaultSystemChatPrompt": "", "defaultConfig": { - "temperature": 1 + "temperature": 1, + "max_tokens": null, + "stream": false } } ], "vectorModels": [ { - "model": "text-embedding-ada-002", // 模型名（与OneAPI对应） - "name": "Embedding-2", // 模型展示名 - "avatar": "/imgs/model/openai.svg", // logo - "charsPointsPrice": 0, // n积分/1k token - "defaultToken": 700, // 默认文本分割时候的 token - "maxToken": 3000, // 最大 token - "weight": 100, // 优先训练权重 - "defaultConfig": {}, // 自定义额外参数。例如，如果希望使用 embedding3-large 的话，可以传入 dimensions:1024，来返回1024维度的向量。（目前必须小于1536维度） - "dbConfig": {}, // 存储时的额外参数（非对称向量模型时候需要用到） - "queryConfig": {} // 参训时的额外参数 + "provider": "OpenAI", + "model": "text-embedding-3-small", + "name": "text-embedding-3-small", + "charsPointsPrice": 0, + "defaultToken": 512, + "maxToken": 3000, + "weight": 100 }, { + "provider": "OpenAI", "model": "text-embedding-3-large", "name": "text-embedding-3-large", - "avatar": "/imgs/model/openai.svg", "charsPointsPrice": 0, "defaultToken": 512, "maxToken": 3000, @@ -150,18 +153,22 @@ weight: 708 } }, { - "model": "text-embedding-3-small", - "name": "text-embedding-3-small", - "avatar": "/imgs/model/openai.svg", - "charsPointsPrice": 0, - "defaultToken": 512, - "maxToken": 3000, - "weight": 100 + "provider": "OpenAI", + "model": "text-embedding-ada-002", // 模型名（与OneAPI对应） + "name": "Embedding-2", // 模型展示名 + "charsPointsPrice": 0, // n积分/1k token + "defaultToken": 700, // 默认文本分割时候的 token + "maxToken": 3000, // 最大 token + "weight": 100, // 优先训练权重 + "defaultConfig": {}, // 自定义额外参数。例如，如果希望使用 embedding3-large 的话，可以传入 dimensions:1024，来返回1024维度的向量。（目前必须小于1536维度） + "dbConfig": {}, // 存储时的额外参数（非对称向量模型时候需要用到） + "queryConfig": {} // 参训时的额外参数 } ], "reRankModels": [], "audioSpeechModels": [ { + "provider": "OpenAI", "model": "tts-1", "name": "OpenAI TTS1", "charsPointsPrice": 0, @@ -176,6 +183,7 @@ weight: 708 } ], "whisperModel": { + "provider": "OpenAI", "model": "whisper-1", "name": "Whisper1", "charsPointsPrice": 0 @@ -183,29 +191,66 @@ weight: 708 } ``` -## 关于模型 logo +## 内置的模型提供商ID -统一放置在项目的`public/imgs/model/xxx`目录中，目前内置了以下几种，如果有需要，可以PR增加。默认logo为 Hugging face 的 logo~ +为了方便模型分类展示，FastGPT 内置了部分模型提供商的名字和 Logo。如果你期望补充提供商，可[提交 Issue](https://github.com/labring/FastGPT/issues)，并提供几个信息： -- /imgs/model/baichuan.svg - 百川智能 -- /imgs/model/chatglm.svg - 智谱清言 -- /imgs/model/claude.svg - claude -- /imgs/model/deepseek.svg - deepseek -- /imgs/model/doubao.svg - 火山豆包 -- /imgs/model/ernie.svg - 文心一言 -- /imgs/model/gemini.svg - gemini -- /imgs/model/huggingface.svg - Hugging face【默认logo】 -- /imgs/model/minimax.svg - minimax -- /imgs/model/moonshot.svg - 月之暗面 -- /imgs/model/openai.svg - OpenAI GPT -- /imgs/model/qwen.svg - 通义千问 -- /imgs/model/sparkDesk.svg - 讯飞星火 -- /imgs/model/yi.svg - 零一万物 -- +1. 厂商官网地址 +2. 厂商 SVG logo，建议是正方形图片。 -## 特殊模型 +目前已支持的提供商, 复制 "-" 之前的字符串，作为 provider 的值。 -### ReRank 接入(私有部署) +- OpenAI +- Claude +- Gemini +- Meta +- MistralAI +- AliCloud - 阿里云 +- Qwen - 通义千问 +- Doubao - 豆包 +- ChatGLM - 智谱 +- DeepSeek - 深度求索 +- Moonshot - 月之暗面 +- MiniMax +- SparkDesk - 讯飞星火 +- Hunyuan - 腾讯混元 +- Baichuan - 百川 +- Yi - 零一万物 +- Ernie - 文心一言 +- StepFun - 阶跃星辰 +- Ollama +- BAAI - 智源研究院 +- FishAudio +- Other - 其他 + + +## ReRank 模型接入 + +由于 OneAPI 不支持 Rerank 模型，所以需要单独配置接入，这里 + + +### 使用硅基流动的在线模型 + +有免费的 `bge-reranker-v2-m3` 模型可以使用。 + +1. [点击注册硅基流动账号](https://cloud.siliconflow.cn/i/TR9Ym0c4) +2. 进入控制台，获取 API key: https://cloud.siliconflow.cn/account/ak +3. 修改 FastGPT 配置文件 + +```json +{ + "reRankModels": [ + { + "model": "BAAI/bge-reranker-v2-m3", // 这里的model需要对应 siliconflow 的模型名 + "name": "BAAI/bge-reranker-v2-m3", + "requestUrl": "https://api.siliconflow.cn/v1/rerank", + "requestAuth": "siliconflow 上申请的 key" + } + ] +} +``` + +### 私有部署模型 请使用 4.6.6-alpha 以上版本，配置文件中的 `reRankModels` 为重排模型，虽然是数组，不过目前仅有第1个生效。 @@ -226,44 +271,3 @@ weight: 708 ] } ``` - -### ReRank 接入（硅基流动） - -有免费的 `bge-reranker-v2-m3` 模型可以使用。 - -1. 注册硅基流动账号: https://siliconflow.cn/ -2. 进入控制台，获取 API key: https://cloud.siliconflow.cn/account/ak -3. 修改 FastGPT 配置文件 - -```json -{ - "reRankModels": [ - { - "model": "BAAI/bge-reranker-v2-m3", // 这里的model需要对应 siliconflow 的模型名 - "name": "BAAI/bge-reranker-v2-m3", - "requestUrl": "https://api.siliconflow.cn/v1/rerank", - "requestAuth": "siliconflow 上申请的 key" - } - ] -} -``` - -### ReRank 接入（Cohere） - -这个重排模型对中文不是很好，不如 bge 的好用。 - -1. 申请 Cohere 官方 Key: https://dashboard.cohere.com/api-keys -2. 修改 FastGPT 配置文件 - -```json -{ - "reRankModels": [ - { - "model": "rerank-multilingual-v2.0", // 这里的model需要对应 cohere 的模型名 - "name": "rerank-multilingual-v2.0", - "requestUrl": "https://api.cohere.ai/v1/rerank", - "requestAuth": "Coherer上申请的key" - } - ] -} -``` diff --git a/docSite/content/zh-cn/docs/development/custom-models/chatglm2-m3e.md b/docSite/content/zh-cn/docs/development/custom-models/chatglm2-m3e.md index 775562d84..add2a09f6 100644 --- a/docSite/content/zh-cn/docs/development/custom-models/chatglm2-m3e.md +++ b/docSite/content/zh-cn/docs/development/custom-models/chatglm2-m3e.md @@ -23,7 +23,7 @@ FastGPT 默认使用了 OpenAI 的 LLM 模型和向量模型，如果想要私 也可以通过环境变量引入：sk-key。有关docker环境变量引入的方法请自寻教程，此处不再赘述。 ``` -## 接入 [One API](/docs/development/one-api/) +## 接入 [One API](/docs/development/modelconfig/one-api/) 为 chatglm2 和 m3e-large 各添加一个渠道，参数如下： diff --git a/docSite/content/zh-cn/docs/development/custom-models/xinference.md b/docSite/content/zh-cn/docs/development/custom-models/xinference.md index 6cd1cdee4..a76fbdf31 100644 --- a/docSite/content/zh-cn/docs/development/custom-models/xinference.md +++ b/docSite/content/zh-cn/docs/development/custom-models/xinference.md @@ -102,7 +102,7 @@ xinference launch -n qwen-chat -s 14 -f pytorch ## 将本地模型接入 One API -One API 的部署和接入请参考[这里](/docs/development/one-api/)。 +One API 的部署和接入请参考[这里](/docs/development/modelconfig/one-api/)。 为 qwen1.5-chat 添加一个渠道，这里的 Base URL 需要填 Xinference 服务的端点，并且注册 qwen-chat (模型的 UID) 。 diff --git a/docSite/content/zh-cn/docs/development/docker.md b/docSite/content/zh-cn/docs/development/docker.md index 9996e3c37..c007f345e 100644 --- a/docSite/content/zh-cn/docs/development/docker.md +++ b/docSite/content/zh-cn/docs/development/docker.md @@ -192,7 +192,7 @@ docker restart oneapi 可以通过`ip:3001`访问OneAPI，默认账号为`root`密码为`123456`。 -在OneApi中添加合适的AI模型渠道。[点击查看相关教程](/docs/development/one-api/) +在OneApi中添加合适的AI模型渠道。[点击查看相关教程](/docs/development/modelconfig/one-api/) ### 5. 访问 FastGPT diff --git a/docSite/content/zh-cn/docs/development/faq.md b/docSite/content/zh-cn/docs/development/faq.md index 8094353f5..91e606503 100644 --- a/docSite/content/zh-cn/docs/development/faq.md +++ b/docSite/content/zh-cn/docs/development/faq.md @@ -1,5 +1,5 @@ --- -weight: 749 +weight: 740 title: "私有部署常见问题" description: "FastGPT 私有部署常见问题" icon: upgrade @@ -19,6 +19,10 @@ images: [] ## 二、通用问题 +### 本地部署的限制 + +具体内容参考https://fael3z0zfze.feishu.cn/wiki/OFpAw8XzAi36Guk8dfucrCKUnjg。 + ### 能否纯本地运行 可以。需要准备好向量模型和LLM模型。 @@ -41,31 +45,6 @@ images: [] 1. 问题补全需要经过一轮AI生成。 2. 会进行3~5轮的查询，如果数据库性能不足，会有明显影响。 -### 对话接口报错或返回为空(core.chat.Chat API is error or undefined) - -1. 检查 AI 的 key 问题：通过 curl 请求看是否正常。务必用 stream=true 模式。并且 maxToken 等相关参数尽量一致。 -2. 如果是国内模型，可能是命中风控了。 -3. 查看模型请求日志，检查出入参数是否异常。 - -```sh -# curl 例子。 -curl --location --request POST 'https://xxx.cn/v1/chat/completions' \ ---header 'Authorization: Bearer sk-xxxx' \ ---header 'Content-Type: application/json' \ ---data-raw '{ - "model": "gpt-3.5-turbo", - "stream": true, - "temperature": 1, - "max_tokens": 3000, - "messages": [ - { - "role": "user", - "content": "你是谁" - } - ] -}' -``` - ### 页面中可以正常回复，API 报错 页面中是用 stream=true 模式，所以API也需要设置 stream=true 来进行测试。部分模型接口（国产居多）非 Stream 的兼容有点垃圾。 @@ -111,6 +90,13 @@ FastGPT 模型配置文件中的 model 必须与 OneAPI 渠道中的模型对应 如果OneAPI中，没有配置对应的模型，`config.json`中也不要配置，否则容易报错。 +### 点击模型测试失败 + +OneAPI 只会测试渠道的第一个模型，并且只会测试对话模型，向量模型无法自动测试，需要手动发起请求进行测试。[查看测试模型命令示例](/docs/development/faq/#如何检查模型问题) +### get request url failed: Post "https://xxx dial tcp: xxxx + +OneAPI 与模型网络不通，需要检查网络配置。 + ### Incorrect API key provided: sk-xxxx.You can find your api Key at xxx OneAPI 的 API Key 配置错误，需要修改`OPENAI_API_KEY`环境变量，并重启容器（先 docker-compose down 然后再 docker-compose up -d 运行一次）。 @@ -126,6 +112,112 @@ OneAPI 的 API Key 配置错误，需要修改`OPENAI_API_KEY`环境变量，并 ## 四、常见模型问题 +### 如何检查模型问题 + +1. 私有部署模型，先确认部署的模型是否正常。 +2. 通过 CURL 请求，直接测试上游模型是否正常运行（云端模型或私有模型均进行测试） +3. 通过 CURL 请求，请求 OneAPI 去测试模型是否正常。 +4. 在 FastGPT 中使用该模型进行测试。 + +下面是几个测试 CURL 示例： + +{{< tabs tabTotal="5" >}} +{{< tab tabName="LLM模型" >}} +{{< markdownify >}} + +```bash +curl https://api.openai.com/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $OPENAI_API_KEY" \ + -d '{ + "model": "gpt-4o", + "messages": [ + { + "role": "system", + "content": "You are a helpful assistant." + }, + { + "role": "user", + "content": "Hello!" + } + ] + }' + +``` + +{{< /markdownify >}} +{{< /tab >}} + +{{< tab tabName="Embedding模型" >}} +{{< markdownify >}} + +```bash +curl https://api.openai.com/v1/embeddings \ + -H "Authorization: Bearer $OPENAI_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "input": "The food was delicious and the waiter...", + "model": "text-embedding-ada-002", + "encoding_format": "float" + }' +``` + +{{< /markdownify >}} +{{< /tab >}} + +{{< tab tabName="Rerank 模型" >}} +{{< markdownify >}} + +```bash +curl --location --request POST 'https://xxxx.com/api/v1/rerank' \ +--header 'Authorization: Bearer {{ACCESS_TOKEN}}' \ +--header 'Content-Type: application/json' \ +--data-raw '{ + "model": "bge-rerank-m3", + "query": "导演是谁", + "documents": [ + "你是谁？\n我是电影《铃芽之旅》助手" + ] +}' +``` + +{{< /markdownify >}} +{{< /tab >}} + +{{< tab tabName="TTS 模型" >}} +{{< markdownify >}} + +```bash +curl https://api.openai.com/v1/audio/speech \ + -H "Authorization: Bearer $OPENAI_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "tts-1", + "input": "The quick brown fox jumped over the lazy dog.", + "voice": "alloy" + }' \ + --output speech.mp3 +``` + +{{< /markdownify >}} +{{< /tab >}} + +{{< tab tabName="Whisper 模型" >}} +{{< markdownify >}} + +```bash +curl https://api.openai.com/v1/audio/transcriptions \ + -H "Authorization: Bearer $OPENAI_API_KEY" \ + -H "Content-Type: multipart/form-data" \ + -F file="@/path/to/file/audio.mp3" \ + -F model="whisper-1" +``` + +{{< /markdownify >}} +{{< /tab >}} + +{{< /tabs >}} + ### 报错 - 模型响应为空/模型报错 该错误是由于 stream 模式下，oneapi 直接结束了流请求，并且未返回任何内容导致。 @@ -165,7 +257,7 @@ curl --location --request POST 'https://api.openai.com/v1/chat/completions' \ 需要模型提供商和 oneapi 同时支持工具调用才可使用，测试方法如下： -1. 通过 `curl` 向 `oneapi` 发起第一轮 stream 模式的 tool 测试。 +##### 1. 通过 `curl` 向 `oneapi` 发起第一轮 stream 模式的 tool 测试。 ```bash curl --location --request POST 'https://oneapi.xxx/v1/chat/completions' \ @@ -200,7 +292,7 @@ curl --location --request POST 'https://oneapi.xxx/v1/chat/completions' \ }' ``` -2. 检查响应参数 +##### 2. 检查响应参数 如果能正常调用工具，会返回对应 `tool_calls` 参数。 @@ -238,7 +330,7 @@ curl --location --request POST 'https://oneapi.xxx/v1/chat/completions' \ } ``` -3. 通过 `curl` 向 `oneapi` 发起第二轮 stream 模式的 tool 测试。 +##### 3. 通过 `curl` 向 `oneapi` 发起第二轮 stream 模式的 tool 测试。 第二轮请求是把工具结果发送给模型。发起后会得到模型回答的结果。 diff --git a/docSite/content/zh-cn/docs/development/intro.md b/docSite/content/zh-cn/docs/development/intro.md index aed6864b9..078b2fd10 100644 --- a/docSite/content/zh-cn/docs/development/intro.md +++ b/docSite/content/zh-cn/docs/development/intro.md @@ -148,7 +148,7 @@ FastGPT 在`pnpm i`后会执行`postinstall`脚本，用于自动生成`ChakraUI ## 加入社区 -遇到困难了吗？有任何问题吗? 加入微信群与开发者和用户保持沟通。 +遇到困难了吗？有任何问题吗? 加入飞书群与开发者和用户保持沟通。 diff --git a/docSite/content/zh-cn/docs/development/modelConfig/_index.md b/docSite/content/zh-cn/docs/development/modelConfig/_index.md new file mode 100644 index 000000000..ba8354c28 --- /dev/null +++ b/docSite/content/zh-cn/docs/development/modelConfig/_index.md @@ -0,0 +1,8 @@ +--- +weight: 745 +title: '模型配置方案' +description: '本模型配置方案' +icon: 'code_blocks' +draft: false +images: [] +--- \ No newline at end of file diff --git a/docSite/content/zh-cn/docs/development/modelConfig/one-api.md b/docSite/content/zh-cn/docs/development/modelConfig/one-api.md new file mode 100644 index 000000000..5a0f74284 --- /dev/null +++ b/docSite/content/zh-cn/docs/development/modelConfig/one-api.md @@ -0,0 +1,189 @@ +--- +title: '通过 OneAPI 接入模型' +description: '通过 OneAPI 接入模型' +icon: 'api' +draft: false +toc: true +weight: 745 +--- + +FastGPT 目前采用模型分离的部署方案，FastGPT 中只兼容 OpenAI 的模型规范（OpenAI 不存在的模型采用一个较为通用的规范），并通过 [One API](https://github.com/songquanpeng/one-api) 来实现对不同模型接口的统一。 + +[One API](https://github.com/songquanpeng/one-api) 是一个 OpenAI 接口管理 & 分发系统，可以通过标准的 OpenAI API 格式访问所有的大模型，开箱即用。 + + +## FastGPT 与 One API 关系 + +可以把 One API 当做一个网关，FastGPT 与 One API 关系： + +![](/imgs/sealos-fastgpt.webp) + +## 部署 + +### Docker 版本 + +`docker-compose.yml` 文件已加入了 OneAPI 配置，可直接使用。默认暴露在 3001 端口。 + +### Sealos 版本 + +* 北京区: [点击部署 OneAPI](https://hzh.sealos.run/?openapp=system-template%3FtemplateName%3Done-api) +* 新加坡区(可用 GPT) [点击部署 OneAPI](https://cloud.sealos.io/?openapp=system-template%3FtemplateName%3Done-api) + +![alt text](/imgs/image-59.png) + +部署完后，可以打开 OneAPI 访问链接，进行下一步操作。 + +## OneAPI 基础教程 + +### 概念 + +1. 渠道： + 1. OneApi 中一个渠道对应一个 `Api Key`，这个 `Api Key` 可以是GPT、微软、ChatGLM、文心一言的。一个`Api Key`通常可以调用同一个厂商的多个模型。 + 2. One API 会根据请求传入的`模型`来决定使用哪一个`渠道`，如果一个模型对应了多个`渠道`，则会随机调用。 +2. 令牌：访问 One API 所需的凭证，只需要这`1`个凭证即可访问`One API`上配置的模型。因此`FastGPT`中，只需要配置`One API`的`baseurl`和`令牌`即可。令牌不要设置任何的模型范围权限，否则容易报错。 + +![alt text](/imgs/image-60.png) + +### 大致工作流程 + +1. 客户端请求 One API +2. 根据请求中的 `model` 参数，匹配对应的渠道（根据渠道里的模型进行匹配，必须完全一致）。如果匹配到多个渠道，则随机选择一个（同优先级）。 +3. One API 向真正的地址发出请求。 +4. One API 将结果返回给客户端。 + +### 1. 登录 One API + +![step5](/imgs/oneapi-step5.png) + +### 2. 创建渠道 + +在 One API 中添加对应渠道，直接点击 【添加基础模型】，不要遗漏了向量模型（Embedding） + +![step6](/imgs/oneapi-step6.png) + +### 3. 创建令牌 + +| | | +| --- | --- | +| ![step7](/imgs/oneapi-step7.png) | ![alt text](/imgs/image-61.png) | + +### 4. 修改账号余额 + +One API 默认 root 用户只有 200刀，可以自行修改编辑。 + +![alt text](/imgs/image-62.png) + +### 5. 修改 FastGPT 的环境变量 + +有了 One API 令牌后，FastGPT 可以通过修改 `baseurl` 和 `key` 去请求到 One API，再由 One API 去请求不同的模型。修改下面两个环境变量： + +```bash +# 务必写上 v1。如果在同一个网络内，可改成内网地址。 +OPENAI_BASE_URL=https://xxxx.cloud.sealos.io/v1 +# 下面的 key 是由 One API 提供的令牌 +CHAT_API_KEY=sk-xxxxxx +``` + +## 接入其他模型 + +**以添加文心一言为例:** + +### 1. OneAPI 新增模型渠道 + +类型选择百度文心千帆。 + +![](/imgs/oneapi-demo1.png) + +### 2. 修改 FastGPT 配置文件 + +可以在 `/projects/app/src/data/config.json` 里找到配置文件（本地开发需要复制成 config.local.json）,按下面内容修改配置文件，最新/更具体的配置说明，可查看[FastGPT 配置文件说明](/docs/development/configuration)。 + +配置模型关键点在于`model` 需要与 OneAPI 渠道中的模型一致。 + +```json +{ + "llmModels": [ // 语言模型配置 + { + "model": "ERNIE-Bot", // 这里的模型需要对应 One API 的模型 + "name": "文心一言", // 对外展示的名称 + "avatar": "/imgs/model/openai.svg", // 模型的logo + "maxContext": 16000, // 最大上下文 + "maxResponse": 4000, // 最大回复 + "quoteMaxToken": 13000, // 最大引用内容 + "maxTemperature": 1.2, // 最大温度 + "charsPointsPrice": 0, + "censor": false, + "vision": false, // 是否支持图片输入 + "datasetProcess": true, // 是否设置为知识库处理模型 + "usedInClassify": true, // 是否用于问题分类 + "usedInExtractFields": true, // 是否用于字段提取 + "usedInToolCall": true, // 是否用于工具调用 + "usedInQueryExtension": true, // 是否用于问题优化 + "toolChoice": true, // 是否支持工具选择 + "functionCall": false, // 是否支持函数调用 + "customCQPrompt": "", // 自定义文本分类提示词（不支持工具和函数调用的模型 + "customExtractPrompt": "", // 自定义内容提取提示词 + "defaultSystemChatPrompt": "", // 对话默认携带的系统提示词 + "defaultConfig":{} // 请求API时，挟带一些默认配置（比如 GLM4 的 top_p） + } + ], + "vectorModels": [ // 向量模型配置 + { + "model": "text-embedding-ada-002", + "name": "Embedding-2", + "avatar": "/imgs/model/openai.svg", + "charsPointsPrice": 0, + "defaultToken": 700, + "maxToken": 3000, + "weight": 100 + }, + ] +} +``` + +### 3. 重启 FastGPT + +**Docker 版本** + +```bash +docker-compose down +docker-compose up -d +``` + +**Sealos 版本** + +直接找到 FastGPT 服务，点击重启即可。 + + +## 其他服务商接入参考 + +这章介绍一些提供商接入 OneAPI 的教程，配置后不要忘记修改 FastGPT 配置文件。 + +### 阿里通义千问 + +千问目前已经兼容 GPT 格式，可以直接选择 OpenAI 类型来接入即可。如下图，选择类型为`OpenAI`，代理填写阿里云的代理地址。 + +目前可以直接使用阿里云的语言模型和 `text-embedding-v3` 向量模型（实测已经归一化，可直接使用） + +![alt text](/imgs/image-63.png) + +### 硅基流动 —— 开源模型大合集 + +[硅基流动](https://cloud.siliconflow.cn/i/TR9Ym0c4) 是一个专门提供开源模型调用平台，并拥有自己的加速引擎。模型覆盖面广，非常适合低成本来测试开源模型。接入教程： + +1. [点击注册硅基流动账号](https://cloud.siliconflow.cn/i/TR9Ym0c4) +2. 进入控制台，获取 API key: https://cloud.siliconflow.cn/account/ak +3. 新增 OneAPI 渠道，选择`OpenAI`类型，代理填写：`https://api.siliconflow.cn`，密钥是第二步创建的密钥。 + +![alt text](/imgs/image-64.png) + +由于 OneAPI 未内置 硅基流动 的模型名，可以通过自定义模型名称来填入，下面是获取模型名称的教程： + +1. 打开[硅基流动模型列表](https://siliconflow.cn/zh-cn/models) +2. 单击模型后，会打开模型详情。 +3. 复制模型名到 OneAPI 中。 + +| | | | +| --- | --- | --- | +| ![alt text](/imgs/image-65.png) | ![alt text](/imgs/image-66.png)| ![alt text](/imgs/image-67.png) | + diff --git a/docSite/content/zh-cn/docs/development/modelConfig/siliconCloud.md b/docSite/content/zh-cn/docs/development/modelConfig/siliconCloud.md new file mode 100644 index 000000000..df4cd67ed --- /dev/null +++ b/docSite/content/zh-cn/docs/development/modelConfig/siliconCloud.md @@ -0,0 +1,220 @@ +--- +title: '通过 SiliconCloud 体验开源模型' +description: '通过 SiliconCloud 体验开源模型' +icon: 'api' +draft: false +toc: true +weight: 746 +--- + +[SiliconCloud(硅基流动)](https://cloud.siliconflow.cn/i/TR9Ym0c4) 是一个以提供开源模型调用为主的平台，并拥有自己的加速引擎。帮助用户低成本、快速的进行开源模型的测试和使用。实际体验下来，他们家模型的速度和稳定性都非常不错，并且种类丰富，覆盖语言、向量、重排、TTS、STT、绘图、视频生成模型，可以满足 FastGPT 中所有模型需求。 + +如果你想部分模型使用 SiliconCloud 的模型，可额外参考[OneAPI接入硅基流动](/docs/development/modelconfig/one-api/#硅基流动--开源模型大合集)。 + +本文会介绍完全使用 SiliconCloud 模型来部署 FastGPT 的方案。 + + +## 1. 注册 SiliconCloud 账号 + +1. [点击注册硅基流动账号](https://cloud.siliconflow.cn/i/TR9Ym0c4) +2. 进入控制台，获取 API key: https://cloud.siliconflow.cn/account/ak + +## 2. 修改 FastGPT 环境变量 + +```bash +OPENAI_BASE_URL=https://api.siliconflow.cn/v1 +# 填写 SiliconCloud 控制台提供的 Api Key +CHAT_API_KEY=sk-xxxxxx +``` + +## 3. 修改 FastGPT 配置文件 + +我们选取 SiliconCloud 中的模型作为 FastGPT 配置。这里配置了 `Qwen2.5 72b` 的纯语言和视觉模型；选择 `bge-m3` 作为向量模型；选择 `bge-reranker-v2-m3` 作为重排模型。选择 `fish-speech-1.5` 作为语音模型；选择 `SenseVoiceSmall` 作为语音输入模型。 + +注意：ReRank 模型仍需配置一次 Api Key + +```json +{ + "llmModels": [ + { + "provider": "Other", // 模型提供商，主要用于分类展示，目前已经内置提供商包括：https://github.com/labring/FastGPT/blob/main/packages/global/core/ai/provider.ts, 可 pr 提供新的提供商，或直接填写 Other + "model": "Qwen/Qwen2.5-72B-Instruct", // 模型名(对应OneAPI中渠道的模型名) + "name": "Qwen2.5-72B-Instruct", // 模型别名 + "maxContext": 32000, // 最大上下文 + "maxResponse": 4000, // 最大回复 + "quoteMaxToken": 30000, // 最大引用内容 + "maxTemperature": 1, // 最大温度 + "charsPointsPrice": 0, // n积分/1k token（商业版） + "censor": false, // 是否开启敏感校验（商业版） + "vision": false, // 是否支持图片输入 + "datasetProcess": true, // 是否设置为文本理解模型（QA），务必保证至少有一个为true，否则知识库会报错 + "usedInClassify": true, // 是否用于问题分类（务必保证至少有一个为true） + "usedInExtractFields": true, // 是否用于内容提取（务必保证至少有一个为true） + "usedInToolCall": true, // 是否用于工具调用（务必保证至少有一个为true） + "usedInQueryExtension": true, // 是否用于问题优化（务必保证至少有一个为true） + "toolChoice": true, // 是否支持工具选择（分类，内容提取，工具调用会用到。） + "functionCall": false, // 是否支持函数调用（分类，内容提取，工具调用会用到。会优先使用 toolChoice，如果为false，则使用 functionCall，如果仍为 false，则使用提示词模式） + "customCQPrompt": "", // 自定义文本分类提示词（不支持工具和函数调用的模型 + "customExtractPrompt": "", // 自定义内容提取提示词 + "defaultSystemChatPrompt": "", // 对话默认携带的系统提示词 + "defaultConfig": {}, // 请求API时，挟带一些默认配置（比如 GLM4 的 top_p） + "fieldMap": {} // 字段映射（o1 模型需要把 max_tokens 映射为 max_completion_tokens） + }, + { + "provider": "Other", + "model": "Qwen/Qwen2-VL-72B-Instruct", + "name": "Qwen2-VL-72B-Instruct", + "maxContext": 32000, + "maxResponse": 4000, + "quoteMaxToken": 30000, + "maxTemperature": 1, + "charsPointsPrice": 0, + "censor": false, + "vision": true, + "datasetProcess": false, + "usedInClassify": false, + "usedInExtractFields": false, + "usedInToolCall": false, + "usedInQueryExtension": false, + "toolChoice": false, + "functionCall": false, + "customCQPrompt": "", + "customExtractPrompt": "", + "defaultSystemChatPrompt": "", + "defaultConfig": {} + } + ], + "vectorModels": [ + { + "provider": "Other", + "model": "Pro/BAAI/bge-m3", + "name": "Pro/BAAI/bge-m3", + "charsPointsPrice": 0, + "defaultToken": 512, + "maxToken": 5000, + "weight": 100 + } + ], + "reRankModels": [ + { + "model": "BAAI/bge-reranker-v2-m3", // 这里的model需要对应 siliconflow 的模型名 + "name": "BAAI/bge-reranker-v2-m3", + "requestUrl": "https://api.siliconflow.cn/v1/rerank", + "requestAuth": "siliconflow 上申请的 key" + } + ], + "audioSpeechModels": [ + { + "model": "fishaudio/fish-speech-1.5", + "name": "fish-speech-1.5", + "voices": [ + { + "label": "fish-alex", + "value": "fishaudio/fish-speech-1.5:alex", + "bufferId": "fish-alex" + }, + { + "label": "fish-anna", + "value": "fishaudio/fish-speech-1.5:anna", + "bufferId": "fish-anna" + }, + { + "label": "fish-bella", + "value": "fishaudio/fish-speech-1.5:bella", + "bufferId": "fish-bella" + }, + { + "label": "fish-benjamin", + "value": "fishaudio/fish-speech-1.5:benjamin", + "bufferId": "fish-benjamin" + }, + { + "label": "fish-charles", + "value": "fishaudio/fish-speech-1.5:charles", + "bufferId": "fish-charles" + }, + { + "label": "fish-claire", + "value": "fishaudio/fish-speech-1.5:claire", + "bufferId": "fish-claire" + }, + { + "label": "fish-david", + "value": "fishaudio/fish-speech-1.5:david", + "bufferId": "fish-david" + }, + { + "label": "fish-diana", + "value": "fishaudio/fish-speech-1.5:diana", + "bufferId": "fish-diana" + } + ] + } + ], + "whisperModel": { + "model": "FunAudioLLM/SenseVoiceSmall", + "name": "SenseVoiceSmall", + "charsPointsPrice": 0 + } +} +``` + +## 4. 重启 FastGPT + +## 5. 体验测试 + +### 测试对话和图片识别 + +随便新建一个简易应用，选择对应模型，并开启图片上传后进行测试： + +| | | +| --- | --- | +| ![alt text](/imgs/image-68.png) | ![alt text](/imgs/image-70.png) | + +可以看到，72B 的模型，性能还是非常快的，这要是本地没几个 4090，不说配置环境，输出怕都要 30s 了。 + +### 测试知识库导入和知识库问答 + +新建一个知识库（由于只配置了一个向量模型，页面上不会展示向量模型选择） + +| | | +| --- | --- | +| ![alt text](/imgs/image-72.png) | ![alt text](/imgs/image-71.png) | + +导入本地文件，直接选择文件，然后一路下一步即可。79 个索引，大概花了 20s 的时间就完成了。现在我们去测试一下知识库问答。 + +首先回到我们刚创建的应用，选择知识库，调整一下参数后即可开始对话： + +| | | | +| --- | --- | --- | +| ![alt text](/imgs/image-73.png) | ![alt text](/imgs/image-75.png) | ![alt text](/imgs/image-76.png) | + +对话完成后，点击底部的引用，可以查看引用详情，同时可以看到具体的检索和重排得分： + +| | | +| --- | --- | +| ![alt text](/imgs/image-77.png) | ![alt text](/imgs/image-78.png) | + +### 测试语音播放 + +继续在刚刚的应用中，左侧配置中找到语音播放，点击后可以从弹窗中选择语音模型，并进行试听： + +![alt text](/imgs/image-79.png) + +### 测试语言输入 + +继续在刚刚的应用中，左侧配置中找到语音输入，点击后可以从弹窗中开启语言输入 + +![alt text](/imgs/image-80.png) + +开启后，对话输入框中，会增加一个话筒的图标，点击可进行语音输入： + +| | | +| --- | --- | +| ![alt text](/imgs/image-81.png) | ![alt text](/imgs/image-82.png) | + +## 总结 + +如果你想快速的体验开源模型或者快速的使用 FastGPT，不想在不同服务商申请各类 Api Key，那么可以选择 SiliconCloud 的模型先进行快速体验。 + +如果你决定未来私有化部署模型和 FastGPT，前期可通过 SiliconCloud 进行测试验证，后期再进行硬件采购，减少 POC 时间和成本。 \ No newline at end of file diff --git a/docSite/content/zh-cn/docs/development/one-api.md b/docSite/content/zh-cn/docs/development/one-api.md deleted file mode 100644 index 79c5c08db..000000000 --- a/docSite/content/zh-cn/docs/development/one-api.md +++ /dev/null @@ -1,179 +0,0 @@ ---- -title: '使用 One API 接入 Azure、ChatGLM 和本地模型' -description: '部署和使用 One API，实现 Azure、ChatGLM 和本地模型的接入。' -icon: 'api' -draft: false -toc: true -weight: 708 ---- - -* 默认情况下，FastGPT 只配置了 GPT 的模型，如果你需要接入其他模型，需要进行一些额外配置。 -* [One API](https://github.com/songquanpeng/one-api) 是一个 OpenAI 接口管理 & 分发系统，可以通过标准的 OpenAI API 格式访问所有的大模型，开箱即用。 -* FastGPT 可以通过接入 One API 来实现对不同大模型的支持。One API 的部署方法也很简单。 - -## FastGPT 与 One API 关系 - -可以把 One API 当做一个网关。 - -![](/imgs/sealos-fastgpt.webp) - -## 部署 - -### Docker 版本 - -已加入最新的 `docker-compose.yml` 文件中。 - -### Sealos - MySQL 版本 - -MySQL 版本支持多实例，高并发。 - -直接点击以下按钮即可一键部署 👇 - -Deploy on Sealos - -部署完后会跳转「应用管理」，数据库在另一个应用「数据库」中。需要等待 1~3 分钟数据库运行后才能访问成功。 - -### Sealos - SqlLite 版本 - -SqlLite 版本不支持多实例，适合个人小流量使用，但是价格非常便宜。 - -**1. [点击打开 Sealos 公有云](https://cloud.sealos.io/)** - -**2. 打开 AppLaunchpad(应用管理) 工具** - -![step1](/imgs/oneapi-step1.webp) - -**3. 点击创建新应用** - -**4. 填写对应参数** - -镜像：ghcr.io/songquanpeng/one-api:latest - -![step2](/imgs/oneapi-step2.png) -打开外网访问开关后，Sealos 会自动分配一个可访问的地址，不需要自己配置。 - -![step3](/imgs/oneapi-step3.png) -填写完参数后，点击右上角部署即可。环境变量： - -``` -SESSION_SECRET=SESSION_SECRET -POLLING_INTERVAL=60 -BATCH_UPDATE_ENABLED=true -BATCH_UPDATE_INTERVAL=60 -``` - -## One API 使用教程 - -### 概念 - -1. 渠道： - 1. OneApi 中一个渠道对应一个 `Api Key`，这个 `Api Key` 可以是GPT、微软、ChatGLM、文心一言的。一个`Api Key`通常可以调用同一个厂商的多个模型。 - 2. One API 会根据请求传入的`模型`来决定使用哪一个`Key`，如果一个模型对应了多个`Key`，则会随机调用。 -2. 令牌：访问 One API 所需的凭证，只需要这`1`个凭证即可访问`One API`上配置的模型。因此`FastGPT`中，只需要配置`One API`的`baseurl`和`令牌`即可。 - -### 大致工作流程 - -1. 客户端请求 One API -2. 根据请求中的 `model` 参数，匹配对应的渠道（根据渠道里的模型进行匹配，必须完全一致）。如果匹配到多个渠道，则随机选择一个（同优先级）。 -3. One API 向真正的地址发出请求。 -4. One API 将结果返回给客户端。 - -### 1. 登录 One API - -打开 【One API 应用详情】，找到访问地址： -![step4](/imgs/oneapi-step4.png) - -登录 One API -![step5](/imgs/oneapi-step5.png) - -### 2. 创建渠道和令牌 - -在 One API 中添加对应渠道，直接点击 【添加基础模型】，不要遗漏了向量模型（Embedding） -![step6](/imgs/oneapi-step6.png) - -创建一个令牌 -![step7](/imgs/oneapi-step7.png) - -### 3. 修改账号余额 - -One API 默认 root 用户只有 200刀，可以自行修改编辑。 - -### 4. 修改 FastGPT 的环境变量 - -有了 One API 令牌后，FastGPT 可以通过修改 `baseurl` 和 `key` 去请求到 One API，再由 One API 去请求不同的模型。修改下面两个环境变量： - -```bash -# 下面的地址是 Sealos 提供的，务必写上 v1， 两个项目都在 sealos 部署时候，https://xxxx.cloud.sealos.io 可以改用内网地址 -OPENAI_BASE_URL=https://xxxx.cloud.sealos.io/v1 -# 下面的 key 是由 One API 提供的令牌 -CHAT_API_KEY=sk-xxxxxx -``` - -## 接入其他模型 - -**以添加文心一言为例:** - -### 1. One API 添加对应模型渠道 - -![](/imgs/oneapi-demo1.png) - -### 2. 修改 FastGPT 配置文件 - -可以在 `/projects/app/src/data/config.json` 里找到配置文件（本地开发需要复制成 config.local.json），配置文件中有一项是**对话模型配置**： - -```json -"llmModels": [ - ... - { - "model": "ERNIE-Bot", // 这里的模型需要对应 One API 的模型 - "name": "文心一言", // 对外展示的名称 - "avatar": "/imgs/model/openai.svg", // 模型的logo - "maxContext": 16000, // 最大上下文 - "maxResponse": 4000, // 最大回复 - "quoteMaxToken": 13000, // 最大引用内容 - "maxTemperature": 1.2, // 最大温度 - "charsPointsPrice": 0, - "censor": false, - "vision": false, // 是否支持图片输入 - "datasetProcess": true, // 是否设置为知识库处理模型 - "usedInClassify": true, // 是否用于问题分类 - "usedInExtractFields": true, // 是否用于字段提取 - "usedInToolCall": true, // 是否用于工具调用 - "usedInQueryExtension": true, // 是否用于问题优化 - "toolChoice": true, // 是否支持工具选择 - "functionCall": false, // 是否支持函数调用 - "customCQPrompt": "", // 自定义文本分类提示词（不支持工具和函数调用的模型 - "customExtractPrompt": "", // 自定义内容提取提示词 - "defaultSystemChatPrompt": "", // 对话默认携带的系统提示词 - "defaultConfig":{} // 请求API时，挟带一些默认配置（比如 GLM4 的 top_p） - } - ... -], -``` - -**添加向量模型:** - -```json -"vectorModels": [ - ...... - { - "model": "text-embedding-ada-002", - "name": "Embedding-2", - "avatar": "/imgs/model/openai.svg", - "charsPointsPrice": 0, - "defaultToken": 700, - "maxToken": 3000, - "weight": 100 - }, - ...... -] -``` - -### 3. 重启 FastGPT - -```bash -docker-compose down -docker-compose up -d -``` - -重启 FastGPT 即可在选择文心一言模型进行对话。**添加向量模型也是类似操作，增加到 `vectorModels`里。** diff --git a/docSite/content/zh-cn/docs/development/openapi/chat.md b/docSite/content/zh-cn/docs/development/openapi/chat.md index 3af0542cd..25f2c835d 100644 --- a/docSite/content/zh-cn/docs/development/openapi/chat.md +++ b/docSite/content/zh-cn/docs/development/openapi/chat.md @@ -866,6 +866,8 @@ curl --location --request DELETE 'http://localhost:3000/api/core/chat/delHistory ### 清空所有历史记录 +仅会情况通过 API Key 创建的对话历史记录，不会清空在线使用、分享链接等其他来源的对话历史记录。 + {{< tabs tabTotal="3" >}} {{< tab tabName="请求示例" >}} {{< markdownify >}} @@ -1313,6 +1315,83 @@ curl --location --request POST 'http://localhost:3000/api/core/chat/feedback/upd ## 猜你想问 +**4.8.16 后新版接口** + +新版猜你想问，必须包含 appId 和 chatId 的参数才可以进行使用。会自动根据 chatId 去拉取最近 6 轮对话记录作为上下文来引导回答。 + +{{< tabs tabTotal="3" >}} +{{< tab tabName="请求示例" >}} +{{< markdownify >}} + +```bash +curl --location --request POST 'http://localhost:3000/api/core/ai/agent/v2/createQuestionGuide' \ +--header 'Authorization: Bearer {{apikey}}' \ +--header 'Content-Type: application/json' \ +--data-raw '{ + "appId": "appId", + "chatId": "chatId", + "questionGuide": { + "open": true, + "model": "GPT-4o-mini", + "customPrompt": "你是一个智能助手，请根据用户的问题生成猜你想问。" + } +}' +``` + +{{< /markdownify >}} +{{< /tab >}} + +{{< tab tabName="参数说明" >}} +{{< markdownify >}} + +{{% alert icon=" " context="success" %}} + +| 参数名 | 类型 | 必填 | 说明 | +| --- | --- | --- | --- | +| appId | string | ✅ | 应用 Id | +| chatId | string | ✅ | 对话 Id | +| questionGuide | object | | 自定义配置，不传的话，则会根据 appId，取最新发布版本的配置 | + +```ts +type CreateQuestionGuideParams = OutLinkChatAuthProps & { + appId: string; + chatId: string; + questionGuide?: { + open: boolean; + model?: string; + customPrompt?: string; + }; +}; +``` + +{{% /alert %}} + +{{< /markdownify >}} +{{< /tab >}} + +{{< tab tabName="响应示例" >}} +{{< markdownify >}} + +```json +{ + "code": 200, + "statusText": "", + "message": "", + "data": [ + "你对AI有什么看法？", + "想了解AI的应用吗？", + "你希望AI能做什么？" + ] +} +``` +{{< /markdownify >}} +{{< /tab >}} +{{< /tabs >}} + +--- + +**4.8.16 前旧版接口：** + {{< tabs tabTotal="3" >}} {{< tab tabName="请求示例" >}} {{< markdownify >}} @@ -1369,3 +1448,5 @@ curl --location --request POST 'http://localhost:3000/api/core/ai/agent/createQu + + diff --git a/docSite/content/zh-cn/docs/development/openapi/dataset.md b/docSite/content/zh-cn/docs/development/openapi/dataset.md index 15f5145a1..88877e3af 100644 --- a/docSite/content/zh-cn/docs/development/openapi/dataset.md +++ b/docSite/content/zh-cn/docs/development/openapi/dataset.md @@ -1424,7 +1424,11 @@ curl --location --request POST 'https://api.fastgpt.in/api/core/dataset/searchTe "limit": 5000, "similarity": 0, "searchMode": "embedding", - "usingReRank": false + "usingReRank": false, + + "datasetSearchUsingExtensionQuery": true, + "datasetSearchExtensionModel": "gpt-4o-mini", + "datasetSearchExtensionBg": "" }' ``` @@ -1441,6 +1445,9 @@ curl --location --request POST 'https://api.fastgpt.in/api/core/dataset/searchTe - similarity - 最低相关度（0~1，可选） - searchMode - 搜索模式：embedding | fullTextRecall | mixedRecall - usingReRank - 使用重排 +- datasetSearchUsingExtensionQuery - 使用问题优化 +- datasetSearchExtensionModel - 问题优化模型 +- datasetSearchExtensionBg - 问题优化背景描述 {{% /alert %}} {{< /markdownify >}} diff --git a/docSite/content/zh-cn/docs/development/sealos.md b/docSite/content/zh-cn/docs/development/sealos.md index eaebf7305..9e4f25e84 100644 --- a/docSite/content/zh-cn/docs/development/sealos.md +++ b/docSite/content/zh-cn/docs/development/sealos.md @@ -15,7 +15,7 @@ weight: 706 FastGPT 使用了 one-api 项目来管理模型池，其可以兼容 OpenAI 、Azure 、国内主流模型和本地模型等。 -可参考：[Sealos 快速部署 OneAPI](/docs/development/one-api) +可参考：[Sealos 快速部署 OneAPI](/docs/development/modelconfig/one-api) ## 一键部署 @@ -163,4 +163,4 @@ SYSTEM_FAVICON 可以是一个网络地址 ### One API 使用 -[参考 OneAPI 使用步骤](/docs/development/one-api/) \ No newline at end of file +[参考 OneAPI 使用步骤](/docs/development/modelconfig/one-api/) \ No newline at end of file diff --git a/docSite/content/zh-cn/docs/development/upgrading/43.md b/docSite/content/zh-cn/docs/development/upgrading/43.md index b8506ac88..5577018ac 100644 --- a/docSite/content/zh-cn/docs/development/upgrading/43.md +++ b/docSite/content/zh-cn/docs/development/upgrading/43.md @@ -1,5 +1,5 @@ --- -title: '升级到 V4.3(需要初始化)' +title: '升级到 V4.3(包含升级脚本)' description: 'FastGPT 从旧版本升级到 V4.3 操作指南' icon: 'upgrade' draft: false diff --git a/docSite/content/zh-cn/docs/development/upgrading/44.md b/docSite/content/zh-cn/docs/development/upgrading/44.md index e9fd3ff0c..4c2ee7037 100644 --- a/docSite/content/zh-cn/docs/development/upgrading/44.md +++ b/docSite/content/zh-cn/docs/development/upgrading/44.md @@ -1,5 +1,5 @@ --- -title: '升级到 V4.4(需要初始化)' +title: '升级到 V4.4(包含升级脚本)' description: 'FastGPT 从旧版本升级到 V4.4 操作指南' icon: 'upgrade' draft: false diff --git a/docSite/content/zh-cn/docs/development/upgrading/441.md b/docSite/content/zh-cn/docs/development/upgrading/441.md index a133038c3..8a885c343 100644 --- a/docSite/content/zh-cn/docs/development/upgrading/441.md +++ b/docSite/content/zh-cn/docs/development/upgrading/441.md @@ -1,5 +1,5 @@ --- -title: '升级到 V4.4.1(需要初始化)' +title: '升级到 V4.4.1(包含升级脚本)' description: 'FastGPT 从旧版本升级到 V4.4.1 操作指南' icon: 'upgrade' draft: false diff --git a/docSite/content/zh-cn/docs/development/upgrading/442.md b/docSite/content/zh-cn/docs/development/upgrading/442.md index de0f66383..b09b75bc9 100644 --- a/docSite/content/zh-cn/docs/development/upgrading/442.md +++ b/docSite/content/zh-cn/docs/development/upgrading/442.md @@ -1,5 +1,5 @@ --- -title: '升级到 V4.4.2(需要初始化)' +title: '升级到 V4.4.2(包含升级脚本)' description: 'FastGPT 从旧版本升级到 V4.4.2 操作指南' icon: 'upgrade' draft: false diff --git a/docSite/content/zh-cn/docs/development/upgrading/445.md b/docSite/content/zh-cn/docs/development/upgrading/445.md index c4162dd07..11914a2c0 100644 --- a/docSite/content/zh-cn/docs/development/upgrading/445.md +++ b/docSite/content/zh-cn/docs/development/upgrading/445.md @@ -1,5 +1,5 @@ --- -title: 'V4.4.5(需要初始化)' +title: 'V4.4.5(包含升级脚本)' description: 'FastGPT V4.4.5 更新' icon: 'upgrade' draft: false diff --git a/docSite/content/zh-cn/docs/development/upgrading/46.md b/docSite/content/zh-cn/docs/development/upgrading/46.md index f71423a3b..4501ab3b7 100644 --- a/docSite/content/zh-cn/docs/development/upgrading/46.md +++ b/docSite/content/zh-cn/docs/development/upgrading/46.md @@ -1,5 +1,5 @@ --- -title: 'V4.6(需要初始化)' +title: 'V4.6(包含升级脚本)' description: 'FastGPT V4.6 更新' icon: 'upgrade' draft: false diff --git a/docSite/content/zh-cn/docs/development/upgrading/462.md b/docSite/content/zh-cn/docs/development/upgrading/462.md index 8f47cc80f..7e5d86f8a 100644 --- a/docSite/content/zh-cn/docs/development/upgrading/462.md +++ b/docSite/content/zh-cn/docs/development/upgrading/462.md @@ -1,5 +1,5 @@ --- -title: 'V4.6.2(需要初始化)' +title: 'V4.6.2(包含升级脚本)' description: 'FastGPT V4.6.2' icon: 'upgrade' draft: false diff --git a/docSite/content/zh-cn/docs/development/upgrading/463.md b/docSite/content/zh-cn/docs/development/upgrading/463.md index c9d5f5e3b..fe5a3c58b 100644 --- a/docSite/content/zh-cn/docs/development/upgrading/463.md +++ b/docSite/content/zh-cn/docs/development/upgrading/463.md @@ -1,5 +1,5 @@ --- -title: 'V4.6.3(需要初始化)' +title: 'V4.6.3(包含升级脚本)' description: 'FastGPT V4.6.3' icon: 'upgrade' draft: false diff --git a/docSite/content/zh-cn/docs/development/upgrading/464.md b/docSite/content/zh-cn/docs/development/upgrading/464.md index 125ba0bc5..0d7c7f297 100644 --- a/docSite/content/zh-cn/docs/development/upgrading/464.md +++ b/docSite/content/zh-cn/docs/development/upgrading/464.md @@ -1,5 +1,5 @@ --- -title: 'V4.6.4(需要初始化)' +title: 'V4.6.4(包含升级脚本)' description: 'FastGPT V4.6.4' icon: 'upgrade' draft: false diff --git a/docSite/content/zh-cn/docs/development/upgrading/469.md b/docSite/content/zh-cn/docs/development/upgrading/469.md index ceca3a483..728cb111b 100644 --- a/docSite/content/zh-cn/docs/development/upgrading/469.md +++ b/docSite/content/zh-cn/docs/development/upgrading/469.md @@ -1,5 +1,5 @@ --- -title: 'V4.6.9(需要初始化)' +title: 'V4.6.9(包含升级脚本)' description: 'FastGPT V4.6.9更新说明' icon: 'upgrade' draft: false diff --git a/docSite/content/zh-cn/docs/development/upgrading/471.md b/docSite/content/zh-cn/docs/development/upgrading/471.md index 4df25de43..f621ffaed 100644 --- a/docSite/content/zh-cn/docs/development/upgrading/471.md +++ b/docSite/content/zh-cn/docs/development/upgrading/471.md @@ -1,5 +1,5 @@ --- -title: 'V4.7.1(需要初始化)' +title: 'V4.7.1(包含升级脚本)' description: 'FastGPT V4.7.1 更新说明' icon: 'upgrade' draft: false diff --git a/docSite/content/zh-cn/docs/development/upgrading/481.md b/docSite/content/zh-cn/docs/development/upgrading/481.md index 24f2e249f..b5afc2f52 100644 --- a/docSite/content/zh-cn/docs/development/upgrading/481.md +++ b/docSite/content/zh-cn/docs/development/upgrading/481.md @@ -1,5 +1,5 @@ --- -title: 'V4.8.1(需要初始化)' +title: 'V4.8.1(包含升级脚本)' description: 'FastGPT V4.8.1 更新说明' icon: 'upgrade' draft: false diff --git a/docSite/content/zh-cn/docs/development/upgrading/4810.md b/docSite/content/zh-cn/docs/development/upgrading/4810.md index f587d2551..4eca91636 100644 --- a/docSite/content/zh-cn/docs/development/upgrading/4810.md +++ b/docSite/content/zh-cn/docs/development/upgrading/4810.md @@ -1,5 +1,5 @@ --- -title: 'V4.8.10(需要初始化)' +title: 'V4.8.10(包含升级脚本)' description: 'FastGPT V4.8.10 更新说明' icon: 'upgrade' draft: false diff --git a/docSite/content/zh-cn/docs/development/upgrading/4812.md b/docSite/content/zh-cn/docs/development/upgrading/4812.md index 4f8391132..646d01686 100644 --- a/docSite/content/zh-cn/docs/development/upgrading/4812.md +++ b/docSite/content/zh-cn/docs/development/upgrading/4812.md @@ -1,5 +1,5 @@ --- -title: 'V4.8.12(需要初始化)' +title: 'V4.8.12(包含升级脚本)' description: 'FastGPT V4.8.12 更新说明' icon: 'upgrade' draft: false diff --git a/docSite/content/zh-cn/docs/development/upgrading/4815.md b/docSite/content/zh-cn/docs/development/upgrading/4815.md index cd17f88a7..56dacbca7 100644 --- a/docSite/content/zh-cn/docs/development/upgrading/4815.md +++ b/docSite/content/zh-cn/docs/development/upgrading/4815.md @@ -1,5 +1,5 @@ --- -title: 'V4.8.15(初始化)' +title: 'V4.8.15(包含升级脚本)' description: 'FastGPT V4.8.15 更新说明' icon: 'upgrade' draft: false @@ -23,23 +23,35 @@ weight: 809 ## 升级指南 -- 更新 FastGPT 镜像 tag: v4.8.15 -- 更新 FastGPT 商业版镜像 tag: v4.8.15 （fastgpt-pro镜像） +- 更新 fastgpt 镜像 tag: v4.8.15-fix3 +- 更新 fastgpt-pro 商业版镜像 tag: v4.8.15 - Sandbox 镜像，可以不更新 -## 运行初始化脚本 +## 运行升级脚本 从任意终端，发起 1 个 HTTP 请求。其中 {{rootkey}} 替换成环境变量里的 `rootkey`；{{host}} 替换成**FastGPT 域名**。 ```bash -curl --location --request POST 'https://{{host}}/admin/initv4815' \ +curl --location --request POST 'https://{{host}}/api/admin/initv4815' \ --header 'rootkey: {{rootkey}}' \ --header 'Content-Type: application/json' ``` 会重置应用定时执行的字段，把 null 去掉，减少索引大小。 +---- + +从任意终端，发起 1 个 HTTP 请求。其中 {{rootkey}} 替换成环境变量里的 `rootkey`；{{host}} 替换成**fastgpt-pro域名**。 + +```bash +curl --location --request POST 'https://{{host}}/api/admin/init/refreshFreeUser' \ +--header 'rootkey: {{rootkey}}' \ +--header 'Content-Type: application/json' +``` + +重新计算一次免费版用户的时长，之前有版本升级时没有重新计算时间，导致会误发通知。 + ## 完整更新内容 @@ -61,4 +73,4 @@ curl --location --request POST 'https://{{host}}/admin/initv4815' \ 16. 修复 - 语言播放鉴权问题。 17. 修复 - 插件应用知识库引用上限始终为 3000 18. 修复 - 工作流编辑记录存储上限，去掉本地存储，增加异常离开时，强制自动保存。 -19. 修复 - 工作流特殊变量替换问题。（$开头的字符串无法替换） \ No newline at end of file +19. 修复 - 工作流特殊变量替换问题。（$开头的字符串无法替换） diff --git a/docSite/content/zh-cn/docs/development/upgrading/4816.md b/docSite/content/zh-cn/docs/development/upgrading/4816.md new file mode 100644 index 000000000..d2f84bc35 --- /dev/null +++ b/docSite/content/zh-cn/docs/development/upgrading/4816.md @@ -0,0 +1,73 @@ +--- +title: 'V4.8.16(更新配置文件)' +description: 'FastGPT V4.8.16 更新说明' +icon: 'upgrade' +draft: false +toc: true +weight: 808 +--- + +## 更新指南 + +### 1. 更新镜像： + +- 更新 fastgpt 镜像 tag: v4.8.16 +- 更新 fastgpt-pro 商业版镜像 tag: v4.8.16 +- Sandbox 镜像 tag: v4.8.16 + +### 2. 更新配置文件 + +参考最新的[配置文件](/docs/development/configuration/)，更新 `config.json` 或 admin 中模型文件配置。给 LLMModel 和 VectorModel 增加 `provider` 字段，以便进行模型分类。例如： + +```json +{ + "provider": "OpenAI", // 这是新增的 + "model": "gpt-4o", + "name": "gpt-4o", + "maxContext": 125000, + "maxResponse": 4000, + "quoteMaxToken": 120000, + "maxTemperature": 1.2, + "charsPointsPrice": 0, + "censor": false, + "vision": true, + "datasetProcess": true, + "usedInClassify": true, + "usedInExtractFields": true, + "usedInToolCall": true, + "usedInQueryExtension": true, + "toolChoice": true, + "functionCall": false, + "customCQPrompt": "", + "customExtractPrompt": "", + "defaultSystemChatPrompt": "", + "defaultConfig": {}, + "fieldMap": {} +} +``` + + + +## 完整更新内容 + +1. 新增 - SearXNG 搜索插件[点击查看教程](/docs/guide/plugins/searxng_plugin_guide/) +2. 新增 - 商业版支持 API 知识库和链接集合定时同步。 +3. 新增 - 猜你想问支持选择模型和自定义提示词。 +4. 新增 - 钉钉和企微机器人 webhook 插件。 +5. 新增 - 商业版支持钉钉 SSO 登录配置。[点击查看教程](/docs/guide/admin/sso_dingtalk/) +6. 新增 - 商业版支持飞书和语雀知识库导入。[点击查看教程](/docs/guide/knowledge_base/lark_dataset/) +7. 新增 - sandbox 新增 createHmac 加密全局方法。 +8. 新增 - 工作流右键支持全部折叠。 +9. 优化 - 模型选择器。 +10. 优化 - SSR 渲染，预判断是移动端还是 pc 端，减少页面抖动。 +11. 优化 - 工作流/简易模式变量初始化代码，去除监听初始化，避免因渲染顺序不一致导致的失败。 +12. 优化 - 工作流获取数据类型不一致数据时，增加类型转化，避免 undefined。 +13. 修复 - 无法自动切换默认语言。增加分享链接，强制执行一次切换默认语言。 +14. 修复 - 数组选择器自动兼容 4.8.13 以前的数据。 +15. 修复 - 站点同步知识库，链接同步时未使用选择器。 +16. 修复 - 简易模式转工作流，没有把系统配置项转化。 +17. 修复 - 插件独立运行，变量初始值未赋上。 +18. 修复 - 工作流使用弹窗组件时，关闭弹窗后，有时候会出现页面偏移。 +19. 修复 - 插件调试时，日志未保存插件输入参数。 +20. 修复 - 部分模板市场模板 +21. 修复 - 设置NEXT_PUBLIC_BASE_URL时，图片文件读取URL不正确 \ No newline at end of file diff --git a/docSite/content/zh-cn/docs/development/upgrading/4817.md b/docSite/content/zh-cn/docs/development/upgrading/4817.md new file mode 100644 index 000000000..64ce31fe5 --- /dev/null +++ b/docSite/content/zh-cn/docs/development/upgrading/4817.md @@ -0,0 +1,52 @@ +--- +title: 'V4.8.17(包含升级脚本)' +description: 'FastGPT V4.8.17 更新说明' +icon: 'upgrade' +draft: false +toc: true +weight: 807 +--- + +## 更新指南 + +### 1. 更新镜像： + +- 更新 fastgpt 镜像 tag: v4.8.17-fix-title +- 更新 fastgpt-pro 商业版镜像 tag: v4.8.17 +- Sandbox 镜像无需更新 + + +### 2. 运行升级脚本 + +从任意终端，发起 1 个 HTTP 请求。其中 {{rootkey}} 替换成环境变量里的 `rootkey`；{{host}} 替换成**FastGPT 域名**。 + +```bash +curl --location --request POST 'https://{{host}}/api/admin/initv4817' \ +--header 'rootkey: {{rootkey}}' \ +--header 'Content-Type: application/json' +``` + +会将用户绑定的 OpenAI 账号移动到团队中。 + + +## 调整 completions 接口返回值 + +/api/v1/chat/completions 接口返回值调整，对话节点、工具节点等使用到模型的节点，将不再返回 `tokens` 字段，改为返回 `inputTokens` 和 `outputTokens` 字段，分别表示输入和输出的 Token 数量。 + +## 完整更新内容 + +1. 新增 - 简易模式工具调用支持数组类型插件。 +2. 新增 - 工作流增加异常离开自动保存，避免工作流丢失。 +3. 新增 - LLM 模型参数支持关闭 max_tokens 和 temperature。 +4. 新增 - 商业版支持后台配置模板市场。 +5. 新增 - 商业版支持后台配置自定义工作流变量，用于与业务系统鉴权打通。 +6. 新增 - 搜索测试接口支持问题优化。 +7. 新增 - 工作流中 Input Token 和 Output Token 分开记录展示。并修复部分请求未记录输出 Token 计费问题。 +8. 优化 - Markdown 大小测试，超出 20 万字符不使用 Markdown 组件，避免崩溃。 +9. 优化 - 知识库搜索参数，滑动条支持输入模式，可以更精准的控制。 +10. 优化 - 可用模型展示UI。 +11. 优化 - Mongo 查询语句，增加 virtual 字段。 +12. 修复 - 文件返回接口缺少 Content-Length 头，导致通过非同源文件上传时，阿里 vision 模型无法识别图片。 +13. 修复 - 去除判断器两端字符串隐藏换行符，避免判断器失效。 +14. 修复 - 变量更新节点，手动输入更新内容时候，非字符串类型数据类型无法自动转化。 +15. 修复 - 豆包模型无法工具调用。 diff --git a/docSite/content/zh-cn/docs/development/upgrading/4818.md b/docSite/content/zh-cn/docs/development/upgrading/4818.md new file mode 100644 index 000000000..aed267f88 --- /dev/null +++ b/docSite/content/zh-cn/docs/development/upgrading/4818.md @@ -0,0 +1,47 @@ +--- +title: 'V4.8.18' +description: 'FastGPT V4.8.18 更新说明' +icon: 'upgrade' +draft: false +toc: true +weight: 806 +--- + +## 更新指南 + +### 1. 更新镜像： + +- 更新 fastgpt 镜像 tag: v4.8.18 +- 更新 fastgpt-pro 商业版镜像 tag: v4.8.18 +- Sandbox 镜像无需更新 + +### 2. 运行升级脚本 + +从任意终端，发起 1 个 HTTP 请求。其中 {{rootkey}} 替换成环境变量里的 `rootkey`；{{host}} 替换成**FastGPT 域名**。 + +```bash +curl --location --request POST 'https://{{host}}/api/admin/initv4818' \ +--header 'rootkey: {{rootkey}}' \ +--header 'Content-Type: application/json' +``` + +会迁移全文检索表，时间较长，迁移期间全文检索会失效，日志中会打印已经迁移的数据长度。 + + +## 完整更新内容 + +1. 新增 - 支持通过 JSON 配置直接创建应用。 +2. 新增 - 支持通过 CURL 脚本快速创建 HTTP 插件。 +3. 新增 - 商业版支持部门架构权限模式。 +4. 新增 - 支持配置自定跨域安全策略，默认全开。 +5. 新增 - 补充私有部署，模型问题排查文档。 +6. 优化 - HTTP Body 增加特殊处理，解决字符串变量带换行时无法解析问题。 +7. 优化 - 分享链接随机生成用户头像。 +8. 优化 - 图片上传安全校验。并增加头像图片唯一存储，确保不会累计存储。 +9. 优化 - Mongo 全文索引表分离。 +10. 优化 - 知识库检索查询语句合并，同时减少查库数量。 +11. 优化 - 文件编码检测，减少 CSV 文件乱码概率。 +12. 优化 - 异步读取文件内容，减少进程阻塞。 +13. 优化 - 文件阅读，HTML 直接下载，不允许在线阅读。 +14. 修复 - HTML 文件上传，base64 图片无法自动转图片链接。 +15. 修复 - 插件计费错误。 diff --git a/docSite/content/zh-cn/docs/development/upgrading/484.md b/docSite/content/zh-cn/docs/development/upgrading/484.md index b415c4f52..9c1cec472 100644 --- a/docSite/content/zh-cn/docs/development/upgrading/484.md +++ b/docSite/content/zh-cn/docs/development/upgrading/484.md @@ -1,5 +1,5 @@ --- -title: 'V4.8.4(需要初始化)' +title: 'V4.8.4(包含升级脚本)' description: 'FastGPT V4.8.4 更新说明' icon: 'upgrade' draft: false diff --git a/docSite/content/zh-cn/docs/development/upgrading/485.md b/docSite/content/zh-cn/docs/development/upgrading/485.md index 6c2e0c764..a6986323e 100644 --- a/docSite/content/zh-cn/docs/development/upgrading/485.md +++ b/docSite/content/zh-cn/docs/development/upgrading/485.md @@ -1,5 +1,5 @@ --- -title: 'V4.8.5(需要初始化)' +title: 'V4.8.5(包含升级脚本)' description: 'FastGPT V4.8.5 更新说明' icon: 'upgrade' draft: false diff --git a/docSite/content/zh-cn/docs/development/upgrading/486.md b/docSite/content/zh-cn/docs/development/upgrading/486.md index bc508ae2e..197f60da6 100644 --- a/docSite/content/zh-cn/docs/development/upgrading/486.md +++ b/docSite/content/zh-cn/docs/development/upgrading/486.md @@ -1,5 +1,5 @@ --- -title: 'V4.8.6(需要初始化)' +title: 'V4.8.6(包含升级脚本)' description: 'FastGPT V4.8.6 更新说明' icon: 'upgrade' draft: false diff --git a/docSite/content/zh-cn/docs/development/upgrading/488.md b/docSite/content/zh-cn/docs/development/upgrading/488.md index fcbf4efbc..859688808 100644 --- a/docSite/content/zh-cn/docs/development/upgrading/488.md +++ b/docSite/content/zh-cn/docs/development/upgrading/488.md @@ -1,5 +1,5 @@ --- -title: 'V4.8.8(需要初始化)' +title: 'V4.8.8(包含升级脚本)' description: 'FastGPT V4.8.8 更新说明' icon: 'upgrade' draft: false diff --git a/docSite/content/zh-cn/docs/development/upgrading/intro.md b/docSite/content/zh-cn/docs/development/upgrading/intro.md index 1bd3d5b53..7a692c07f 100644 --- a/docSite/content/zh-cn/docs/development/upgrading/intro.md +++ b/docSite/content/zh-cn/docs/development/upgrading/intro.md @@ -55,14 +55,14 @@ docker-compose up -d ## 执行升级初始化脚本 -镜像更新完后，可以查看文档中的`版本介绍`，通常需要执行升级脚本的版本都会标明`需要初始化`，打开对应的文档，参考说明执行初始化脚本即可，大部分时候都是需要发送一个`POST`请求。 +镜像更新完后，可以查看文档中的`版本介绍`，通常需要执行升级脚本的版本都会标明`包含升级脚本`，打开对应的文档，参考说明执行**升级脚本**即可，大部分时候都是需要发送一个`POST`请求。 ## QA -### 为什么需要初始化 +### 为什么需要执行升级脚本 -数据表出现大幅度变更，无法通过设置默认值，或复杂度较高时，会通过初始化来更新部分数据表字段。 +数据表出现大幅度变更，无法通过设置默认值，或复杂度较高时，会通过升级脚本来更新部分数据表字段。 严格按初始化步骤进行操作，不会造成旧数据丢失。但在初始化过程中，如果数据量大，需要初始化的时间较长，这段时间可能会造成服务无法正常使用。 ### {{host}} 是什么 diff --git a/docSite/content/zh-cn/docs/faq/app.md b/docSite/content/zh-cn/docs/faq/app.md index 41e427276..5797a3f60 100644 --- a/docSite/content/zh-cn/docs/faq/app.md +++ b/docSite/content/zh-cn/docs/faq/app.md @@ -7,10 +7,34 @@ toc: true weight: 908 --- -## 工作流中多轮对话场景中如何使连续问题被问题分类节点正确的归类 +## 多轮对话中如何使连续问题被问题分类节点正确的归类 问题分类节点具有获取上下文信息的能力，当处理两个关联性较大的问题时，模型的判断准确性往往依赖于这两个问题之间的联系和模型的能力。例如，当用户先问“我该如何使用这个功能？”接着又询问“这个功能有什么限制？”时，模型借助上下文信息，就能够更精准地理解并响应。 但是，当连续问题之间的关联性较小，模型判断的准确度可能会受到限制。在这种情况下，我们可以引入全局变量的概念来记录分类结果。在后续的问题分类阶段，首先检查全局变量是否存有分类结果。如果有，那么直接沿用该结果；若没有，则让模型自行判断。 -建议：构建批量运行脚本进行测试，评估问题分类的准确性。 \ No newline at end of file +建议：构建批量运行脚本进行测试，评估问题分类的准确性。 + +## 定时执行的时机问题 + +系统编排配置中的定时执行，如果用户打开分享的连接，停留在那个页面，定时执行触发问题： + +定时执行会在应用发布后生效，会在后台生效。 + +## AI对话回答要求中的Markdown语法取消 + +修改知识库默认提示词, 默认用的是标准模板提示词，会要求按 Markdown 输出，可以去除该要求： + +| | | +| --- | --- | +| ![](/imgs/image-83.png) | ![](/imgs/image-84.png) | + +## 应用在不同来源效果不一致 + +Q: 应用在调试和正式发布后，效果不一致；在 API 调用时，效果不一致。 + +A: 通常是由于上下文不一致导致，可以在对话日志中，找到对应的记录，并查看运行详情来进行比对。 + +| | | | +| --- | --- | --- | +| ![](/imgs/image-85.png) | ![](/imgs/image-86.png) | ![](/imgs/image-87.png) | diff --git a/docSite/content/zh-cn/docs/faq/dataset.md b/docSite/content/zh-cn/docs/faq/dataset.md index e66ebc01e..589b57160 100644 --- a/docSite/content/zh-cn/docs/faq/dataset.md +++ b/docSite/content/zh-cn/docs/faq/dataset.md @@ -14,4 +14,51 @@ weight: 910 ## 知识库配置里的文件处理模型是什么？与索引模型有什么区别？ * **文件处理模型**：用于数据处理的【增强处理】和【问答拆分】。在【增强处理】中，生成相关问题和摘要，在【问答拆分】中执行问答对生成。 -* **索引模型**：用于向量化，即通过对文本数据进行处理和组织，构建出一个能够快速查询的数据结构。 \ No newline at end of file +* **索引模型**：用于向量化，即通过对文本数据进行处理和组织，构建出一个能够快速查询的数据结构。 + +## 基于知识库的查询，但是问题相关的答案过多。ai回答到一半就不继续回答。 + +FastGPT回复长度计算公式: + +最大回复=min(配置的最大回复（内置的限制），最大上下文（输入和输出的总和）-历史记录) + +18K模型->输入与输出的和 + +输出增多->输入减小 + +所以可以： + +1. 检查配置的最大回复（回复上限） +2. 减小输入来增大输出，即减小历史记录，在工作流其实也就是“聊天记录” + +配置的最大回复： + +![](/imgs/dataset1.png) + +![](/imgs/dataset2.png) + +1. 私有化部署的时候，后台配模型参数，可以在配置最大上文时候，预留一些空间，比如 128000 的模型，可以只配置 120000, 剩余的空间后续会被安排给输出 + + +## 受到模型上下文的限制，有时候达不到聊天记录的轮次，连续对话字数过多就会报上下文不够的错误。 + +FastGPT回复长度计算公式: + +最大回复=min(配置的最大回复（内置的限制），最大上下文（输入和输出的总和）-历史记录) + +18K模型->输入与输出的和 + +输出增多->输入减小 + +所以可以： + +1. 检查配置的最大回复（回复上限） +2. 减小输入来增大输出，即减小历史记录，在工作流其实也就是“聊天记录” + +配置的最大回复： + +![](/imgs/dataset1.png) + +![](/imgs/dataset2.png) + +1. 私有化部署的时候，后台配模型参数，可以在配置最大上文时候，预留一些空间，比如 128000 的模型，可以只配置 120000, 剩余的空间后续会被安排给输出 \ No newline at end of file diff --git a/docSite/content/zh-cn/docs/guide/admin/_index.md b/docSite/content/zh-cn/docs/guide/admin/_index.md new file mode 100644 index 000000000..fba2dab4e --- /dev/null +++ b/docSite/content/zh-cn/docs/guide/admin/_index.md @@ -0,0 +1,9 @@ +--- +weight: 490 +title: '商业版后台' +description: '商业版后台使用教程' +icon: 'chat_bubble' +draft: false +images: [] +--- + diff --git a/docSite/content/zh-cn/docs/guide/admin/sso_dingtalk.md b/docSite/content/zh-cn/docs/guide/admin/sso_dingtalk.md new file mode 100644 index 000000000..18bb00c81 --- /dev/null +++ b/docSite/content/zh-cn/docs/guide/admin/sso_dingtalk.md @@ -0,0 +1,44 @@ +--- +weight: 490 +title: '钉钉 SSO 配置' +description: '钉钉 SSO 登录' +icon: 'chat_bubble' +draft: false +images: [] +--- + +## 1. 注册钉钉应用 + +登录 [钉钉开放平台](https://open-dev.dingtalk.com/fe/app?hash=%23%2Fcorp%2Fapp#/corp/app)，创建一个应用。 + +![alt text](/imgs/image-25.png) + +## 2. 配置钉钉应用安全设置 + +点击进入创建好的应用后，点开`安全设置`，配置出口 IP（服务器 IP），和重定向 URL。重定向 URL 填写逻辑： + +`{{fastgpt 域名}}/login/provider` + +![alt text](/imgs/image-26.png) + +## 3. 设置钉钉应用权限 + +点击进入创建好的应用后，点开`权限设置`，开放两个权限： `个人手机号信息`和`通讯录个人信息读权限` + +![alt text](/imgs/image-27.png) + +## 4. 发布应用 + +点击进入创建好的应用后，点开`版本管理与发布`，随便创建一个新版本即可。 + +## 5. 在 FastGPT Admin 配置钉钉应用 id + +名字都是对应上，直接填写即可。 + +| | | +| --- | --- | +| ![alt text](/imgs/image-28.png)| ![alt text](/imgs/image-29.png) | + +## 6. 测试 + +![alt text](/imgs/image-30.png) \ No newline at end of file diff --git a/docSite/content/zh-cn/docs/guide/course/ai_settings.md b/docSite/content/zh-cn/docs/guide/course/ai_settings.md index 8f748bfb5..9d69335fa 100644 --- a/docSite/content/zh-cn/docs/guide/course/ai_settings.md +++ b/docSite/content/zh-cn/docs/guide/course/ai_settings.md @@ -11,11 +11,14 @@ weight: 104 | | | | | --- | --- | --- | -| ![](/imgs/aichat0.png) | ![](/imgs/aichat02.png) | ![](/imgs/aichat2.png) | +| ![alt text](/imgs/image-51.png) | ![alt text](/imgs/image-52.png) | ![alt text](/imgs/image-53.png) | -## 返回AI内容（高级编排特有） +## 流响应（高级编排 AI 对话 特有） -这是一个开关，打开的时候，当 AI 对话模块运行时，会将其输出的内容返回到浏览器（API响应）；如果关闭，AI 输出的内容不会返回到浏览器，但是生成的内容仍可以通过【AI回复】进行输出。你可以将【AI回复】连接到其他模块中。 +旧版名字叫做：返回 AI 内容；新版改名：流响应。 + +这是一个开关，打开的时候，当 AI 对话模块运行时，会将其输出的内容返回到浏览器（API响应）； +如果关闭，会强制使用非流模式调用模型，并且 AI 输出的内容不会返回到浏览器，但是生成的内容仍可以通过【AI回复】进行输出。你可以将【AI回复】连接到其他模块中进行二次使用。 ### 最大上下文 @@ -33,13 +36,25 @@ weight: 104 最大回复 token 数量。注意，是回复的Tokens！不是上下文 tokens。 +通常，回复上限=min(模型允许的最大回复上限, 最大上下文-已用上下文) + +所以，一般配置模型时，不会把最大上下文配置成模型实际最大上下文，而是预留预定空间给回答，例如 128k 模型，可以配置 max_context=115000 + ### 系统提示词 被放置在上下文数组的最前面，role 为 system，用于引导模型。 +### 记忆轮数（仅简易模式） + +可以配置模型支持的记忆轮数，如果模型的超出上下文，系统会自动截断，尽可能保证不超模型上下文。 + +所以尽管配置 30 轮对话，实际运行时候，不一定会达到 30 轮。 + ## 引用模板 & 引用提示词 -这两个参数与知识库问答场景相关，可以控制知识库相关的提示词。 +进行知识库搜索后，你可以自定义组织检索结果构成的提示词，这个配置，仅工作流中 AI 对话节点可用。并且，只会在有引用知识库内容时才会生效。 + +![alt text](/imgs/image-54.png) ### AI 对话消息组成 diff --git a/docSite/content/zh-cn/docs/guide/knowledge_base/lark_dataset.md b/docSite/content/zh-cn/docs/guide/knowledge_base/lark_dataset.md new file mode 100644 index 000000000..8c7a8ab55 --- /dev/null +++ b/docSite/content/zh-cn/docs/guide/knowledge_base/lark_dataset.md @@ -0,0 +1,59 @@ +--- +title: '飞书知识库' +description: 'FastGPT 飞书知识库功能介绍和使用方式' +icon: 'language' +draft: false +toc: true +weight: 405 +--- + +| | | +| --- | --- | +| ![alt text](/imgs/image-39.png) | ![alt text](/imgs/image-40.png) | + +FastGPT v4.8.16 版本开始，商业版用户支持飞书知识库导入，用户可以通过配置飞书应用的 appId 和 appSecret，并选中一个**文档空间的顶层文件夹**来导入飞书知识库。目前处于测试阶段，部分交互有待优化。 + +由于飞书限制，无法直接获取所有文档内容，目前仅可以获取共享空间下文件目录的内容，无法获取个人空间和知识库里的内容。 + + +## 1. 创建飞书应用 + +打开 [飞书开放平台](https://open.feishu.cn/?lang=zh-CN)，点击**创建应用**，选择**自建应用**，然后填写应用名称。 + +## 2. 配置应用权限 + +创建应用后，进入应用可以配置相关权限，这里需要增加两个权限： + +1. 获取云空间文件夹下的云文档清单 +2. 查看新版文档 + +![alt text](/imgs/image-41.png) + +## 3. 获取 appId 和 appSecret + +![alt text](/imgs/image-42.png) + +## 4. 给 Folder 增加权限 + +可参考飞书教程： https://open.feishu.cn/document/server-docs/docs/drive-v1/faq#b02e5bfb + +大致总结为： + +1. 把刚刚创建的应用拉入一个群里 +2. 给这个群增加目录权限 + +如果你的目录已经给全员组增加权限了，则可以跳过上面步骤，直接获取 Folder Token。 + +![alt text](/imgs/image-43.png) + +## 5. 获取 Folder Token + +可以页面路径上获取 Folder Token，注意不要把问号复制进来。 + +![alt text](/imgs/image-44.png) + +## 6. 创建知识库 + +根据 3 和 5 获取到的 3 个参数，创建知识库，选择飞书文件库类型，然后填入对应的参数，点击创建。 + +![alt text](/imgs/image-39.png) \ No newline at end of file diff --git a/docSite/content/zh-cn/docs/guide/knowledge_base/yuque_dataset.md b/docSite/content/zh-cn/docs/guide/knowledge_base/yuque_dataset.md new file mode 100644 index 000000000..96b990260 --- /dev/null +++ b/docSite/content/zh-cn/docs/guide/knowledge_base/yuque_dataset.md @@ -0,0 +1,42 @@ +--- +title: '语雀文件库' +description: 'FastGPT 语雀文件库功能介绍和使用方式' +icon: 'language' +draft: false +toc: true +weight: 405 +--- + +| | | +| --- | --- | +| ![alt text](/imgs/image-31.png) | ![alt text](/imgs/image-32.png) | + +FastGPT v4.8.16 版本开始，商业版用户支持语雀文件库导入，用户可以通过配置语雀的 token 和 uid 来导入语雀文档库。目前处于测试阶段，部分交互有待优化。 + +## 1. 获取语雀的 token 和 uid + +在语雀首页 - 个人头像 - 设置，可找到对应参数。 + +![alt text](/imgs/image-36.png) + +参考下图获取 Token 和 User ID，注意给 Token 赋值权限： + +| 获取 Token | 增加权限 | 获取 User ID | +| --- | --- | --- | +| ![alt text](/imgs/image-33.png) | ![alt text](/imgs/image-34.png) | ![alt text](/imgs/image-35.png) | + +## 2. 创建知识库 + +使用上一步获取的 token 和 uid，创建知识库，选择语雀文件库类型，然后填入对应的参数，点击创建。 + +![alt text](/imgs/image-37.png) + +![alt text](/imgs/image-31.png) + +## 3. 导入文档 + +创建完知识库后，点击`添加文件`即可导入语雀的文档库，跟随引导即可。 + +语雀知识库支持定时同步功能，每天会不定时的扫描一次，如果文档有更新，则会进行同步，也可以进行手动同步。 + +![alt text](/imgs/image-38.png) \ No newline at end of file diff --git a/docSite/content/zh-cn/docs/guide/plugins/searxng_plugin_guide.md b/docSite/content/zh-cn/docs/guide/plugins/searxng_plugin_guide.md new file mode 100644 index 000000000..6652d7ff1 --- /dev/null +++ b/docSite/content/zh-cn/docs/guide/plugins/searxng_plugin_guide.md @@ -0,0 +1,190 @@ +--- +title: "SearXNG 搜索插件配置与使用说明" +description: "FastGPT SearXNG 搜索插件配置指南" +icon: "search" +draft: false +toc: true +weight: 303 +--- + +[SearXNG](https://github.com/searxng/searxng)是一款免费的互联网元搜索引擎，它汇总了来自各种搜索服务和数据库的结果。它不会跟踪或分析用户。用户可以自行部署它进行使用。本文介绍 Searxng 的部署以及接入 FastGPT 插件。 + + +## 1. 部署应用 + +这里介绍在 Sealos 中部署 SearXNG 的方法。Docker 部署，可以直接参考 [SearXNG 官方教程](https://github.com/searxng/searxng)。 + +点击打开 [Sealos 北京区](https://bja.sealos.run/)，点击应用部署，并新建一个应用： + +| 打开应用部署 | 点击新建应用 | +| --- | --- | +| ![](/imgs/searxng_plugin_guide1.png) | ![alt text](/imgs/image-45.png) | + +## 2. 部署配置 + +把下面参数，填入配置中： + +* 镜像名: searxng/searxng:latest +* CPU: 0.2 +* 内存: 512M +* 容器暴露端口: 8080 +* 开启公网访问 +* 点击高级配置，填写环境变量和配置文件 + +![alt text](/imgs/image-50.png) + +**环境变量** + +填下面两个内容，主要是为了减小并发，不然内存占用非常大。 + +``` +UWSGI_WORKERS=4 +UWSGI_THREADS=4 +``` + +**配置文件** + +新增一个配置文件，文件名：`/etc/searx/settings.yml` +文件内容： + +```txt +general: + debug: false + instance_name: "searxng" + privacypolicy_url: false + donation_url: false + contact_url: false + enable_metrics: true + open_metrics: '' + +brand: + new_issue_url: https://github.com/searxng/searxng/issues/new + docs_url: https://docs.searxng.org/ + public_instances: https://searx.space + wiki_url: https://github.com/searxng/searxng/wiki + issue_url: https://github.com/searxng/searxng/issues + +search: + safe_search: 0 + autocomplete: "" + autocomplete_min: 4 + default_lang: "auto" + ban_time_on_fail: 5 + max_ban_time_on_fail: 120 + formats: + - html + +server: + port: 8080 + bind_address: "0.0.0.0" + base_url: false + limiter: false + public_instance: false + secret_key: "example" + image_proxy: false + http_protocol_version: "1.0" + method: "POST" + default_http_headers: + X-Content-Type-Options: nosniff + X-Download-Options: noopen + X-Robots-Tag: noindex, nofollow + Referrer-Policy: no-referrer + +redis: + url: false + +ui: + static_path: "" + static_use_hash: false + templates_path: "" + default_theme: simple + default_locale: "" + query_in_title: false + infinite_scroll: false + center_alignment: false + theme_args: + simple_style: auto + +outgoing: + request_timeout: 30.0 + max_request_timeout: 40.0 + pool_connections: 200 + pool_maxsize: 50 + enable_http2: false + retries: 5 + +engines: + + - name: bing + engine: bing + shortcut: bi + +doi_resolvers: + oadoi.org: 'https://oadoi.org/' + doi.org: 'https://doi.org/' + doai.io: 'https://dissem.in/' + sci-hub.se: 'https://sci-hub.se/' + sci-hub.st: 'https://sci-hub.st/' + sci-hub.ru: 'https://sci-hub.ru/' + +default_doi_resolver: 'oadoi.org' +``` + +国内目前只有 Bing 引擎可以正常用，所以上面的配置只配置了 bing 引擎。如果在海外部署，可以使用[Sealos 新加坡可用区](https://cloud.sealos.io/)，并配置其他搜索引擎，可以参考[SearXNG 默认配置文件](https://github.com/searxng/searxng/blob/master/searx/settings.yml), 从里面复制一些 engine 配置。例如： + +``` + - name: duckduckgo + engine: duckduckgo + shortcut: ddg + + - name: google + engine: google + shortcut: go +``` + +## 3. FastGPT 使用 + +复制 Sealos 部署后提供的公网地址，填入 FastGPT 的 SearXNG 插件的 URL 中。 + +| 复制公网地址| 填入 URL | +| --- | --- | +| ![alt text](/imgs/image-48.png) | ![alt text](/imgs/image-49.png) | + +## 返回格式 + +* 成功时返回搜索结果数组： + +```Bash +{ + "result": "[{\"title\":\"标题1\",\"link\":\"链接1\",\"snippet\":\"摘要1\"}, ...]" +} +``` + +* 搜索结果为空时会返回友好提示： + +```Bash +{ + "result": "[]", + "error": { + "message": "No search results", + "code": 500 + } +} +``` + +* 失败时通过 Promise.reject 可能返回错误信息： + +```Bash +- "缺少查询参数" +- "缺少url" +- "Failed to fetch data from Search XNG" +``` + +一般问题来源于参数缺失与服务部署，如有更多问题可在用户群提问。 + +## FAQ + +### 无搜索结果 + +1. 先直接打开外网地址，测试是否可以正常搜索。 +2. 检查是否有超时的搜索引擎，通过 API 调用时不会返回结果。 \ No newline at end of file diff --git a/docSite/content/zh-cn/docs/guide/workbench/intro.md b/docSite/content/zh-cn/docs/guide/workbench/intro.md index e5d8b1c5c..e458512fc 100644 --- a/docSite/content/zh-cn/docs/guide/workbench/intro.md +++ b/docSite/content/zh-cn/docs/guide/workbench/intro.md @@ -45,7 +45,7 @@ FastGPT 从 V4 版本开始采用新的交互方式来构建 AI 应用。使用 ## 重点 - 工作流是如何运行的 -FastGPT的工作流从【流程开始】节点开始执行，可以理解为从用户输入问题开始，没有**固定的出口**，是以节点运行结束作为出口，如果在一个轮调用中，所有节点都不再允许，则工作流结束。 +FastGPT的工作流从【流程开始】节点开始执行，可以理解为从用户输入问题开始，没有**固定的出口**，是以节点运行结束作为出口，如果在一个轮调用中，所有节点都不再运行，则工作流结束。 下面我们来看下，工作流是如何运行的，以及每个节点何时被触发执行。 diff --git a/docSite/content/zh-cn/docs/guide/workbench/workflow/ai_chat.md b/docSite/content/zh-cn/docs/guide/workbench/workflow/ai_chat.md index 8ee015a13..a3968115a 100644 --- a/docSite/content/zh-cn/docs/guide/workbench/workflow/ai_chat.md +++ b/docSite/content/zh-cn/docs/guide/workbench/workflow/ai_chat.md @@ -19,7 +19,7 @@ weight: 232 ## AI模型 -可以通过 [config.json](/docs/development/configuration/) 配置可选的对话模型，通过 [one-api](/docs/development/one-api/) 来实现多模型接入。 +可以通过 [config.json](/docs/development/configuration/) 配置可选的对话模型，通过 [one-api](/docs/development/modelconfig/one-api/) 来实现多模型接入。 点击AI模型后，可以配置模型的相关参数。 diff --git a/docSite/content/zh-cn/docs/guide/workbench/workflow/sandbox.md b/docSite/content/zh-cn/docs/guide/workbench/workflow/sandbox.md index 4a6a7aad7..61b5eb3ea 100644 --- a/docSite/content/zh-cn/docs/guide/workbench/workflow/sandbox.md +++ b/docSite/content/zh-cn/docs/guide/workbench/workflow/sandbox.md @@ -84,4 +84,18 @@ function main({input}){ } ``` -![alt text](/imgs/image-2.png) \ No newline at end of file +![alt text](/imgs/image-2.png) + +### createHmac 加密 + +与 node 中 crypto 的 createHmac 方法一致。 + +```js +function main({secret}){ + const {sign,timestamp} = createHmac('sha256',secret) + + return { + sign,timestamp + } +} +``` \ No newline at end of file diff --git a/docSite/content/zh-cn/docs/guide/workbench/workflow/tool.md b/docSite/content/zh-cn/docs/guide/workbench/workflow/tool.md index 65b7c0f1e..e9419d302 100644 --- a/docSite/content/zh-cn/docs/guide/workbench/workflow/tool.md +++ b/docSite/content/zh-cn/docs/guide/workbench/workflow/tool.md @@ -58,7 +58,7 @@ weight: 236 #### 用途 -默认清空下，工具调用节点，在决定调用工具后，会将工具运行的结果，返回给AI，让 AI 对工具运行的结果进行总结输出。有时候，如果你不需要 AI 进行进一步的总结输出，可以使用该节点，将其接入对于工具流程的末尾。 +默认情况下，工具调用节点，在决定调用工具后，会将工具运行的结果，返回给AI，让 AI 对工具运行的结果进行总结输出。有时候，如果你不需要 AI 进行进一步的总结输出，可以使用该节点，将其接入对于工具流程的末尾。 如下图，在执行知识库搜索后，发送给了 HTTP 请求，搜索将不会返回搜索的结果给工具调用进行 AI 总结。 diff --git a/docSite/content/zh-cn/docs/intro.md b/docSite/content/zh-cn/docs/intro.md index 5f060c939..90a808125 100644 --- a/docSite/content/zh-cn/docs/intro.md +++ b/docSite/content/zh-cn/docs/intro.md @@ -54,7 +54,7 @@ FastGPT 对外的 API 接口对齐了 OpenAI 官方接口，可以直接接入 1. **项目开源** - FastGPT 遵循附加条件 Apache License 2.0 开源协议，你可以 [Fork](https://github.com/labring/FastGPT/fork) 之后进行二次开发和发布。FastGPT 社区版将保留核心功能，商业版仅在社区版基础上使用 API 的形式进行扩展，不影响学习使用。 + FastGPT 遵循**附加条件 Apache License 2.0 开源协议**，你可以 [Fork](https://github.com/labring/FastGPT/fork) 之后进行二次开发和发布。FastGPT 社区版将保留核心功能，商业版仅在社区版基础上使用 API 的形式进行扩展，不影响学习使用。 2. **独特的 QA 结构** diff --git a/docSite/content/zh-cn/docs/shopping_cart/intro.md b/docSite/content/zh-cn/docs/shopping_cart/intro.md index 117144827..2483fdb5e 100644 --- a/docSite/content/zh-cn/docs/shopping_cart/intro.md +++ b/docSite/content/zh-cn/docs/shopping_cart/intro.md @@ -19,17 +19,20 @@ FastGPT 商业版是基于 FastGPT 开源版的增强版本，增加了一些独 | 应用管理与高级编排 | ✅ | ✅ | ✅ | | 文档知识库 | ✅ | ✅ | ✅ | | 外部使用 | ✅ | ✅ | ✅ | +| API 知识库 | ✅ | ✅ | ✅ | | 最大应用数量 | 500 | 无限制 | 由付费套餐决定 | | 最大知识库数量（单个知识库内容无限制） | 30 | 无限制 | 由付费套餐决定 | | 自定义版权信息 | ❌ | ✅ | 设计中 | | 多租户与支付 | ❌ | ✅ | ✅ | -| 团队空间 | ❌ | ✅ | ✅ | +| 团队空间 & 权限 | ❌ | ✅ | ✅ | | 应用发布安全配置 | ❌ | ✅ | ✅ | | 内容审核 | ❌ | ✅ | ✅ | | web站点同步 | ❌ | ✅ | ✅ | -| 管理后台 | ❌ | ✅ | 不需要 | +| 主流文档库接入（目前支持：语雀、飞书） | ❌ | ✅ | ✅ | | 增强训练模式 | ❌ | ✅ | ✅ | | 第三方应用快速接入（飞书、公众号） | ❌ | ✅ | ✅ | +| 管理后台 | ❌ | ✅ | 不需要 | +| SSO 登录（可自定义，也可使用内置：Github、公众号、钉钉、谷歌等） | ❌ | ✅ | 不需要 | | 图片知识库 | ❌ | 设计中 | 设计中 | | 对话日志运营分析 | ❌ | 设计中 | 设计中 | | 完整商业授权 | ❌ | ✅ | ✅ | @@ -50,8 +53,9 @@ FastGPT 商业版软件根据不同的部署方式，分为 3 类收费模式。 {{< table "table-hover table-striped-columns" >}} | 部署方式 | 特有服务 | 上线时长 | 标品价格 | | ---- | ---- | ---- | ---- | -| Sealos全托管 | 1. 有效期内免费升级。
2. 免运维服务&数据库。 | 半天 | 5000元起/月（3个月起）
或
50000元起/年 | -| 自有服务器部署 | 1. 6个版本的升级服务。 | 14天内 | 具体价格可[联系咨询](https://fael3z0zfze.feishu.cn/share/base/form/shrcnRxj3utrzjywsom96Px4sud) | +| Sealos全托管 | 1. 有效期内免费升级。
2. 免运维服务&数据库。 | 半天 | 10000元起/月（3个月起）
或
120000元起/年
8C32G 资源，额外资源另外收费。 | +| Sealos全托管（多节点） | 1. 有效期内免费升级。
2. 免运维服务&数据库。 | 半天 | 22000元起/月（3个月起）
或
264000元起/年
32C128G 资源，额外资源另外收费。 | +| 自有服务器部署 | 1. 6个版本免费升级支持。 | 14天内 | 具体价格和优惠可[联系咨询](https://fael3z0zfze.feishu.cn/share/base/form/shrcnRxj3utrzjywsom96Px4sud) | {{< /table >}} {{% alert icon="🤖 " context="success" %}} @@ -62,6 +66,10 @@ FastGPT 商业版软件根据不同的部署方式，分为 3 类收费模式。 - 高可用版适合对外提供在线服务，包含可视化监控、多副本、负载均衡、数据库自动备份等生产环境的基础设施。 {{% /alert %}} +## 联系方式 + +请填写[咨询问卷](https://fael3z0zfze.feishu.cn/share/base/form/shrcnRxj3utrzjywsom96Px4sud)，我们会尽快与您联系。 + ## 技术支持 @@ -79,9 +87,6 @@ FastGPT 商业版软件根据不同的部署方式，分为 3 类收费模式。 跨版本更新或复杂更新可参考文档自行更新；或付费支持，标准与技术服务费一致。 -## 联系方式 - -请填写[咨询问卷](https://fael3z0zfze.feishu.cn/share/base/form/shrcnRxj3utrzjywsom96Px4sud)，我们会尽快与您联系。 ## QA @@ -95,8 +100,14 @@ FastGPT 商业版软件根据不同的部署方式，分为 3 类收费模式。 可以修改开源版部分代码，不支持修改商业版镜像。完整版本=开源版+商业版镜像，所以是可以修改部分内容的。但是如果二开了，后续则需要自己进行代码合并升级。 -## Sealos 费用 +### Sealos 运行费用 Sealos 云服务属于按量计费，下面是它的价格表： -![](/imgs/sealos_price.jpg) +![alt text](/imgs/image-58.png) + +## 管理后台部分截图 + +| | | | +| ---- | ---- | ---- | +| ![alt text](/imgs/image-55.png) | ![alt text](/imgs/image-56.png) | ![alt text](/imgs/image-57.png) | \ No newline at end of file diff --git a/docSite/content/zh-cn/docs/shopping_cart/saas.md b/docSite/content/zh-cn/docs/shopping_cart/saas.md index f3c1d29cc..b25199b1c 100644 --- a/docSite/content/zh-cn/docs/shopping_cart/saas.md +++ b/docSite/content/zh-cn/docs/shopping_cart/saas.md @@ -5,8 +5,9 @@ icon: 'currency_yen' draft: false toc: true weight: 1102 -type: redirect -target: https://cloud.tryfastgpt.ai/price --- -线上版价格请查看：[https://cloud.tryfastgpt.ai/price](https://cloud.tryfastgpt.ai/price) \ No newline at end of file +线上版价格按套餐订阅模式，具体价格和计费请查看（请正确选择版本，账号不互通）： + +- [海外版](https://cloud.tryfastgpt.ai/price) +- [国内版](https://cloud.fastgpt.cn/price) \ No newline at end of file diff --git a/docSite/content/zh-cn/docs/use-cases/app-cases/submit_application_template.md b/docSite/content/zh-cn/docs/use-cases/app-cases/submit_application_template.md index 1df015e97..2ef61eaf9 100644 --- a/docSite/content/zh-cn/docs/use-cases/app-cases/submit_application_template.md +++ b/docSite/content/zh-cn/docs/use-cases/app-cases/submit_application_template.md @@ -32,11 +32,11 @@ weight: 602 1. ### 创建应用模板 -应用模板配置以及相关资源，都会在 **projects/app/public/appMarketTemplates** 目录下。 +应用模板配置以及相关资源，都会在 **packages/templates/src** 目录下。 ![](/imgs/template_submission2.png) -1. 在 **projects/app/public/appMarketTemplates** 目录下，创建一个文件夹，名称为模板对应的 id。 +1. 在**packages/templates/src** 目录下，创建一个文件夹，名称为模板对应的 id。 2. 在刚刚创建的文件夹中，再创建一个 **template.json** 文件，复制粘贴并填写如下配置： ```JSON @@ -83,4 +83,4 @@ weight: 602 - 写清楚模板的介绍和功能 - 配上模板运行的效果图 -- 模板参数填写说明，需要在 PR 中写清楚。例如，有些模板需要去某个提供商申请 key，需要附上对应的地址和教程，后续我们会加入到文档中。 \ No newline at end of file +- 模板参数填写说明，需要在 PR 中写清楚。例如，有些模板需要去某个提供商申请 key，需要附上对应的地址和教程，后续我们会加入到文档中。 diff --git a/docSite/content/zh-cn/docs/use-cases/external-integration/dingtalk.md b/docSite/content/zh-cn/docs/use-cases/external-integration/dingtalk.md new file mode 100644 index 000000000..2bdad3503 --- /dev/null +++ b/docSite/content/zh-cn/docs/use-cases/external-integration/dingtalk.md @@ -0,0 +1,57 @@ +--- +title: "接入钉钉机器人教程" +description: "FastGPT 接入钉钉机器人教程" +icon: "chat" +draft: false +toc: true +weight: 505 +--- + +从 4.8.16 版本起，FastGPT 商业版支持直接接入钉钉机器人，无需额外的 API。 + +## 1. 创建钉钉企业内部应用 + +1. 在[钉钉开发者后台](https://open-dev.dingtalk.com/fe/app)创建企业内部应用。 + +![图片1](/imgs/dingtalk-bot-1.png) + +2. 获取**Client ID**和**Client Secret**。 + +![图片2](/imgs/dingtalk-bot-2.png) + +## 2. 为 FastGPT 添加发布渠道 + +在 FastGPT 中选择要接入的应用，在**发布渠道**页面，新建一个接入钉钉机器人的发布渠道。 + +将前面拿到的 **Client ID** 和 **Client Secret** 填入配置弹窗中。 + +![图片3](/imgs/dingtalk-bot-3.png) + +创建完成后，点击**请求地址**按钮，然后复制回调地址。 + +## 3. 为应用添加**机器人**应用能力。 + +在钉钉开发者后台，点击左侧**添加应用能力**，为刚刚创建的企业内部应用添加 **机器人** 应用能力。 + +![图片4](/imgs/dingtalk-bot-4.png) + +## 4. 配置机器人回调地址 + +点击左侧**机器人** 应用能力，然后将底部**消息接受模式**设置为**HTTP模式**，消息接收地址填入前面复制的 FastGPT 的回调地址。 + +![图片5](/imgs/dingtalk-bot-5.png) + +调试完成后，点击**发布**。 + +## 5. 发布应用 +机器人发布后，还需要在**版本管理与发布**页面发布应用版本。 + +![图片6](/imgs/dingtalk-bot-6.png) + +点击**创建新版本**后，设置版本号和版本描述后点击保存发布即可。 + +![图片7](/imgs/dingtalk-bot-7.png) + +应用发布后，即可在钉钉企业中使用机器人功能，可对机器人私聊。或者在群组添加机器人后`@机器人`，触发对话。 + +![图片8](/imgs/dingtalk-bot-8.png) diff --git a/docSite/content/zh-cn/docs/use-cases/external-integration/official_account.md b/docSite/content/zh-cn/docs/use-cases/external-integration/official_account.md index 0f87d3265..c91a2c58c 100644 --- a/docSite/content/zh-cn/docs/use-cases/external-integration/official_account.md +++ b/docSite/content/zh-cn/docs/use-cases/external-integration/official_account.md @@ -17,9 +17,11 @@ weight: 506 ![图片](/imgs/offiaccount-1.png) -## 2. 登录微信公众平台，获取 AppID 、 Secret和Token +## 2. 获取 AppID 、 Secret和Token -### 1. https://mp.weixin.qq.com 登录微信公众平台，选择您的公众号。 +### 1. 登录微信公众平台，选择您的公众号。 + +打开微信公众号官网：https://mp.weixin.qq.com **只支持通过验证的公众号，未通过验证的公众号暂不支持。** @@ -28,6 +30,7 @@ weight: 506 ![图片](/imgs/offiaccount-2.png) ### 2. 把3个参数填入 FastGPT 配置弹窗中。 + ![图片](/imgs/offiaccount-3.png) ## 3. 在 IP 白名单中加入 FastGPT 的 IP @@ -36,7 +39,7 @@ weight: 506 私有部署的用户可自行查阅自己的 IP 地址。 -海外版用户（cloud.tryfastgpt.ai)可以填写下面的 IP 白名单： +海外版用户（cloud.tryfastgpt.ai）可以填写下面的 IP 白名单： ``` 35.240.227.100 diff --git a/files/docker/docker-compose-milvus.yml b/files/docker/docker-compose-milvus.yml index fb3ab3c2b..e6d1ce279 100644 --- a/files/docker/docker-compose-milvus.yml +++ b/files/docker/docker-compose-milvus.yml @@ -114,15 +114,15 @@ services: # fastgpt sandbox: container_name: sandbox - image: ghcr.io/labring/fastgpt-sandbox:v4.8.15 # git - # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-sandbox:v4.8.15 # 阿里云 + image: ghcr.io/labring/fastgpt-sandbox:v4.8.17 # git + # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-sandbox:v4.8.17 # 阿里云 networks: - fastgpt restart: always fastgpt: container_name: fastgpt - image: ghcr.io/labring/fastgpt:v4.8.15 # git - # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt:v4.8.15 # 阿里云 + image: ghcr.io/labring/fastgpt:v4.8.17 # git + # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt:v4.8.17 # 阿里云 ports: - 3000:3000 networks: @@ -159,6 +159,14 @@ services: # 日志等级: debug, info, warn, error - LOG_LEVEL=info - STORE_LOG_LEVEL=warn + # 工作流最大运行次数 + - WORKFLOW_MAX_RUN_TIMES=1000 + # 批量执行节点，最大输入长度 + - WORKFLOW_MAX_LOOP_TIMES=100 + # 自定义跨域，不配置时，默认都允许跨域（多个域名通过逗号分割） + - ALLOWED_ORIGINS= + # 是否开启IP限制，默认不开启 + - USE_IP_LIMIT=false volumes: - ./config.json:/app/data/config.json diff --git a/files/docker/docker-compose-pgvector.yml b/files/docker/docker-compose-pgvector.yml index 3ebecfda3..e8da9abe8 100644 --- a/files/docker/docker-compose-pgvector.yml +++ b/files/docker/docker-compose-pgvector.yml @@ -72,15 +72,15 @@ services: # fastgpt sandbox: container_name: sandbox - image: ghcr.io/labring/fastgpt-sandbox:v4.8.15 # git - # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-sandbox:v4.8.15 # 阿里云 + image: ghcr.io/labring/fastgpt-sandbox:v4.8.17 # git + # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-sandbox:v4.8.17 # 阿里云 networks: - fastgpt restart: always fastgpt: container_name: fastgpt - image: ghcr.io/labring/fastgpt:v4.8.15 # git - # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt:v4.8.15 # 阿里云 + image: ghcr.io/labring/fastgpt:v4.8.17 # git + # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt:v4.8.17 # 阿里云 ports: - 3000:3000 networks: @@ -116,6 +116,14 @@ services: # 日志等级: debug, info, warn, error - LOG_LEVEL=info - STORE_LOG_LEVEL=warn + # 工作流最大运行次数 + - WORKFLOW_MAX_RUN_TIMES=1000 + # 批量执行节点，最大输入长度 + - WORKFLOW_MAX_LOOP_TIMES=100 + # 自定义跨域，不配置时，默认都允许跨域（多个域名通过逗号分割） + - ALLOWED_ORIGINS= + # 是否开启IP限制，默认不开启 + - USE_IP_LIMIT=false volumes: - ./config.json:/app/data/config.json diff --git a/files/docker/docker-compose-zilliz.yml b/files/docker/docker-compose-zilliz.yml index 49f2385c7..6b53232e7 100644 --- a/files/docker/docker-compose-zilliz.yml +++ b/files/docker/docker-compose-zilliz.yml @@ -53,15 +53,15 @@ services: wait $$! sandbox: container_name: sandbox - image: ghcr.io/labring/fastgpt-sandbox:v4.8.15 # git - # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-sandbox:v4.8.15 # 阿里云 + image: ghcr.io/labring/fastgpt-sandbox:v4.8.17 # git + # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-sandbox:v4.8.17 # 阿里云 networks: - fastgpt restart: always fastgpt: container_name: fastgpt - image: ghcr.io/labring/fastgpt:v4.8.15 # git - # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt:v4.8.15 # 阿里云 + image: ghcr.io/labring/fastgpt:v4.8.17 # git + # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt:v4.8.17 # 阿里云 ports: - 3000:3000 networks: @@ -97,6 +97,14 @@ services: # 日志等级: debug, info, warn, error - LOG_LEVEL=info - STORE_LOG_LEVEL=warn + # 工作流最大运行次数 + - WORKFLOW_MAX_RUN_TIMES=1000 + # 批量执行节点，最大输入长度 + - WORKFLOW_MAX_LOOP_TIMES=100 + # 自定义跨域，不配置时，默认都允许跨域（多个域名通过逗号分割） + - ALLOWED_ORIGINS= + # 是否开启IP限制，默认不开启 + - USE_IP_LIMIT=false volumes: - ./config.json:/app/data/config.json diff --git a/packages/global/common/error/code/common.ts b/packages/global/common/error/code/common.ts index 13f74643b..38d76dbec 100644 --- a/packages/global/common/error/code/common.ts +++ b/packages/global/common/error/code/common.ts @@ -1,14 +1,20 @@ +import { i18nT } from '../../../../web/i18n/utils'; import { ErrType } from '../errorCode'; /* dataset: 507000 */ const startCode = 507000; export enum CommonErrEnum { + invalidParams = 'invalidParams', fileNotFound = 'fileNotFound', unAuthFile = 'unAuthFile', missingParams = 'missingParams', inheritPermissionError = 'inheritPermissionError' } const datasetErr = [ + { + statusText: CommonErrEnum.fileNotFound, + message: i18nT('common:error.invalid_params') + }, { statusText: CommonErrEnum.fileNotFound, message: 'error.fileNotFound' diff --git a/packages/global/common/error/code/dataset.ts b/packages/global/common/error/code/dataset.ts index e5b8abd83..cca81331f 100644 --- a/packages/global/common/error/code/dataset.ts +++ b/packages/global/common/error/code/dataset.ts @@ -12,7 +12,8 @@ export enum DatasetErrEnum { unLinkCollection = 'unLinkCollection', invalidVectorModelOrQAModel = 'invalidVectorModelOrQAModel', notSupportSync = 'notSupportSync', - sameApiCollection = 'sameApiCollection' + sameApiCollection = 'sameApiCollection', + noApiServer = 'noApiServer' } const datasetErr = [ { diff --git a/packages/global/common/error/code/team.ts b/packages/global/common/error/code/team.ts index 8ad9abe02..7c2ab6790 100644 --- a/packages/global/common/error/code/team.ts +++ b/packages/global/common/error/code/team.ts @@ -1,9 +1,11 @@ -import { ErrType } from '../errorCode'; import { i18nT } from '../../../../web/i18n/utils'; +import type { ErrType } from '../errorCode'; /* team: 500000 */ export enum TeamErrEnum { + notUser = 'notUser', teamOverSize = 'teamOverSize', unAuthTeam = 'unAuthTeam', + teamMemberOverSize = 'teamMemberOverSize', aiPointsNotEnough = 'aiPointsNotEnough', datasetSizeNotEnough = 'datasetSizeNotEnough', datasetAmountNotEnough = 'datasetAmountNotEnough', @@ -14,11 +16,22 @@ export enum TeamErrEnum { groupNameEmpty = 'groupNameEmpty', groupNameDuplicate = 'groupNameDuplicate', groupNotExist = 'groupNotExist', + orgMemberNotExist = 'orgMemberNotExist', + orgMemberDuplicated = 'orgMemberDuplicated', + orgNotExist = 'orgNotExist', + orgParentNotExist = 'orgParentNotExist', + cannotMoveToSubPath = 'cannotMoveToSubPath', + cannotModifyRootOrg = 'cannotModifyRootOrg', + cannotDeleteNonEmptyOrg = 'cannotDeleteNonEmptyOrg', cannotDeleteDefaultGroup = 'cannotDeleteDefaultGroup', userNotActive = 'userNotActive' } const teamErr = [ + { + statusText: TeamErrEnum.notUser, + message: i18nT('common:code_error.team_error.not_user') + }, { statusText: TeamErrEnum.teamOverSize, message: i18nT('common:code_error.team_error.over_size') @@ -71,6 +84,34 @@ const teamErr = [ { statusText: TeamErrEnum.userNotActive, message: i18nT('common:code_error.team_error.user_not_active') + }, + { + statusText: TeamErrEnum.orgMemberNotExist, + message: i18nT('common:code_error.team_error.org_member_not_exist') + }, + { + statusText: TeamErrEnum.orgMemberDuplicated, + message: i18nT('common:code_error.team_error.org_member_duplicated') + }, + { + statusText: TeamErrEnum.orgNotExist, + message: i18nT('common:code_error.team_error.org_not_exist') + }, + { + statusText: TeamErrEnum.orgParentNotExist, + message: i18nT('common:code_error.team_error.org_parent_not_exist') + }, + { + statusText: TeamErrEnum.cannotMoveToSubPath, + message: i18nT('common:code_error.team_error.cannot_move_to_sub_path') + }, + { + statusText: TeamErrEnum.cannotModifyRootOrg, + message: i18nT('common:code_error.team_error.cannot_modify_root_org') + }, + { + statusText: TeamErrEnum.cannotDeleteNonEmptyOrg, + message: i18nT('common:code_error.team_error.cannot_delete_non_empty_org') } ]; diff --git a/packages/global/common/error/code/user.ts b/packages/global/common/error/code/user.ts index e58136041..98d8e9a0f 100644 --- a/packages/global/common/error/code/user.ts +++ b/packages/global/common/error/code/user.ts @@ -2,25 +2,16 @@ import { ErrType } from '../errorCode'; import { i18nT } from '../../../../web/i18n/utils'; /* team: 503000 */ export enum UserErrEnum { - unAuthUser = 'unAuthUser', unAuthRole = 'unAuthRole', - binVisitor = 'binVisitor', + account_psw_error = 'account_psw_error', balanceNotEnough = 'balanceNotEnough', unAuthSso = 'unAuthSso' } const errList = [ { - statusText: UserErrEnum.unAuthUser, - message: i18nT('common:code_error.user_error.un_auth_user') + statusText: UserErrEnum.account_psw_error, + message: i18nT('common:code_error.account_error') }, - { - statusText: UserErrEnum.binVisitor, - message: i18nT('common:code_error.user_error.bin_visitor') - }, // 身份校验未通过 - { - statusText: UserErrEnum.binVisitor, - message: i18nT('common:code_error.user_error.bin_visitor_guest') - }, // 游客身份 { statusText: UserErrEnum.balanceNotEnough, message: i18nT('common:code_error.user_error.balance_not_enough') diff --git a/packages/global/common/error/errorCode.ts b/packages/global/common/error/errorCode.ts index a11bc974b..478694742 100644 --- a/packages/global/common/error/errorCode.ts +++ b/packages/global/common/error/errorCode.ts @@ -72,7 +72,7 @@ export const ERROR_RESPONSE: Record< [ERROR_ENUM.tooManyRequest]: { code: 429, statusText: ERROR_ENUM.tooManyRequest, - message: 'Too many request', + message: i18nT('common:error.too_many_request'), data: null }, [ERROR_ENUM.insufficientQuota]: { diff --git a/packages/global/common/error/utils.ts b/packages/global/common/error/utils.ts index 54a47ed01..88a9ea9da 100644 --- a/packages/global/common/error/utils.ts +++ b/packages/global/common/error/utils.ts @@ -5,6 +5,6 @@ export const getErrText = (err: any, def = ''): any => { typeof err === 'string' ? err : err?.response?.data?.message || err?.response?.message || err?.message || def; - msg && console.log('error =>', msg); + // msg && console.log('error =>', msg); return replaceSensitiveText(msg); }; diff --git a/packages/global/common/file/api.d.ts b/packages/global/common/file/api.d.ts index e26033ae7..5da698b58 100644 --- a/packages/global/common/file/api.d.ts +++ b/packages/global/common/file/api.d.ts @@ -1,10 +1,7 @@ -import { MongoImageTypeEnum } from './image/constants'; import { OutLinkChatAuthProps } from '../../support/permission/chat.d'; export type preUploadImgProps = OutLinkChatAuthProps & { - type: `${MongoImageTypeEnum}`; - - expiredTime?: Date; + // expiredTime?: Date; metadata?: Record; }; export type UploadImgProps = preUploadImgProps & { diff --git a/packages/global/common/file/image/constants.ts b/packages/global/common/file/image/constants.ts index 6a178e2af..5e511e4b2 100644 --- a/packages/global/common/file/image/constants.ts +++ b/packages/global/common/file/image/constants.ts @@ -1,61 +1,5 @@ export const imageBaseUrl = '/api/system/img/'; -export enum MongoImageTypeEnum { - systemAvatar = 'systemAvatar', - appAvatar = 'appAvatar', - pluginAvatar = 'pluginAvatar', - datasetAvatar = 'datasetAvatar', - userAvatar = 'userAvatar', - teamAvatar = 'teamAvatar', - groupAvatar = 'groupAvatar', - - chatImage = 'chatImage', - collectionImage = 'collectionImage' -} -export const mongoImageTypeMap = { - [MongoImageTypeEnum.systemAvatar]: { - label: 'appAvatar', - unique: true - }, - [MongoImageTypeEnum.appAvatar]: { - label: 'appAvatar', - unique: true - }, - [MongoImageTypeEnum.pluginAvatar]: { - label: 'pluginAvatar', - unique: true - }, - [MongoImageTypeEnum.datasetAvatar]: { - label: 'datasetAvatar', - unique: true - }, - [MongoImageTypeEnum.userAvatar]: { - label: 'userAvatar', - unique: true - }, - [MongoImageTypeEnum.teamAvatar]: { - label: 'teamAvatar', - unique: true - }, - [MongoImageTypeEnum.groupAvatar]: { - label: 'groupAvatar', - unique: true - }, - - [MongoImageTypeEnum.chatImage]: { - label: 'chatImage', - unique: false - }, - [MongoImageTypeEnum.collectionImage]: { - label: 'collectionImage', - unique: false - } -}; - -export const uniqueImageTypeList = Object.entries(mongoImageTypeMap) - .filter(([key, value]) => value.unique) - .map(([key]) => key as `${MongoImageTypeEnum}`); - export const FolderIcon = 'file/fill/folder'; export const FolderImgUrl = '/imgs/files/folder.svg'; export const HttpPluginImgUrl = '/imgs/app/httpPluginFill.svg'; diff --git a/packages/global/common/file/image/type.d.ts b/packages/global/common/file/image/type.d.ts index 6fe274794..320df97fd 100644 --- a/packages/global/common/file/image/type.d.ts +++ b/packages/global/common/file/image/type.d.ts @@ -1,12 +1,8 @@ -import { MongoImageTypeEnum } from './constants'; - export type MongoImageSchemaType = { _id: string; teamId: string; binary: Buffer; - createTime: Date; expiredTime?: Date; - type: `${MongoImageTypeEnum}`; metadata?: { mime?: string; // image mime type. diff --git a/packages/global/common/file/tools.ts b/packages/global/common/file/tools.ts index df0de53c0..4f75f4e5d 100644 --- a/packages/global/common/file/tools.ts +++ b/packages/global/common/file/tools.ts @@ -2,6 +2,7 @@ import { detect } from 'jschardet'; import { documentFileType, imageFileType } from './constants'; import { ChatFileTypeEnum } from '../../core/chat/constants'; import { UserChatItemValueItemType } from '../../core/chat/type'; +import * as fs from 'fs'; export const formatFileSize = (bytes: number): string => { if (bytes === 0) return '0 B'; @@ -16,6 +17,22 @@ export const formatFileSize = (bytes: number): string => { export const detectFileEncoding = (buffer: Buffer) => { return detect(buffer.slice(0, 200))?.encoding?.toLocaleLowerCase(); }; +export const detectFileEncodingByPath = async (path: string) => { + // Get 64KB file head + const MAX_BYTES = 64 * 1024; + const buffer = Buffer.alloc(MAX_BYTES); + + const fd = await fs.promises.open(path, 'r'); + try { + // Read file head + const { bytesRead } = await fd.read(buffer, 0, MAX_BYTES, 0); + const actualBuffer = buffer.slice(0, bytesRead); + + return detect(actualBuffer)?.encoding?.toLocaleLowerCase(); + } finally { + await fd.close(); + } +}; // Url => user upload file type export const parseUrlToFileType = (url: string): UserChatItemValueItemType['file'] | undefined => { diff --git a/packages/global/common/middle/tracks/constants.ts b/packages/global/common/middle/tracks/constants.ts new file mode 100644 index 000000000..f8473fa05 --- /dev/null +++ b/packages/global/common/middle/tracks/constants.ts @@ -0,0 +1,7 @@ +export enum TrackEnum { + login = 'login', + createApp = 'createApp', + useAppTemplate = 'useAppTemplate', + createDataset = 'createDataset', + appNodes = 'appNodes' +} diff --git a/packages/global/common/middle/tracks/type.d.ts b/packages/global/common/middle/tracks/type.d.ts new file mode 100644 index 000000000..16d6be3c3 --- /dev/null +++ b/packages/global/common/middle/tracks/type.d.ts @@ -0,0 +1,18 @@ +import { TrackEnum } from './constants'; +import { OAuthEnum } from '../../../support/user/constant'; +import { AppTypeEnum } from '../../../core/app/constants'; + +export type PushTrackCommonType = { + uid: string; + teamId: string; + tmbId: string; +}; + +export type TrackSchemaType = { + event: TrackEnum; + createTime: Date; + uid?: string; + teamId?: string; + tmbId?: string; + data: Record; +}; diff --git a/packages/global/common/string/http.ts b/packages/global/common/string/http.ts new file mode 100644 index 000000000..d0636e0fb --- /dev/null +++ b/packages/global/common/string/http.ts @@ -0,0 +1,44 @@ +import parse from '@bany/curl-to-json'; + +type RequestMethod = 'get' | 'post' | 'put' | 'delete' | 'patch'; +const methodMap: { [K in RequestMethod]: string } = { + get: 'GET', + post: 'POST', + put: 'PUT', + delete: 'DELETE', + patch: 'PATCH' +}; + +export const parseCurl = (curlContent: string) => { + const parsed = parse(curlContent); + + if (!parsed.url) { + throw new Error('url not found'); + } + + const newParams = Object.keys(parsed.params || {}).map((key) => ({ + key, + value: parsed.params?.[key], + type: 'string' + })); + const newHeaders = Object.keys(parsed.header || {}).map((key) => ({ + key, + value: parsed.header?.[key], + type: 'string' + })); + const newBody = JSON.stringify(parsed.data, null, 2); + const bodyArray = Object.keys(parsed.data || {}).map((key) => ({ + key, + value: parsed.data?.[key], + type: 'string' + })); + + return { + url: parsed.url, + method: methodMap[parsed.method?.toLowerCase() as RequestMethod] || 'GET', + params: newParams, + headers: newHeaders, + body: newBody, + bodyArray + }; +}; diff --git a/packages/global/common/string/tools.ts b/packages/global/common/string/tools.ts index bc324c804..c44e145b1 100644 --- a/packages/global/common/string/tools.ts +++ b/packages/global/common/string/tools.ts @@ -25,17 +25,22 @@ export const simpleText = (text = '') => { return text; }; -/* - replace {{variable}} to value -*/ +export const valToStr = (val: any) => { + if (val === undefined) return 'undefined'; + if (val === null) return 'null'; + + if (typeof val === 'object') return JSON.stringify(val); + return String(val); +}; + +// replace {{variable}} to value export function replaceVariable(text: any, obj: Record) { - if (!(typeof text === 'string')) return text; + if (typeof text !== 'string') return text; for (const key in obj) { const val = obj[key]; - const formatVal = typeof val === 'object' ? JSON.stringify(val) : String(val); - - text = text.replace(new RegExp(`{{(${key})}}`, 'g'), formatVal); + const formatVal = valToStr(val); + text = text.replace(new RegExp(`{{(${key})}}`, 'g'), () => formatVal); } return text || ''; } diff --git a/packages/global/common/system/constants.ts b/packages/global/common/system/constants.ts index 5c6b192d6..4dcc4d276 100644 --- a/packages/global/common/system/constants.ts +++ b/packages/global/common/system/constants.ts @@ -1,4 +1,9 @@ export const HUMAN_ICON = `/icon/human.svg`; export const LOGO_ICON = `/icon/logo.svg`; export const HUGGING_FACE_ICON = `/imgs/model/huggingface.svg`; + export const DEFAULT_TEAM_AVATAR = `/imgs/avatar/defaultTeamAvatar.svg`; +export const DEFAULT_ORG_AVATAR = '/imgs/avatar/defaultOrgAvatar.svg'; +export const DEFAULT_USER_AVATAR = '/imgs/avatar/BlueAvatar.svg'; + +export const isProduction = process.env.NODE_ENV === 'production'; diff --git a/packages/global/common/system/types/index.d.ts b/packages/global/common/system/types/index.d.ts index c3e4c8052..90bcb7dfd 100644 --- a/packages/global/common/system/types/index.d.ts +++ b/packages/global/common/system/types/index.d.ts @@ -5,7 +5,7 @@ import type { LLMModelItemType, VectorModelItemType, AudioSpeechModels, - WhisperModelType, + STTModelType, ReRankModelItemType } from '../../../core/ai/model.d'; import { SubTypeEnum } from '../../../support/wallet/sub/constants'; @@ -18,6 +18,14 @@ export type NavbarItemType = { isActive: boolean; }; +export type ExternalProviderWorkflowVarType = { + name: string; + key: string; + intro: string; + isOpen: boolean; + url?: string; +}; + /* fastgpt main */ export type FastGPTConfigFileType = { feConfigs: FastGPTFeConfigsType; @@ -27,7 +35,7 @@ export type FastGPTConfigFileType = { vectorModels: VectorModelItemType[]; reRankModels: ReRankModelItemType[]; audioSpeechModels: AudioSpeechModelType[]; - whisperModel: WhisperModelType; + whisperModel: STTModelType; }; export type FastGPTFeConfigsType = { @@ -58,11 +66,18 @@ export type FastGPTFeConfigsType = { icon?: string; title?: string; url?: string; + autoLogin?: boolean; }; oauth?: { github?: string; google?: string; wechat?: string; + dingtalk?: string; + wecom?: { + corpid?: string; + agentid?: string; + secret?: string; + }; microsoft?: { clientId?: string; tenantId?: string; @@ -82,6 +97,7 @@ export type FastGPTFeConfigsType = { uploadFileMaxSize?: number; lafEnv?: string; navbarItems?: NavbarItemType[]; + externalProviderWorkflowVariables?: ExternalProviderWorkflowVarType[]; }; export type SystemEnvType = { diff --git a/packages/global/core/ai/model.d.ts b/packages/global/core/ai/model.d.ts index 1d31fa83c..533ced31d 100644 --- a/packages/global/core/ai/model.d.ts +++ b/packages/global/core/ai/model.d.ts @@ -1,14 +1,22 @@ -export type LLMModelItemType = { +import type { ModelProviderIdType } from './provider'; + +type PriceType = { + charsPointsPrice?: number; // 1k chars=n points; 60s=n points; + + // If inputPrice is set, the input-output charging scheme is adopted + inputPrice?: number; // 1k tokens=n points + outputPrice?: number; // 1k tokens=n points +}; +export type LLMModelItemType = PriceType & { + provider: ModelProviderIdType; model: string; name: string; - avatar?: string; + avatar?: string; // model icon, from provider maxContext: number; maxResponse: number; quoteMaxToken: number; maxTemperature: number; - charsPointsPrice: number; // 1k chars=n points - censor?: boolean; vision?: boolean; @@ -30,12 +38,12 @@ export type LLMModelItemType = { fieldMap?: Record; }; -export type VectorModelItemType = { +export type VectorModelItemType = PriceType & { + provider: ModelProviderIdType; model: string; // model name name: string; // show name avatar?: string; defaultToken: number; // split text default token - charsPointsPrice: number; // 1k tokens=n points maxToken: number; // model max token weight: number; // training weight hidden?: boolean; // Disallow creation @@ -44,23 +52,23 @@ export type VectorModelItemType = { queryConfig?: Record; // Custom parameters for query }; -export type ReRankModelItemType = { +export type ReRankModelItemType = PriceType & { + provider: ModelProviderIdType; model: string; name: string; - charsPointsPrice: number; requestUrl: string; requestAuth: string; }; -export type AudioSpeechModelType = { +export type AudioSpeechModelType = PriceType & { + provider: ModelProviderIdType; model: string; name: string; - charsPointsPrice: number; voices: { label: string; value: string; bufferId: string }[]; }; -export type WhisperModelType = { +export type STTModelType = PriceType & { + provider: ModelProviderIdType; model: string; name: string; - charsPointsPrice: number; // 60s = n points }; diff --git a/packages/global/core/ai/model.ts b/packages/global/core/ai/model.ts index f40e31354..4a40c83c5 100644 --- a/packages/global/core/ai/model.ts +++ b/packages/global/core/ai/model.ts @@ -1,7 +1,10 @@ -import type { LLMModelItemType, VectorModelItemType } from './model.d'; +import { i18nT } from '../../../web/i18n/utils'; +import type { LLMModelItemType, STTModelType, VectorModelItemType } from './model.d'; +import { getModelProvider, ModelProviderIdType } from './provider'; export const defaultQAModels: LLMModelItemType[] = [ { + provider: 'OpenAI', model: 'gpt-4o-mini', name: 'gpt-4o-mini', maxContext: 16000, @@ -23,6 +26,7 @@ export const defaultQAModels: LLMModelItemType[] = [ export const defaultVectorModels: VectorModelItemType[] = [ { + provider: 'OpenAI', model: 'text-embedding-3-small', name: 'Embedding-2', charsPointsPrice: 0, @@ -31,3 +35,35 @@ export const defaultVectorModels: VectorModelItemType[] = [ weight: 100 } ]; + +export const defaultWhisperModel: STTModelType = { + provider: 'OpenAI', + model: 'whisper-1', + name: 'whisper-1', + charsPointsPrice: 0 +}; + +export const getModelFromList = ( + modelList: { provider: ModelProviderIdType; name: string; model: string }[], + model: string +) => { + const modelData = modelList.find((item) => item.model === model) ?? modelList[0]; + const provider = getModelProvider(modelData.provider); + return { + ...modelData, + avatar: provider.avatar + }; +}; + +export enum ModelTypeEnum { + chat = 'chat', + embedding = 'embedding', + tts = 'tts', + stt = 'stt' +} +export const modelTypeList = [ + { label: i18nT('common:model.type.chat'), value: ModelTypeEnum.chat }, + { label: i18nT('common:model.type.embedding'), value: ModelTypeEnum.embedding }, + { label: i18nT('common:model.type.tts'), value: ModelTypeEnum.tts }, + { label: i18nT('common:model.type.stt'), value: ModelTypeEnum.stt } +]; diff --git a/packages/global/core/ai/prompt/agent.ts b/packages/global/core/ai/prompt/agent.ts index ca93dc83d..ea3362ed8 100644 --- a/packages/global/core/ai/prompt/agent.ts +++ b/packages/global/core/ai/prompt/agent.ts @@ -65,3 +65,13 @@ export const Prompt_CQJson = `请帮我执行一个“问题分类”任务， 问题："{{question}}" 类型ID= `; + +export const PROMPT_QUESTION_GUIDE = `You are an AI assistant tasked with predicting the user's next question based on the conversation history. Your goal is to generate 3 potential questions that will guide the user to continue the conversation. When generating these questions, adhere to the following rules: + +1. Use the same language as the user's last question in the conversation history. +2. Keep each question under 20 characters in length. + +Analyze the conversation history provided to you and use it as context to generate relevant and engaging follow-up questions. Your predictions should be logical extensions of the current topic or related areas that the user might be interested in exploring further. + +Remember to maintain consistency in tone and style with the existing conversation while providing diverse options for the user to choose from. Your goal is to keep the conversation flowing naturally and help the user delve deeper into the subject matter or explore related topics.`; +export const PROMPT_QUESTION_GUIDE_FOOTER = `Please strictly follow the format rules: \nReturn questions in JSON format: ['Question 1', 'Question 2', 'Question 3']. Your output: `; diff --git a/packages/global/core/ai/provider.ts b/packages/global/core/ai/provider.ts new file mode 100644 index 000000000..cbdb67bce --- /dev/null +++ b/packages/global/core/ai/provider.ts @@ -0,0 +1,158 @@ +import { i18nT } from '../../../web/i18n/utils'; + +export type ModelProviderIdType = + | 'OpenAI' + | 'Claude' + | 'Gemini' + | 'Meta' + | 'MistralAI' + | 'Groq' + | 'AliCloud' + | 'Qwen' + | 'Doubao' + | 'ChatGLM' + | 'DeepSeek' + | 'Ernie' + | 'Moonshot' + | 'MiniMax' + | 'SparkDesk' + | 'Hunyuan' + | 'Baichuan' + | 'StepFun' + | 'Yi' + | 'Ollama' + | 'BAAI' + | 'FishAudio' + | 'Other'; + +export type ModelProviderType = { + id: ModelProviderIdType; + name: string; + avatar: string; +}; + +export const ModelProviderList: ModelProviderType[] = [ + { + id: 'OpenAI', + name: 'OpenAI', + avatar: 'model/openai' + }, + { + id: 'Claude', + name: 'Claude', + avatar: 'model/claude' + }, + { + id: 'Gemini', + name: 'Gemini', + avatar: 'model/gemini' + }, + { + id: 'Meta', + name: 'Meta', + avatar: 'model/meta' + }, + { + id: 'MistralAI', + name: 'MistralAI', + avatar: 'model/mistral' + }, + { + id: 'Groq', + name: 'Groq', + avatar: 'model/groq' + }, + { + id: 'AliCloud', + name: i18nT('common:model_alicloud'), + avatar: 'model/alicloud' + }, + { + id: 'Qwen', + name: i18nT('common:model_qwen'), + avatar: 'model/qwen' + }, + { + id: 'Doubao', + name: i18nT('common:model_doubao'), + avatar: 'model/doubao' + }, + { + id: 'ChatGLM', + name: i18nT('common:model_chatglm'), + avatar: 'model/chatglm' + }, + { + id: 'Ernie', + name: i18nT('common:model_ernie'), + avatar: 'model/ernie' + }, + { + id: 'DeepSeek', + name: 'DeepSeek', + avatar: 'model/deepseek' + }, + { + id: 'Moonshot', + name: i18nT('common:model_moonshot'), + avatar: 'model/moonshot' + }, + { + id: 'MiniMax', + name: 'MiniMax', + avatar: 'model/minimax' + }, + { + id: 'SparkDesk', + name: i18nT('common:model_sparkdesk'), + avatar: 'model/sparkDesk' + }, + { + id: 'Hunyuan', + name: i18nT('common:model_hunyuan'), + avatar: 'model/hunyuan' + }, + { + id: 'Baichuan', + name: i18nT('common:model_baichuan'), + avatar: 'model/baichuan' + }, + { + id: 'StepFun', + name: i18nT('common:model_stepfun'), + avatar: 'model/stepfun' + }, + { + id: 'Yi', + name: i18nT('common:model_yi'), + avatar: 'model/yi' + }, + + { + id: 'Ollama', + name: 'Ollama', + avatar: 'model/ollama' + }, + { + id: 'BAAI', + name: i18nT('common:model_baai'), + avatar: 'model/BAAI' + }, + { + id: 'FishAudio', + name: 'FishAudio', + avatar: 'model/fishaudio' + }, + { + id: 'Other', + name: i18nT('common:model_other'), + avatar: 'model/huggingface' + } +]; +export const ModelProviderMap = Object.fromEntries( + ModelProviderList.map((item, index) => [item.id, { ...item, order: index }]) +); + +export const getModelProvider = (provider: ModelProviderIdType) => { + return ModelProviderMap[provider] ?? ModelProviderMap.Other; +}; diff --git a/packages/global/core/app/collaborator.d.ts b/packages/global/core/app/collaborator.d.ts index ca0fec721..66a456379 100644 --- a/packages/global/core/app/collaborator.d.ts +++ b/packages/global/core/app/collaborator.d.ts @@ -1,6 +1,6 @@ -import { RequireOnlyOne } from '../../common/type/utils'; +import type { RequireOnlyOne } from '../../common/type/utils'; import { - UpdateClbPermissionProps, + type UpdateClbPermissionProps, UpdatePermissionBody } from '../../support/permission/collaborator'; import { PermissionValueType } from '../../support/permission/type'; @@ -14,4 +14,5 @@ export type AppCollaboratorDeleteParams = { } & RequireOnlyOne<{ tmbId: string; groupId: string; + orgId: string; }>; diff --git a/packages/global/core/app/constants.ts b/packages/global/core/app/constants.ts index 62d1f5874..ba9485036 100644 --- a/packages/global/core/app/constants.ts +++ b/packages/global/core/app/constants.ts @@ -1,8 +1,10 @@ +import { PROMPT_QUESTION_GUIDE } from '../ai/prompt/agent'; import { AppTTSConfigType, AppFileSelectConfigType, AppWhisperConfigType, - AppAutoExecuteConfigType + AppAutoExecuteConfigType, + AppQGConfigType } from './type'; export enum AppTypeEnum { @@ -28,6 +30,12 @@ export const defaultWhisperConfig: AppWhisperConfigType = { autoTTSResponse: false }; +export const defaultQGConfig: AppQGConfigType = { + open: false, + model: 'gpt-4o-mini', + customPrompt: PROMPT_QUESTION_GUIDE +}; + export const defaultChatInputGuideConfig = { open: false, textList: [], diff --git a/packages/global/core/app/type.d.ts b/packages/global/core/app/type.d.ts index b735a1e30..86885c71e 100644 --- a/packages/global/core/app/type.d.ts +++ b/packages/global/core/app/type.d.ts @@ -13,6 +13,7 @@ import { StoreEdgeItemType } from '../workflow/type/edge'; import { AppPermission } from '../../support/permission/app/controller'; import { ParentIdType } from '../../common/parentFolder/type'; import { FlowNodeInputTypeEnum } from 'core/workflow/node/constant'; +import { WorkflowTemplateBasicType } from '@fastgpt/global/core/workflow/type'; export type AppSchema = { _id: string; @@ -73,8 +74,8 @@ export type AppSimpleEditFormType = { aiSettings: { model: string; systemPrompt?: string | undefined; - temperature: number; - maxToken: number; + temperature?: number; + maxToken?: number; isResponseAnswerText: boolean; maxHistories: number; }; @@ -97,7 +98,7 @@ export type AppChatConfigType = { welcomeText?: string; variables?: VariableItemType[]; autoExecute?: AppAutoExecuteConfigType; - questionGuide?: boolean; + questionGuide?: AppQGConfigType; ttsConfig?: AppTTSConfigType; whisperConfig?: AppWhisperConfigType; scheduledTriggerConfig?: AppScheduledTriggerConfigType; @@ -109,8 +110,8 @@ export type AppChatConfigType = { }; export type SettingAIDataType = { model: string; - temperature: number; - maxToken: number; + temperature?: number; + maxToken?: number; isResponseAnswerText?: boolean; maxHistories?: number; [NodeInputKeyEnum.aiChatVision]?: boolean; // Is open vision mode @@ -148,6 +149,14 @@ export type AppWhisperConfigType = { autoSend: boolean; autoTTSResponse: boolean; }; + +// question guide +export type AppQGConfigType = { + open: boolean; + model?: string; + customPrompt?: string; +}; + // question guide text export type ChatInputGuideConfigType = { open: boolean; @@ -176,3 +185,28 @@ export type SystemPluginListItemType = { name: string; avatar: string; }; + +export type AppTemplateSchemaType = { + templateId: string; + name: string; + intro: string; + avatar: string; + tags: string[]; + type: string; + author?: string; + isActive?: boolean; + userGuide?: { + type: 'markdown' | 'link'; + content?: string; + link?: string; + }; + isQuickTemplate?: boolean; + order?: number; + workflow: WorkflowTemplateBasicType; +}; + +export type TemplateTypeSchemaType = { + typeName: string; + typeId: string; + typeOrder: number; +}; diff --git a/packages/global/core/app/utils.ts b/packages/global/core/app/utils.ts index c574cac5f..1c793cfea 100644 --- a/packages/global/core/app/utils.ts +++ b/packages/global/core/app/utils.ts @@ -5,6 +5,8 @@ import type { FlowNodeInputItemType } from '../workflow/type/io.d'; import { getAppChatConfig } from '../workflow/utils'; import { StoreNodeItemType } from '../workflow/type/node'; import { DatasetSearchModeEnum } from '../dataset/constants'; +import { WorkflowTemplateBasicType } from '../workflow/type'; +import { AppTypeEnum } from './constants'; export const getDefaultAppForm = (): AppSimpleEditFormType => { return { @@ -127,3 +129,20 @@ export const appWorkflow2Form = ({ return defaultAppForm; }; + +export const getAppType = (config?: WorkflowTemplateBasicType | AppSimpleEditFormType) => { + if (!config) return ''; + + if ('aiSettings' in config) { + return AppTypeEnum.simple; + } + + if (!('nodes' in config)) return ''; + if (config.nodes.some((node) => node.flowNodeType === 'workflowStart')) { + return AppTypeEnum.workflow; + } + if (config.nodes.some((node) => node.flowNodeType === 'pluginInput')) { + return AppTypeEnum.plugin; + } + return ''; +}; diff --git a/packages/global/core/chat/type.d.ts b/packages/global/core/chat/type.d.ts index 72be2e388..a9171b156 100644 --- a/packages/global/core/chat/type.d.ts +++ b/packages/global/core/chat/type.d.ts @@ -16,6 +16,7 @@ import { DatasetSearchModeEnum } from '../dataset/constants'; import { DispatchNodeResponseType } from '../workflow/runtime/type.d'; import { ChatBoxInputType } from '../../../../projects/app/src/components/core/chat/ChatContainer/ChatBox/type'; import { WorkflowInteractiveResponseType } from '../workflow/template/system/interactive/type'; +import { FlowNodeInputItemType } from '../workflow/type/io'; export type ChatSchema = { _id: string; @@ -35,6 +36,7 @@ export type ChatSchema = { variableList?: VariableItemType[]; welcomeText?: string; variables: Record; + pluginInputs?: FlowNodeInputItemType[]; metadata?: Record; }; diff --git a/packages/global/core/dataset/api.d.ts b/packages/global/core/dataset/api.d.ts index 63e931a85..5cf6c860f 100644 --- a/packages/global/core/dataset/api.d.ts +++ b/packages/global/core/dataset/api.d.ts @@ -17,6 +17,11 @@ export type DatasetUpdateBody = { externalReadUrl?: DatasetSchemaType['externalReadUrl']; defaultPermission?: DatasetSchemaType['defaultPermission']; apiServer?: DatasetSchemaType['apiServer']; + yuqueServer?: DatasetSchemaType['yuqueServer']; + feishuServer?: DatasetSchemaType['feishuServer']; + + // sync schedule + autoSync?: boolean; }; /* ================= collection ===================== */ @@ -47,6 +52,8 @@ export type CreateDatasetCollectionParams = DatasetCollectionChunkMetadataType & tags?: string[]; createTime?: Date; + updateTime?: Date; + nextSyncTime?: Date; }; export type ApiCreateDatasetCollectionParams = DatasetCollectionChunkMetadataType & { diff --git a/packages/global/core/dataset/apiDataset.d.ts b/packages/global/core/dataset/apiDataset.d.ts index 152fae126..1d64a8a59 100644 --- a/packages/global/core/dataset/apiDataset.d.ts +++ b/packages/global/core/dataset/apiDataset.d.ts @@ -22,3 +22,14 @@ export type APIFileContentResponse = { export type APIFileReadResponse = { url: string; }; + +export type FeishuServer = { + appId: string; + appSecret: string; + folderToken: string; +}; + +export type YuqueServer = { + userId: string; + token: string; +}; diff --git a/packages/global/core/dataset/collaborator.d.ts b/packages/global/core/dataset/collaborator.d.ts index 7f33f4d51..672430c1d 100644 --- a/packages/global/core/dataset/collaborator.d.ts +++ b/packages/global/core/dataset/collaborator.d.ts @@ -11,4 +11,5 @@ export type DatasetCollaboratorDeleteParams = { } & RequireOnlyOne<{ tmbId: string; groupId: string; + orgId: string; }>; diff --git a/packages/global/core/dataset/collection/utils.ts b/packages/global/core/dataset/collection/utils.ts index 97aec9ac9..f82a689c4 100644 --- a/packages/global/core/dataset/collection/utils.ts +++ b/packages/global/core/dataset/collection/utils.ts @@ -1,9 +1,7 @@ import { DatasetCollectionTypeEnum, TrainingModeEnum, TrainingTypeMap } from '../constants'; -import { CollectionWithDatasetType, DatasetCollectionSchemaType } from '../type'; +import { DatasetCollectionSchemaType } from '../type'; -export const getCollectionSourceData = ( - collection?: CollectionWithDatasetType | DatasetCollectionSchemaType -) => { +export const getCollectionSourceData = (collection?: DatasetCollectionSchemaType) => { return { sourceId: collection?.fileId || diff --git a/packages/global/core/dataset/constants.ts b/packages/global/core/dataset/constants.ts index 5a5f578b6..a522645f9 100644 --- a/packages/global/core/dataset/constants.ts +++ b/packages/global/core/dataset/constants.ts @@ -6,7 +6,9 @@ export enum DatasetTypeEnum { dataset = 'dataset', websiteDataset = 'websiteDataset', // depp link externalFile = 'externalFile', - apiDataset = 'apiDataset' + apiDataset = 'apiDataset', + feishu = 'feishu', + yuque = 'yuque' } export const DatasetTypeMap = { [DatasetTypeEnum.folder]: { @@ -33,6 +35,16 @@ export const DatasetTypeMap = { icon: 'core/dataset/externalDatasetOutline', label: 'api_file', collectionLabel: 'common.File' + }, + [DatasetTypeEnum.feishu]: { + icon: 'core/dataset/feishuDatasetOutline', + label: 'feishu_dataset', + collectionLabel: 'common.File' + }, + [DatasetTypeEnum.yuque]: { + icon: 'core/dataset/yuqueDatasetOutline', + label: 'yuque_dataset', + collectionLabel: 'common.File' } }; @@ -82,7 +94,8 @@ export const DatasetCollectionTypeMap = { export enum DatasetCollectionSyncResultEnum { sameRaw = 'sameRaw', - success = 'success' + success = 'success', + failed = 'failed' } export const DatasetCollectionSyncResultMap = { [DatasetCollectionSyncResultEnum.sameRaw]: { @@ -90,6 +103,9 @@ export const DatasetCollectionSyncResultMap = { }, [DatasetCollectionSyncResultEnum.success]: { label: i18nT('common:core.dataset.collection.sync.result.success') + }, + [DatasetCollectionSyncResultEnum.failed]: { + label: i18nT('dataset:sync_collection_failed') } }; diff --git a/packages/global/core/dataset/type.d.ts b/packages/global/core/dataset/type.d.ts index d70cc5c0e..7a772f78d 100644 --- a/packages/global/core/dataset/type.d.ts +++ b/packages/global/core/dataset/type.d.ts @@ -10,7 +10,7 @@ import { } from './constants'; import { DatasetPermission } from '../../support/permission/dataset/controller'; import { Permission } from '../../support/permission/controller'; -import { APIFileServer } from './apiDataset'; +import { APIFileServer, FeishuServer, YuqueServer } from './apiDataset'; export type DatasetSchemaType = { _id: string; @@ -33,9 +33,10 @@ export type DatasetSchemaType = { }; inheritPermission: boolean; apiServer?: APIFileServer; + feishuServer?: FeishuServer; + yuqueServer?: YuqueServer; - syncSchedule?: { cronString: string; timezone: string }; - syncNextTime?: Date; + autoSync?: boolean; // abandon externalReadUrl?: string; @@ -65,11 +66,13 @@ export type DatasetCollectionSchemaType = { fileId?: string; // local file id rawLink?: string; // link url externalFileId?: string; //external file id + apiFileId?: string; // api file id + externalFileUrl?: string; // external import url + + nextSyncTime?: Date; rawTextLength?: number; hashRawText?: string; - externalFileUrl?: string; // external import url - apiFileId?: string; // api file id metadata?: { webPageSelector?: string; relatedImgId?: string; // The id of the associated image collections @@ -109,6 +112,15 @@ export type DatasetDataSchemaType = { rebuilding?: boolean; }; +export type DatasetDataTextSchemaType = { + _id: string; + teamId: string; + datasetId: string; + collectionId: string; + dataId: string; + fullTextToken: string; +}; + export type DatasetTrainingSchemaType = { _id: string; userId: string; @@ -130,11 +142,8 @@ export type DatasetTrainingSchemaType = { indexes: Omit[]; }; -export type CollectionWithDatasetType = Omit & { - datasetId: DatasetSchemaType; -}; -export type DatasetDataWithCollectionType = Omit & { - collectionId: DatasetCollectionSchemaType; +export type CollectionWithDatasetType = DatasetCollectionSchemaType & { + dataset: DatasetSchemaType; }; /* ================= dataset ===================== */ diff --git a/packages/global/core/workflow/runtime/type.d.ts b/packages/global/core/workflow/runtime/type.d.ts index de469cfe1..7084740b1 100644 --- a/packages/global/core/workflow/runtime/type.d.ts +++ b/packages/global/core/workflow/runtime/type.d.ts @@ -21,13 +21,20 @@ import { ReadFileNodeResponse } from '../template/system/readFiles/type'; import { UserSelectOptionType } from '../template/system/userSelect/type'; import { WorkflowResponseType } from '../../../../service/core/workflow/dispatch/type'; import { AiChatQuoteRoleType } from '../template/system/aiChat/type'; +import { LafAccountType, OpenaiAccountType } from '../../../support/user/team/type'; + +export type ExternalProviderType = { + openaiAccount?: OpenaiAccountType; + externalWorkflowVariables?: Record; +}; /* workflow props */ export type ChatDispatchProps = { res?: NextApiResponse; requestOrigin?: string; mode: 'test' | 'chat' | 'debug'; - user: UserModelSchema; + timezone: string; + externalProvider: ExternalProviderType; runningAppInfo: { id: string; // May be the id of the system plug-in (cannot be used directly to look up the table) @@ -100,7 +107,9 @@ export type DispatchNodeResponseType = { mergeSignId?: string; // bill - tokens?: number; + tokens?: number; // deprecated + inputTokens?: number; + outputTokens?: number; model?: string; contextTotalLen?: number; totalPoints?: number; @@ -150,6 +159,8 @@ export type DispatchNodeResponseType = { // tool toolCallTokens?: number; + toolCallInputTokens?: number; + toolCallOutputTokens?: number; toolDetail?: ChatHistoryItemResType[]; toolStop?: boolean; @@ -201,13 +212,14 @@ export type DispatchNodeResultType = { export type AIChatNodeProps = { [NodeInputKeyEnum.aiModel]: string; [NodeInputKeyEnum.aiSystemPrompt]?: string; - [NodeInputKeyEnum.aiChatTemperature]: number; - [NodeInputKeyEnum.aiChatMaxToken]: number; + [NodeInputKeyEnum.aiChatTemperature]?: number; + [NodeInputKeyEnum.aiChatMaxToken]?: number; [NodeInputKeyEnum.aiChatIsResponseText]: boolean; + [NodeInputKeyEnum.aiChatVision]?: boolean; + [NodeInputKeyEnum.aiChatQuoteRole]?: AiChatQuoteRoleType; [NodeInputKeyEnum.aiChatQuoteTemplate]?: string; [NodeInputKeyEnum.aiChatQuotePrompt]?: string; - [NodeInputKeyEnum.aiChatVision]?: boolean; [NodeInputKeyEnum.stringQuoteText]?: string; [NodeInputKeyEnum.fileUrlList]?: string[]; diff --git a/packages/global/core/workflow/runtime/utils.ts b/packages/global/core/workflow/runtime/utils.ts index 54e4c372b..e7523f123 100644 --- a/packages/global/core/workflow/runtime/utils.ts +++ b/packages/global/core/workflow/runtime/utils.ts @@ -9,6 +9,7 @@ import { isValidReferenceValueFormat } from '../utils'; import { FlowNodeOutputItemType, ReferenceValueType } from '../type/io'; import { ChatItemType, NodeOutputItemType } from '../../../core/chat/type'; import { ChatItemValueTypeEnum, ChatRoleEnum } from '../../../core/chat/constants'; +import { replaceVariable, valToStr } from '../../../common/string/tools'; export const getMaxHistoryLimitFromNodes = (nodes: StoreNodeItemType[]): number => { let limit = 10; @@ -251,6 +252,7 @@ export const getReferenceVariableValue = ({ return variables[outputId]; } + // 避免 value 刚好就是二个元素的字符串数组 const node = nodes.find((node) => node.nodeId === sourceNodeId); if (!node) { return value; @@ -283,9 +285,13 @@ export const formatVariableValByType = (val: any, valueType?: WorkflowIOValueTyp if (!valueType) return val; // Value type check, If valueType invalid, return undefined if (valueType.startsWith('array') && !Array.isArray(val)) return undefined; - if (valueType === WorkflowIOValueTypeEnum.boolean && typeof val !== 'boolean') return undefined; - if (valueType === WorkflowIOValueTypeEnum.number && typeof val !== 'number') return undefined; - if (valueType === WorkflowIOValueTypeEnum.string && typeof val !== 'string') return undefined; + if (valueType === WorkflowIOValueTypeEnum.boolean) return Boolean(val); + if (valueType === WorkflowIOValueTypeEnum.number) return Number(val); + if (valueType === WorkflowIOValueTypeEnum.string) { + if (val === undefined) return 'undefined'; + if (val === null) return 'null'; + return typeof val === 'object' ? JSON.stringify(val) : String(val); + } if ( [ WorkflowIOValueTypeEnum.object, @@ -312,6 +318,8 @@ export function replaceEditorVariable({ }) { if (typeof text !== 'string') return text; + text = replaceVariable(text, variables); + const variablePattern = /\{\{\$([^.]+)\.([^$]+)\$\}\}/g; const matches = [...text.matchAll(variablePattern)]; if (matches.length === 0) return text; @@ -331,15 +339,12 @@ export function replaceEditorVariable({ const output = node.outputs.find((output) => output.id === id); if (output) return formatVariableValByType(output.value, output.valueType); + // Use the node's input as the variable value(Example: HTTP data will reference its own dynamic input) const input = node.inputs.find((input) => input.key === id); if (input) return getReferenceVariableValue({ value: input.value, nodes, variables }); })(); - const formatVal = (() => { - if (variableVal === undefined) return 'undefined'; - if (variableVal === null) return 'null'; - return typeof variableVal === 'object' ? JSON.stringify(variableVal) : String(variableVal); - })(); + const formatVal = valToStr(variableVal); const regex = new RegExp(`\\{\\{\\$(${nodeId}\\.${id})\\$\\}\\}`, 'g'); text = text.replace(regex, () => formatVal); diff --git a/packages/global/core/workflow/template/system/workflowStart.ts b/packages/global/core/workflow/template/system/workflowStart.ts index 42d08b824..ef3250796 100644 --- a/packages/global/core/workflow/template/system/workflowStart.ts +++ b/packages/global/core/workflow/template/system/workflowStart.ts @@ -30,7 +30,6 @@ export const WorkflowStart: FlowNodeTemplateType = { intro: '', forbidDelete: true, unique: true, - courseUrl: '/docs/guide/workbench/workflow/input/', version: '481', inputs: [{ ...Input_Template_UserChatInput, toolDescription: i18nT('workflow:user_question') }], outputs: [ diff --git a/packages/global/core/workflow/type/io.d.ts b/packages/global/core/workflow/type/io.d.ts index 3653e2a12..7f6b46eb7 100644 --- a/packages/global/core/workflow/type/io.d.ts +++ b/packages/global/core/workflow/type/io.d.ts @@ -49,9 +49,10 @@ export type FlowNodeInputItemType = InputComponentPropsType & { debugLabel?: string; description?: string; // field desc required?: boolean; - toolDescription?: string; // If this field is not empty, it is entered as a tool enum?: string; + toolDescription?: string; // If this field is not empty, it is entered as a tool + // render components params canEdit?: boolean; // dynamic inputs isPro?: boolean; // Pro version field diff --git a/packages/global/core/workflow/type/node.d.ts b/packages/global/core/workflow/type/node.d.ts index 2359ac0ee..d0bc5b919 100644 --- a/packages/global/core/workflow/type/node.d.ts +++ b/packages/global/core/workflow/type/node.d.ts @@ -69,6 +69,7 @@ export type FlowNodeTemplateType = FlowNodeCommonType & { diagram?: string; // diagram url courseUrl?: string; // course url + userGuide?: string; // user guide }; export type NodeTemplateListItemType = { @@ -87,6 +88,7 @@ export type NodeTemplateListItemType = { currentCost?: number; // 当前积分消耗 hasTokenFee?: boolean; // 是否配置积分 instructions?: string; // 使用说明 + courseUrl?: string; // 教程链接 }; export type NodeTemplateListType = { diff --git a/packages/global/core/workflow/utils.ts b/packages/global/core/workflow/utils.ts index 3e082c235..06307c403 100644 --- a/packages/global/core/workflow/utils.ts +++ b/packages/global/core/workflow/utils.ts @@ -26,12 +26,14 @@ import type { AppScheduledTriggerConfigType, ChatInputGuideConfigType, AppChatConfigType, - AppAutoExecuteConfigType + AppAutoExecuteConfigType, + AppQGConfigType } from '../app/type'; import { EditorVariablePickerType } from '../../../web/components/common/Textarea/PromptEditor/type'; import { defaultAutoExecuteConfig, defaultChatInputGuideConfig, + defaultQGConfig, defaultTTSConfig, defaultWhisperConfig } from '../app/constants'; @@ -76,9 +78,14 @@ export const splitGuideModule = (guideModules?: StoreNodeItemType) => { const variables: VariableItemType[] = guideModules?.inputs.find((item) => item.key === NodeInputKeyEnum.variables)?.value ?? []; - const questionGuide: boolean = - !!guideModules?.inputs?.find((item) => item.key === NodeInputKeyEnum.questionGuide)?.value ?? - false; + // Adapt old version + const questionGuideVal = guideModules?.inputs?.find( + (item) => item.key === NodeInputKeyEnum.questionGuide + )?.value; + const questionGuide: AppQGConfigType = + typeof questionGuideVal === 'boolean' + ? { ...defaultQGConfig, open: questionGuideVal } + : questionGuideVal ?? defaultQGConfig; const ttsConfig: AppTTSConfigType = guideModules?.inputs?.find((item) => item.key === NodeInputKeyEnum.tts)?.value ?? diff --git a/packages/global/package.json b/packages/global/package.json index faaab7015..b00061a0b 100644 --- a/packages/global/package.json +++ b/packages/global/package.json @@ -13,7 +13,9 @@ "next": "14.2.5", "openai": "4.61.0", "openapi-types": "^12.1.3", - "timezones-list": "^3.0.2" + "json5": "^2.2.3", + "timezones-list": "^3.0.2", + "@bany/curl-to-json": "^1.2.8" }, "devDependencies": { "@types/js-yaml": "^4.0.9", diff --git a/packages/global/support/outLink/constant.ts b/packages/global/support/outLink/constant.ts index aaa1bafb6..0fcba2ec4 100644 --- a/packages/global/support/outLink/constant.ts +++ b/packages/global/support/outLink/constant.ts @@ -3,6 +3,7 @@ export enum PublishChannelEnum { iframe = 'iframe', apikey = 'apikey', feishu = 'feishu', + dingtalk = 'dingtalk', wecom = 'wecom', officialAccount = 'official_account' } diff --git a/packages/global/support/outLink/type.d.ts b/packages/global/support/outLink/type.d.ts index e33179c1a..b1c74e60c 100644 --- a/packages/global/support/outLink/type.d.ts +++ b/packages/global/support/outLink/type.d.ts @@ -14,6 +14,11 @@ export interface FeishuAppType { verificationToken?: string; } +export interface DingtalkAppType { + clientId: string; + clientSecret: string; +} + export interface WecomAppType { AgentId: string; CorpId: string; @@ -36,7 +41,12 @@ export interface OffiAccountAppType { // because we can not reply anything in 15s. Thus, the wechat server will treat this request as a failed request. } -export type OutlinkAppType = FeishuAppType | WecomAppType | OffiAccountAppType | undefined; +export type OutlinkAppType = + | FeishuAppType + | WecomAppType + | OffiAccountAppType + | DingtalkAppType + | undefined; export type OutLinkSchema = { _id: string; @@ -73,11 +83,6 @@ export type OutLinkSchema = { app: T; }; -// to handle MongoDB querying -export type OutLinkWithAppType = Omit & { - appId: AppSchema; -}; - // Edit the Outlink export type OutLinkEditType = { _id?: string; diff --git a/packages/global/support/permission/collaborator.d.ts b/packages/global/support/permission/collaborator.d.ts index 60a84a9d5..af7ee84e1 100644 --- a/packages/global/support/permission/collaborator.d.ts +++ b/packages/global/support/permission/collaborator.d.ts @@ -10,22 +10,18 @@ export type CollaboratorItemType = { } & RequireOnlyOne<{ tmbId: string; groupId: string; + orgId: string; }>; export type UpdateClbPermissionProps = { members?: string[]; groups?: string[]; + orgs?: string[]; permission: PermissionValueType; }; -export type DeleteClbPermissionProps = RequireOnlyOne<{ - tmbId: string; - groupId: string; -}>; - -export type UpdatePermissionBody = { - permission: PermissionValueType; -} & RequireOnlyOne<{ - memberId: string; - groupId: string; +export type DeletePermissionQuery = RequireOnlyOne<{ + tmbId?: string; + groupId?: string; + orgId?: string; }>; diff --git a/packages/global/support/permission/type.d.ts b/packages/global/support/permission/type.d.ts index f6f29c52a..f473d2941 100644 --- a/packages/global/support/permission/type.d.ts +++ b/packages/global/support/permission/type.d.ts @@ -1,7 +1,9 @@ +import { UserModelSchema } from '../user/type'; import { RequireOnlyOne } from '../../common/type/utils'; -import { TeamMemberWithUserSchema } from '../user/team/type'; -import { AuthUserTypeEnum, PermissionKeyEnum, PerResourceTypeEnum } from './constant'; +import { TeamMemberSchema } from '../user/team/type'; import { MemberGroupSchemaType } from './memberGroup/type'; +import type { TeamMemberWithUserSchema } from '../user/team/type'; +import { AuthUserTypeEnum, type PermissionKeyEnum, type PerResourceTypeEnum } from './constant'; // PermissionValueType, the type of permission's value is a number, which is a bit field actually. // It is spired by the permission system in Linux. @@ -28,14 +30,11 @@ export type ResourcePermissionType = { } & RequireOnlyOne<{ tmbId: string; groupId: string; + orgId: string; }>; export type ResourcePerWithTmbWithUser = Omit & { - tmbId: TeamMemberWithUserSchema; -}; - -export type ResourcePerWithGroup = Omit & { - groupId: MemberGroupSchemaType; + tmbId: TeamMemberSchema & { user: UserModelSchema }; }; export type PermissionSchemaType = { diff --git a/packages/global/support/user/api.d.ts b/packages/global/support/user/api.d.ts index 85f60bf2b..31da306b0 100644 --- a/packages/global/support/user/api.d.ts +++ b/packages/global/support/user/api.d.ts @@ -1,4 +1,5 @@ import { OAuthEnum } from './constant'; +import { TrackRegisterParams } from './login/api'; export type PostLoginProps = { username: string; @@ -9,8 +10,7 @@ export type OauthLoginProps = { type: `${OAuthEnum}`; code: string; callbackUrl: string; - inviterId?: string; -}; +} & TrackRegisterParams; export type WxLoginProps = { inviterId?: string; diff --git a/packages/global/support/user/constant.ts b/packages/global/support/user/constant.ts index 0fae675cb..8d8d24e27 100644 --- a/packages/global/support/user/constant.ts +++ b/packages/global/support/user/constant.ts @@ -16,5 +16,7 @@ export enum OAuthEnum { google = 'google', wechat = 'wechat', microsoft = 'microsoft', + dingtalk = 'dingtalk', + wecom = 'wecom', sso = 'sso' } diff --git a/packages/global/support/user/login/api.d.ts b/packages/global/support/user/login/api.d.ts index d71181491..d994a24ec 100644 --- a/packages/global/support/user/login/api.d.ts +++ b/packages/global/support/user/login/api.d.ts @@ -2,3 +2,17 @@ export type GetWXLoginQRResponse = { code: string; codeUrl: string; }; + +export type TrackRegisterParams = { + inviterId?: string; + bd_vid?: string; + fastgpt_sem?: { + keyword: string; + }; + sourceDomain?: string; +}; +export type AccountRegisterBody = { + username: string; + code: string; + password: string; +} & TrackRegisterParams; diff --git a/packages/global/support/user/login/constants.ts b/packages/global/support/user/login/constants.ts new file mode 100644 index 000000000..0093f9367 --- /dev/null +++ b/packages/global/support/user/login/constants.ts @@ -0,0 +1,3 @@ +export function checkIsWecomTerminal() { + return /wxwork/i.test(navigator.userAgent); +} diff --git a/packages/global/support/user/team/controller.d.ts b/packages/global/support/user/team/controller.d.ts index d485b39ae..98a64af30 100644 --- a/packages/global/support/user/team/controller.d.ts +++ b/packages/global/support/user/team/controller.d.ts @@ -1,6 +1,6 @@ import { PermissionValueType } from '../../permission/type'; import { TeamMemberRoleEnum } from './constant'; -import { LafAccountType, TeamMemberSchema } from './type'; +import { LafAccountType, TeamMemberSchema, ThirdPartyAccountType } from './type'; export type AuthTeamRoleProps = { teamId: string; @@ -11,13 +11,13 @@ export type CreateTeamProps = { name: string; avatar?: string; defaultTeam?: boolean; - lafAccount?: LafAccountType; + memberName?: string; }; -export type UpdateTeamProps = { +export type UpdateTeamProps = Omit & { name?: string; avatar?: string; teamDomain?: string; - lafAccount?: null | LafAccountType; + externalWorkflowVariable?: { key: string; value: string }; }; /* ------------- member ----------- */ diff --git a/packages/global/support/user/team/org/api.d.ts b/packages/global/support/user/team/org/api.d.ts new file mode 100644 index 000000000..69ca09796 --- /dev/null +++ b/packages/global/support/user/team/org/api.d.ts @@ -0,0 +1,32 @@ +export type postCreateOrgData = { + name: string; + parentId: string; + description?: string; + avatar?: string; +}; + +export type putUpdateOrgMembersData = { + orgId: string; + members: { + tmbId: string; + // role: `${OrgMemberRole}`; + }[]; +}; + +export type putUpdateOrgData = { + orgId: string; + name?: string; + avatar?: string; + description?: string; +}; + +export type putMoveOrgType = { + orgId: string; + targetOrgId: string; +}; + +// type putChnageOrgOwnerData = { +// orgId: string; +// tmbId: string; +// toAdmin?: boolean; +// }; diff --git a/packages/global/support/user/team/org/constant.ts b/packages/global/support/user/team/org/constant.ts new file mode 100644 index 000000000..370cdad82 --- /dev/null +++ b/packages/global/support/user/team/org/constant.ts @@ -0,0 +1,12 @@ +import { OrgSchemaType } from './type'; + +export const OrgCollectionName = 'team_orgs'; +export const OrgMemberCollectionName = 'team_org_members'; + +export const getOrgChildrenPath = (org: OrgSchemaType) => `${org.path}/${org.pathId}`; + +// export enum OrgMemberRole { +// owner = 'owner', +// admin = 'admin', +// member = 'member' +// } diff --git a/packages/global/support/user/team/org/type.d.ts b/packages/global/support/user/team/org/type.d.ts new file mode 100644 index 000000000..ca4a73292 --- /dev/null +++ b/packages/global/support/user/team/org/type.d.ts @@ -0,0 +1,25 @@ +import type { TeamPermission } from 'support/permission/user/controller'; +import { ResourcePermissionType } from '../type'; + +type OrgSchemaType = { + _id: string; + teamId: string; + pathId: string; + path: string; + name: string; + avatar?: string; + description?: string; + updateTime: Date; +}; + +type OrgMemberSchemaType = { + teamId: string; + orgId: string; + tmbId: string; +}; + +type OrgType = Omit & { + avatar: string; + members: OrgMemberSchemaType[]; + permission: TeamPermission; +}; diff --git a/packages/global/support/user/team/type.d.ts b/packages/global/support/user/team/type.d.ts index 61e518257..9c7dc12b4 100644 --- a/packages/global/support/user/team/type.d.ts +++ b/packages/global/support/user/team/type.d.ts @@ -4,6 +4,12 @@ import { LafAccountType } from './type'; import { PermissionValueType, ResourcePermissionType } from '../../permission/type'; import { TeamPermission } from '../../permission/user/controller'; +export type ThirdPartyAccountType = { + lafAccount?: LafAccountType; + openaiAccount?: OpenaiAccountType; + externalWorkflowVariables?: Record; +}; + export type TeamSchema = { _id: string; name: string; @@ -16,9 +22,8 @@ export type TeamSchema = { lastExportDatasetTime: Date; lastWebsiteSyncTime: Date; }; - lafAccount: LafAccountType; notificationAccount?: string; -}; +} & ThirdPartyAccountType; export type tagsType = { label: string; @@ -42,34 +47,27 @@ export type TeamMemberSchema = { defaultTeam: boolean; }; -export type TeamMemberWithUserSchema = Omit & { - userId: UserModelSchema; -}; - -export type TeamMemberWithTeamSchema = Omit & { - teamId: TeamSchema; -}; - -export type TeamMemberWithTeamAndUserSchema = Omit & { - userId: UserModelSchema; +export type TeamMemberWithTeamAndUserSchema = TeamMemberSchema & { + team: TeamSchema; + user: UserModelSchema; }; export type TeamTmbItemType = { userId: string; teamId: string; + teamAvatar?: string; teamName: string; memberName: string; avatar: string; - balance: number; + balance?: number; tmbId: string; teamDomain: string; defaultTeam: boolean; role: `${TeamMemberRoleEnum}`; status: `${TeamMemberStatusEnum}`; - lafAccount?: LafAccountType; notificationAccount?: string; permission: TeamPermission; -}; +} & ThirdPartyAccountType; export type TeamMemberItemType = { userId: string; @@ -88,11 +86,16 @@ export type TeamTagItemType = { }; export type LafAccountType = { - token: string; appid: string; + token: string; pat: string; }; +export type OpenaiAccountType = { + key: string; + baseUrl: string; +}; + export type TeamInvoiceHeaderType = { teamName: string; unifiedCreditCode: string; @@ -101,6 +104,7 @@ export type TeamInvoiceHeaderType = { bankName?: string; bankAccount?: string; needSpecialInvoice: boolean; + contactPhone: string; emailAddress: string; }; diff --git a/packages/global/support/user/type.d.ts b/packages/global/support/user/type.d.ts index 1cb768dd3..0f8729266 100644 --- a/packages/global/support/user/type.d.ts +++ b/packages/global/support/user/type.d.ts @@ -14,10 +14,6 @@ export type UserModelSchema = { timezone: string; status: `${UserStatusEnum}`; lastLoginTmbId?: string; - openaiAccount?: { - key: string; - baseUrl: string; - }; fastgpt_sem?: { keyword: string; }; @@ -29,7 +25,6 @@ export type UserType = { avatar: string; timezone: string; promotionRate: UserModelSchema['promotionRate']; - openaiAccount: UserModelSchema['openaiAccount']; team: TeamTmbItemType; standardInfo?: standardInfoType; notificationAccount?: string; diff --git a/packages/global/support/user/utils.ts b/packages/global/support/user/utils.ts new file mode 100644 index 000000000..b0db78a92 --- /dev/null +++ b/packages/global/support/user/utils.ts @@ -0,0 +1,16 @@ +export const getRandomUserAvatar = () => { + const defaultAvatars = [ + '/imgs/avatar/RoyalBlueAvatar.svg', + '/imgs/avatar/PurpleAvatar.svg', + '/imgs/avatar/AdoraAvatar.svg', + '/imgs/avatar/OrangeAvatar.svg', + '/imgs/avatar/RedAvatar.svg', + '/imgs/avatar/GrayModernAvatar.svg', + '/imgs/avatar/TealAvatar.svg', + '/imgs/avatar/GreenAvatar.svg', + '/imgs/avatar/BrightBlueAvatar.svg', + '/imgs/avatar/BlueAvatar.svg' + ]; + + return defaultAvatars[Math.floor(Math.random() * defaultAvatars.length)]; +}; diff --git a/packages/global/support/wallet/bill/type.d.ts b/packages/global/support/wallet/bill/type.d.ts index 51b52f1be..1a97e29c3 100644 --- a/packages/global/support/wallet/bill/type.d.ts +++ b/packages/global/support/wallet/bill/type.d.ts @@ -23,7 +23,8 @@ export type BillSchemaType = { }; export type ChatNodeUsageType = { - tokens?: number; + inputTokens?: number; + outputTokens?: number; totalPoints: number; moduleName: string; model?: string; diff --git a/packages/global/support/wallet/usage/constants.ts b/packages/global/support/wallet/usage/constants.ts index 6e0dab53b..dfbff7f4d 100644 --- a/packages/global/support/wallet/usage/constants.ts +++ b/packages/global/support/wallet/usage/constants.ts @@ -9,6 +9,7 @@ export enum UsageSourceEnum { share = 'share', wecom = 'wecom', feishu = 'feishu', + dingtalk = 'dingtalk', official_account = 'official_account' } @@ -39,5 +40,8 @@ export const UsageSourceMap = { }, [UsageSourceEnum.wecom]: { label: i18nT('user:usage.wecom') + }, + [UsageSourceEnum.dingtalk]: { + label: i18nT('user:usage.dingtalk') } }; diff --git a/packages/global/support/wallet/usage/tools.ts b/packages/global/support/wallet/usage/tools.ts index 33dae3892..da2981605 100644 --- a/packages/global/support/wallet/usage/tools.ts +++ b/packages/global/support/wallet/usage/tools.ts @@ -37,6 +37,8 @@ export const getUsageSourceByPublishChannel = (publishchannel: PublishChannelEnu return UsageSourceEnum.wecom; case PublishChannelEnum.officialAccount: return UsageSourceEnum.official_account; + case PublishChannelEnum.dingtalk: + return UsageSourceEnum.dingtalk; default: return UsageSourceEnum.fastgpt; } diff --git a/packages/global/support/wallet/usage/type.d.ts b/packages/global/support/wallet/usage/type.d.ts index 09a1c4182..076c589b5 100644 --- a/packages/global/support/wallet/usage/type.d.ts +++ b/packages/global/support/wallet/usage/type.d.ts @@ -2,9 +2,13 @@ import { CreateUsageProps } from './api'; import { UsageSourceEnum } from './constants'; export type UsageListItemCountType = { - tokens?: number; + inputTokens?: number; + outputTokens?: number; charsLength?: number; duration?: number; + + // deprecated + tokens?: number; }; export type UsageListItemType = UsageListItemCountType & { moduleName: string; diff --git a/packages/plugins/package.json b/packages/plugins/package.json index 3120ad41e..639a38e68 100644 --- a/packages/plugins/package.json +++ b/packages/plugins/package.json @@ -3,6 +3,7 @@ "version": "1.0.0", "type": "module", "dependencies": { + "cheerio": "1.0.0-rc.12", "@types/pg": "^8.6.6", "axios": "^1.5.1", "duck-duck-scrape": "^2.2.5", diff --git a/packages/plugins/register.ts b/packages/plugins/register.ts index 3940ff1a1..9989fbc78 100644 --- a/packages/plugins/register.ts +++ b/packages/plugins/register.ts @@ -5,7 +5,15 @@ import { cloneDeep } from 'lodash'; import { WorkerNameEnum, runWorker } from '@fastgpt/service/worker/utils'; // Run in main thread -const staticPluginList = ['getTime', 'fetchUrl', 'feishu', 'google', 'bing']; +const staticPluginList = [ + 'getTime', + 'fetchUrl', + 'feishu', + 'DingTalkWebhook', + 'WeWorkWebhook', + 'google', + 'bing' +]; // Run in worker thread (Have npm packages) const packagePluginList = [ 'mathExprVal', @@ -19,31 +27,34 @@ const packagePluginList = [ 'wiki', 'databaseConnection', 'Doc2X', - 'Doc2X/PDF2text' + 'Doc2X/PDF2text', + 'searchXNG' ]; export const list = [...staticPluginList, ...packagePluginList]; /* Get plugins */ export const getCommunityPlugins = () => { - return list.map((name) => { - const config = require(`./src/${name}/template.json`); + return Promise.all( + list.map>(async (name) => { + const config = (await import(`./src/${name}/template.json`))?.default; - const isFolder = list.find((item) => item.startsWith(`${name}/`)); + const isFolder = list.find((item) => item.startsWith(`${name}/`)); - const parentIdList = name.split('/').slice(0, -1); - const parentId = - parentIdList.length > 0 ? `${PluginSourceEnum.community}-${parentIdList.join('/')}` : null; + const parentIdList = name.split('/').slice(0, -1); + const parentId = + parentIdList.length > 0 ? `${PluginSourceEnum.community}-${parentIdList.join('/')}` : null; - return { - ...config, - id: `${PluginSourceEnum.community}-${name}`, - isFolder, - parentId, - isActive: true, - isOfficial: true - }; - }); + return { + ...config, + id: `${PluginSourceEnum.community}-${name}`, + isFolder, + parentId, + isActive: true, + isOfficial: true + }; + }) + ); }; export const getSystemPluginTemplates = () => { diff --git a/packages/plugins/src/DingTalkWebhook/template.json b/packages/plugins/src/DingTalkWebhook/template.json new file mode 100644 index 000000000..5195cc9ab --- /dev/null +++ b/packages/plugins/src/DingTalkWebhook/template.json @@ -0,0 +1,536 @@ +{ + "author": "", + "version": "4816", + "name": "钉钉 webhook", + "avatar": "plugins/dingding", + "intro": "向钉钉机器人发起 webhook 请求。", + "courseUrl": "https://open.dingtalk.com/document/robots/custom-robot-access", + "showStatus": false, + "weight": 10, + "isTool": true, + "templateType": "communication", + "workflow": { + "nodes": [ + { + "nodeId": "pluginInput", + "name": "插件开始", + "intro": "自定义配置外部输入，使用插件时，仅暴露自定义配置的输入", + "avatar": "core/workflow/template/workflowStart", + "flowNodeType": "pluginInput", + "showStatus": false, + "position": { + "x": 557.4542421888484, + "y": -131.2827008898969 + }, + "version": "481", + "inputs": [ + { + "inputType": "input", + "valueType": "string", + "key": "钉钉机器人地址", + "label": "钉钉机器人地址", + "description": "", + "isToolInput": false, + "defaultValue": "", + "editField": { + "key": true + }, + "dynamicParamDefaultValue": { + "inputType": "reference", + "valueType": "string", + "required": true + }, + "renderTypeList": ["input"], + "required": true, + "canEdit": true, + "value": "", + "list": [] + }, + { + "renderTypeList": ["input", "reference"], + "selectedTypeIndex": 0, + "valueType": "string", + "canEdit": true, + "key": "加签值", + "label": "加签值", + "description": "钉钉机器人加签值", + "defaultValue": "", + "list": [ + { + "label": "", + "value": "" + } + ], + "maxFiles": 5, + "canSelectFile": true, + "canSelectImg": true, + "required": true + }, + { + "renderTypeList": ["input", "reference"], + "selectedTypeIndex": 0, + "valueType": "string", + "canEdit": true, + "key": "发送的消息", + "label": "发送的消息", + "description": "发送的消息", + "defaultValue": "", + "list": [ + { + "label": "", + "value": "" + } + ], + "maxFiles": 5, + "canSelectFile": true, + "canSelectImg": true, + "required": true, + "toolDescription": "发送的消息" + } + ], + "outputs": [ + { + "id": "mv52BrPVE6bm", + "key": "钉钉机器人地址", + "valueType": "string", + "label": "钉钉机器人地址", + "type": "static" + }, + { + "id": "srcret", + "valueType": "string", + "key": "加签值", + "label": "加签值", + "type": "hidden" + }, + { + "id": "发送的消息", + "valueType": "string", + "key": "发送的消息", + "label": "发送的消息", + "type": "hidden" + } + ] + }, + { + "nodeId": "pluginOutput", + "name": "插件输出", + "intro": "自定义配置外部输出，使用插件时，仅暴露自定义配置的输出", + "avatar": "core/workflow/template/pluginOutput", + "flowNodeType": "pluginOutput", + "showStatus": false, + "position": { + "x": 2420.0305926489386, + "y": -106.28270088989689 + }, + "version": "481", + "inputs": [], + "outputs": [] + }, + { + "nodeId": "rKBYGQuYefae", + "name": "HTTP 请求", + "intro": "可以发出一个 HTTP 请求，实现更为复杂的操作（联网搜索、数据库查询等）", + "avatar": "core/workflow/template/httpRequest", + "flowNodeType": "httpRequest468", + "showStatus": true, + "position": { + "x": 1645.779103978597, + "y": -431.7827008898969 + }, + "version": "481", + "inputs": [ + { + "key": "system_addInputParam", + "renderTypeList": ["addInputParam"], + "valueType": "dynamic", + "label": "", + "required": false, + "description": "common:core.module.input.description.HTTP Dynamic Input", + "customInputConfig": { + "selectValueTypeList": [ + "string", + "number", + "boolean", + "object", + "arrayString", + "arrayNumber", + "arrayBoolean", + "arrayObject", + "arrayAny", + "any", + "chatHistory", + "datasetQuote", + "dynamic", + "selectApp", + "selectDataset" + ], + "showDescription": false, + "showDefaultValue": true + }, + "debugLabel": "", + "toolDescription": "" + }, + { + "key": "system_httpMethod", + "renderTypeList": ["custom"], + "valueType": "string", + "label": "", + "value": "POST", + "required": true, + "debugLabel": "", + "toolDescription": "" + }, + { + "key": "system_httpTimeout", + "renderTypeList": ["custom"], + "valueType": "number", + "label": "", + "value": 30, + "min": 5, + "max": 600, + "required": true, + "debugLabel": "", + "toolDescription": "" + }, + { + "key": "system_httpReqUrl", + "renderTypeList": ["hidden"], + "valueType": "string", + "label": "", + "description": "common:core.module.input.description.Http Request Url", + "placeholder": "https://api.ai.com/getInventory", + "required": false, + "value": "{{$a5qdMS7ECNYE.qLUQfhG0ILRX$}}", + "debugLabel": "", + "toolDescription": "" + }, + { + "key": "system_httpHeader", + "renderTypeList": ["custom"], + "valueType": "any", + "value": [], + "label": "", + "description": "common:core.module.input.description.Http Request Header", + "placeholder": "common:core.module.input.description.Http Request Header", + "required": false, + "debugLabel": "", + "toolDescription": "" + }, + { + "key": "system_httpParams", + "renderTypeList": ["hidden"], + "valueType": "any", + "value": [], + "label": "", + "required": false, + "debugLabel": "", + "toolDescription": "" + }, + { + "key": "system_httpJsonBody", + "renderTypeList": ["hidden"], + "valueType": "any", + "value": "{\r\n \"msgtype\": \"text\",\r\n \"text\": {\r\n \"content\": \"{{$pluginInput.发送的消息$}}\"\r\n }\r\n}", + "label": "", + "required": false, + "debugLabel": "", + "toolDescription": "" + }, + { + "key": "system_httpFormBody", + "renderTypeList": ["hidden"], + "valueType": "any", + "value": [], + "label": "", + "required": false, + "debugLabel": "", + "toolDescription": "" + }, + { + "key": "system_httpContentType", + "renderTypeList": ["hidden"], + "valueType": "string", + "value": "json", + "label": "", + "required": false, + "debugLabel": "", + "toolDescription": "" + } + ], + "outputs": [ + { + "id": "error", + "key": "error", + "label": "workflow:request_error", + "description": "HTTP请求错误信息，成功时返回空", + "valueType": "object", + "type": "static" + }, + { + "id": "httpRawResponse", + "key": "httpRawResponse", + "required": true, + "label": "workflow:raw_response", + "description": "HTTP请求的原始响应。只能接受字符串或JSON类型响应数据。", + "valueType": "any", + "type": "static" + }, + { + "id": "system_addOutputParam", + "key": "system_addOutputParam", + "type": "dynamic", + "valueType": "dynamic", + "label": "", + "editField": { + "key": true, + "valueType": true + } + } + ] + }, + { + "nodeId": "q3ccNXiZIHoS", + "name": "系统配置", + "intro": "", + "avatar": "core/workflow/template/systemConfig", + "flowNodeType": "pluginConfig", + "position": { + "x": 99.73879703925843, + "y": -201.26482361861054 + }, + "version": "4811", + "inputs": [], + "outputs": [] + }, + { + "nodeId": "a5qdMS7ECNYE", + "name": "代码运行", + "intro": "执行一段简单的脚本代码，通常用于进行复杂的数据处理。", + "avatar": "core/workflow/template/codeRun", + "flowNodeType": "code", + "showStatus": true, + "position": { + "x": 1106.1011901190363, + "y": -407.7827008898969 + }, + "version": "482", + "inputs": [ + { + "key": "system_addInputParam", + "renderTypeList": ["addInputParam"], + "valueType": "dynamic", + "label": "", + "required": false, + "description": "workflow:these_variables_will_be_input_parameters_for_code_execution", + "customInputConfig": { + "selectValueTypeList": [ + "string", + "number", + "boolean", + "object", + "arrayString", + "arrayNumber", + "arrayBoolean", + "arrayObject", + "arrayAny", + "any", + "chatHistory", + "datasetQuote", + "dynamic", + "selectApp", + "selectDataset" + ], + "showDescription": false, + "showDefaultValue": true + }, + "valueDesc": "", + "debugLabel": "", + "toolDescription": "" + }, + { + "key": "codeType", + "renderTypeList": ["hidden"], + "label": "", + "value": "js", + "valueDesc": "", + "description": "", + "debugLabel": "", + "toolDescription": "" + }, + { + "key": "code", + "renderTypeList": ["custom"], + "label": "", + "value": "function main({url, secret}){\n const {sign,timestamp} = createHmac('sha256',secret)\n\n return {\n result: `${url}×tamp=${timestamp}&sign=${sign}`\n }\n}", + "valueDesc": "", + "description": "", + "debugLabel": "", + "toolDescription": "" + }, + { + "renderTypeList": ["reference"], + "valueType": "string", + "canEdit": true, + "key": "url", + "label": "url", + "customInputConfig": { + "selectValueTypeList": [ + "string", + "number", + "boolean", + "object", + "arrayString", + "arrayNumber", + "arrayBoolean", + "arrayObject", + "arrayAny", + "any", + "chatHistory", + "datasetQuote", + "dynamic", + "selectApp", + "selectDataset" + ], + "showDescription": false, + "showDefaultValue": true + }, + "required": true, + "value": ["pluginInput", "mv52BrPVE6bm"] + }, + { + "renderTypeList": ["reference"], + "valueType": "string", + "canEdit": true, + "key": "secret", + "label": "secret", + "customInputConfig": { + "selectValueTypeList": [ + "string", + "number", + "boolean", + "object", + "arrayString", + "arrayNumber", + "arrayBoolean", + "arrayObject", + "arrayAny", + "any", + "chatHistory", + "datasetQuote", + "dynamic", + "selectApp", + "selectDataset" + ], + "showDescription": false, + "showDefaultValue": true + }, + "required": true, + "value": ["pluginInput", "srcret"] + } + ], + "outputs": [ + { + "id": "system_rawResponse", + "key": "system_rawResponse", + "label": "workflow:full_response_data", + "valueType": "object", + "type": "static", + "description": "" + }, + { + "id": "error", + "key": "error", + "label": "workflow:execution_error", + "description": "代码运行错误信息，成功时返回空", + "valueType": "object", + "type": "static" + }, + { + "id": "system_addOutputParam", + "key": "system_addOutputParam", + "type": "dynamic", + "valueType": "dynamic", + "label": "", + "customFieldConfig": { + "selectValueTypeList": [ + "string", + "number", + "boolean", + "object", + "arrayString", + "arrayNumber", + "arrayBoolean", + "arrayObject", + "arrayAny", + "any", + "chatHistory", + "datasetQuote", + "dynamic", + "selectApp", + "selectDataset" + ], + "showDescription": false, + "showDefaultValue": false + }, + "description": "将代码中 return 的对象作为输出，传递给后续的节点。变量名需要对应 return 的 key", + "valueDesc": "" + }, + { + "id": "qLUQfhG0ILRX", + "type": "dynamic", + "key": "result", + "valueType": "string", + "label": "result", + "valueDesc": "", + "description": "" + } + ] + } + ], + "edges": [ + { + "source": "rKBYGQuYefae", + "target": "pluginOutput", + "sourceHandle": "rKBYGQuYefae-source-right", + "targetHandle": "pluginOutput-target-left" + }, + { + "source": "pluginInput", + "target": "a5qdMS7ECNYE", + "sourceHandle": "pluginInput-source-right", + "targetHandle": "a5qdMS7ECNYE-target-left" + }, + { + "source": "a5qdMS7ECNYE", + "target": "rKBYGQuYefae", + "sourceHandle": "a5qdMS7ECNYE-source-right", + "targetHandle": "rKBYGQuYefae-target-left" + } + ], + "chatConfig": { + "welcomeText": "", + "variables": [], + "questionGuide": false, + "ttsConfig": { + "type": "web" + }, + "whisperConfig": { + "open": false, + "autoSend": false, + "autoTTSResponse": false + }, + "chatInputGuide": { + "open": false, + "textList": [], + "customUrl": "" + }, + "instruction": "", + "autoExecute": { + "open": false, + "defaultPrompt": "" + }, + "_id": "6710a5619c45325525326719" + } + } +} diff --git a/packages/plugins/src/Doc2X/PDF2text/template.json b/packages/plugins/src/Doc2X/PDF2text/template.json index 9a84c51a6..8d2368967 100644 --- a/packages/plugins/src/Doc2X/PDF2text/template.json +++ b/packages/plugins/src/Doc2X/PDF2text/template.json @@ -439,7 +439,9 @@ } ], "chatConfig": { - "questionGuide": false, + "questionGuide": { + "open": false + }, "ttsConfig": { "type": "web" }, diff --git a/packages/plugins/src/WeWorkWebhook/template.json b/packages/plugins/src/WeWorkWebhook/template.json new file mode 100644 index 000000000..1e5f7c2ad --- /dev/null +++ b/packages/plugins/src/WeWorkWebhook/template.json @@ -0,0 +1,386 @@ +{ + "author": "", + "version": "4816", + "name": "企业微信 webhook", + "avatar": "plugins/qiwei", + "intro": "向企业微信机器人发起 webhook 请求。只能内部群使用。", + "courseUrl": "https://developer.work.weixin.qq.com/document/path/91770", + "showStatus": false, + "weight": 10, + "isTool": true, + "templateType": "communication", + "workflow": { + "nodes": [ + { + "nodeId": "pluginInput", + "name": "插件开始", + "intro": "自定义配置外部输入，使用插件时，仅暴露自定义配置的输入", + "avatar": "core/workflow/template/workflowStart", + "flowNodeType": "pluginInput", + "showStatus": false, + "position": { + "x": 537.6357458754286, + "y": -201.26482361861054 + }, + "version": "481", + "inputs": [ + { + "inputType": "input", + "valueType": "string", + "key": "企微机器人地址", + "label": "企微机器人地址", + "description": "", + "isToolInput": false, + "defaultValue": "", + "editField": { + "key": true + }, + "dynamicParamDefaultValue": { + "inputType": "reference", + "valueType": "string", + "required": true + }, + "renderTypeList": ["input"], + "required": true, + "canEdit": true, + "value": "" + }, + { + "key": "发送的消息", + "valueType": "string", + "label": "发送的消息", + "renderTypeList": ["input", "reference"], + "required": true, + "description": "发送的消息", + "canEdit": true, + "value": "", + "editField": { + "key": true + }, + "dynamicParamDefaultValue": { + "inputType": "reference", + "valueType": "string", + "required": true + }, + "list": [], + "defaultValue": "", + "toolDescription": "发送的消息" + } + ], + "outputs": [ + { + "id": "mv52BrPVE6bm", + "key": "企微机器人地址", + "valueType": "string", + "label": "企微机器人地址", + "type": "static" + }, + { + "id": "p0m68Dv5KaIp", + "key": "发送的消息", + "valueType": "string", + "label": "发送的消息", + "type": "static" + } + ] + }, + { + "nodeId": "pluginOutput", + "name": "插件输出", + "intro": "自定义配置外部输出，使用插件时，仅暴露自定义配置的输出", + "avatar": "core/workflow/template/pluginOutput", + "flowNodeType": "pluginOutput", + "showStatus": false, + "position": { + "x": 1776.027569211593, + "y": -58.264823618610535 + }, + "version": "481", + "inputs": [], + "outputs": [] + }, + { + "nodeId": "rKBYGQuYefae", + "name": "HTTP 请求", + "intro": "可以发出一个 HTTP 请求，实现更为复杂的操作（联网搜索、数据库查询等）", + "avatar": "core/workflow/template/httpRequest", + "flowNodeType": "httpRequest468", + "showStatus": true, + "position": { + "x": 1069.7228495148624, + "y": -392.26482361861054 + }, + "version": "481", + "inputs": [ + { + "key": "system_addInputParam", + "renderTypeList": ["addInputParam"], + "valueType": "dynamic", + "label": "", + "required": false, + "description": "common:core.module.input.description.HTTP Dynamic Input", + "customInputConfig": { + "selectValueTypeList": [ + "string", + "number", + "boolean", + "object", + "arrayString", + "arrayNumber", + "arrayBoolean", + "arrayObject", + "arrayAny", + "any", + "chatHistory", + "datasetQuote", + "dynamic", + "selectApp", + "selectDataset" + ], + "showDescription": false, + "showDefaultValue": true + }, + "debugLabel": "", + "toolDescription": "" + }, + { + "key": "system_httpMethod", + "renderTypeList": ["custom"], + "valueType": "string", + "label": "", + "value": "POST", + "required": true, + "debugLabel": "", + "toolDescription": "" + }, + { + "key": "system_httpTimeout", + "renderTypeList": ["custom"], + "valueType": "number", + "label": "", + "value": 30, + "min": 5, + "max": 600, + "required": true, + "debugLabel": "", + "toolDescription": "" + }, + { + "key": "system_httpReqUrl", + "renderTypeList": ["hidden"], + "valueType": "string", + "label": "", + "description": "common:core.module.input.description.Http Request Url", + "placeholder": "https://api.ai.com/getInventory", + "required": false, + "value": "{{url}}", + "debugLabel": "", + "toolDescription": "" + }, + { + "key": "system_httpHeader", + "renderTypeList": ["custom"], + "valueType": "any", + "value": [], + "label": "", + "description": "common:core.module.input.description.Http Request Header", + "placeholder": "common:core.module.input.description.Http Request Header", + "required": false, + "debugLabel": "", + "toolDescription": "" + }, + { + "key": "system_httpParams", + "renderTypeList": ["hidden"], + "valueType": "any", + "value": [], + "label": "", + "required": false, + "debugLabel": "", + "toolDescription": "" + }, + { + "key": "system_httpJsonBody", + "renderTypeList": ["hidden"], + "valueType": "any", + "value": "{\r\n \"msgtype\": \"text\",\r\n \"text\": {\r\n \"content\": \"{{text}}\"\r\n }\r\n}", + "label": "", + "required": false, + "debugLabel": "", + "toolDescription": "" + }, + { + "key": "system_httpFormBody", + "renderTypeList": ["hidden"], + "valueType": "any", + "value": [], + "label": "", + "required": false, + "debugLabel": "", + "toolDescription": "" + }, + { + "key": "system_httpContentType", + "renderTypeList": ["hidden"], + "valueType": "string", + "value": "json", + "label": "", + "required": false, + "debugLabel": "", + "toolDescription": "" + }, + { + "key": "text", + "valueType": "string", + "label": "text", + "renderTypeList": ["reference"], + "description": "", + "canEdit": true, + "editField": { + "key": true, + "valueType": true + }, + "value": ["pluginInput", "p0m68Dv5KaIp"], + "customInputConfig": { + "selectValueTypeList": [ + "string", + "number", + "boolean", + "object", + "arrayString", + "arrayNumber", + "arrayBoolean", + "arrayObject", + "arrayAny", + "any", + "chatHistory", + "datasetQuote", + "dynamic", + "selectApp", + "selectDataset" + ], + "showDescription": false, + "showDefaultValue": true + } + }, + { + "key": "url", + "valueType": "string", + "label": "url", + "renderTypeList": ["reference"], + "description": "", + "canEdit": true, + "editField": { + "key": true, + "valueType": true + }, + "value": ["pluginInput", "mv52BrPVE6bm"], + "customInputConfig": { + "selectValueTypeList": [ + "string", + "number", + "boolean", + "object", + "arrayString", + "arrayNumber", + "arrayBoolean", + "arrayObject", + "arrayAny", + "any", + "chatHistory", + "datasetQuote", + "dynamic", + "selectApp", + "selectDataset" + ], + "showDescription": false, + "showDefaultValue": true + } + } + ], + "outputs": [ + { + "id": "error", + "key": "error", + "label": "workflow:request_error", + "description": "HTTP请求错误信息，成功时返回空", + "valueType": "object", + "type": "static" + }, + { + "id": "httpRawResponse", + "key": "httpRawResponse", + "required": true, + "label": "workflow:raw_response", + "description": "HTTP请求的原始响应。只能接受字符串或JSON类型响应数据。", + "valueType": "any", + "type": "static" + }, + { + "id": "system_addOutputParam", + "key": "system_addOutputParam", + "type": "dynamic", + "valueType": "dynamic", + "label": "", + "editField": { + "key": true, + "valueType": true + } + } + ] + }, + { + "nodeId": "q3ccNXiZIHoS", + "name": "系统配置", + "intro": "", + "avatar": "core/workflow/template/systemConfig", + "flowNodeType": "pluginConfig", + "position": { + "x": 99.73879703925843, + "y": -201.26482361861054 + }, + "version": "4811", + "inputs": [], + "outputs": [] + } + ], + "edges": [ + { + "source": "pluginInput", + "target": "rKBYGQuYefae", + "sourceHandle": "pluginInput-source-right", + "targetHandle": "rKBYGQuYefae-target-left" + }, + { + "source": "rKBYGQuYefae", + "target": "pluginOutput", + "sourceHandle": "rKBYGQuYefae-source-right", + "targetHandle": "pluginOutput-target-left" + } + ], + "chatConfig": { + "welcomeText": "", + "variables": [], + "questionGuide": false, + "ttsConfig": { + "type": "web" + }, + "whisperConfig": { + "open": false, + "autoSend": false, + "autoTTSResponse": false + }, + "chatInputGuide": { + "open": false, + "textList": [], + "customUrl": "" + }, + "instruction": "", + "autoExecute": { + "open": false, + "defaultPrompt": "" + }, + "_id": "6710a5619c45325525326719" + } + } +} diff --git a/packages/plugins/src/bing/template.json b/packages/plugins/src/bing/template.json index 65780248c..0cb410b49 100644 --- a/packages/plugins/src/bing/template.json +++ b/packages/plugins/src/bing/template.json @@ -489,7 +489,9 @@ "chatConfig": { "welcomeText": "", "variables": [], - "questionGuide": false, + "questionGuide": { + "open": false + }, "ttsConfig": { "type": "web" }, diff --git a/packages/plugins/src/databaseConnection/template.json b/packages/plugins/src/databaseConnection/template.json index 6aaa4cbf3..97caec65e 100644 --- a/packages/plugins/src/databaseConnection/template.json +++ b/packages/plugins/src/databaseConnection/template.json @@ -666,7 +666,9 @@ "chatConfig": { "welcomeText": "", "variables": [], - "questionGuide": false, + "questionGuide": { + "open": false + }, "ttsConfig": { "type": "web" }, diff --git a/packages/plugins/src/drawing/baseChart/index.ts b/packages/plugins/src/drawing/baseChart/index.ts index f3e34b7ac..dc05be89d 100644 --- a/packages/plugins/src/drawing/baseChart/index.ts +++ b/packages/plugins/src/drawing/baseChart/index.ts @@ -4,8 +4,8 @@ import { SystemPluginSpecialResponse } from '../../../type.d'; type Props = { title: string; - xAxis: string; - yAxis: string; + xAxis: string[]; + yAxis: string[]; chartType: string; }; @@ -27,7 +27,12 @@ type Option = { series: SeriesData[]; // 使用定义的类型 }; -const generateChart = async (title: string, xAxis: string, yAxis: string, chartType: string) => { +const generateChart = async ( + title: string, + xAxis: string[], + yAxis: string[], + chartType: string +) => { // @ts-ignore 无法使用dom，如使用jsdom会出现生成图片无法正常展示，有高手可以帮忙解决 const chart = echarts.init(undefined, undefined, { renderer: 'svg', // 必须使用 SVG 模式 @@ -36,21 +41,11 @@ const generateChart = async (title: string, xAxis: string, yAxis: string, chartT height: 300 }); - let parsedXAxis: string[] = []; - let parsedYAxis: number[] = []; - try { - parsedXAxis = json5.parse(xAxis); - parsedYAxis = json5.parse(yAxis); - } catch (error: any) { - console.error('解析数据时出错:', error); - return Promise.reject('Data error'); - } - const option: Option = { backgroundColor: '#f5f5f5', title: { text: title }, tooltip: {}, - xAxis: { data: parsedXAxis }, + xAxis: { data: xAxis }, yAxis: {}, series: [] // 初始化为空数组 }; @@ -58,18 +53,18 @@ const generateChart = async (title: string, xAxis: string, yAxis: string, chartT // 根据 chartType 生成不同的图表 switch (chartType) { case '柱状图': - option.series.push({ name: 'Sample', type: 'bar', data: parsedYAxis }); + option.series.push({ name: 'Sample', type: 'bar', data: yAxis.map(Number) }); break; case '折线图': - option.series.push({ name: 'Sample', type: 'line', data: parsedYAxis }); + option.series.push({ name: 'Sample', type: 'line', data: yAxis.map(Number) }); break; case '饼图': option.series.push({ name: 'Sample', type: 'pie', - data: parsedYAxis.map((value, index) => ({ - value, - name: parsedXAxis[index] // 使用 xAxis 作为饼图的名称 + data: yAxis.map((value, index) => ({ + value: Number(value), + name: xAxis[index] // 使用 xAxis 作为饼图的名称 })) }); break; diff --git a/packages/plugins/src/drawing/baseChart/template.json b/packages/plugins/src/drawing/baseChart/template.json index c249ced14..15402be49 100644 --- a/packages/plugins/src/drawing/baseChart/template.json +++ b/packages/plugins/src/drawing/baseChart/template.json @@ -1,6 +1,6 @@ { "author": "silencezhang", - "version": "4812", + "version": "4817", "name": "基础图表", "avatar": "core/workflow/template/baseChart", "intro": "根据数据生成图表，可根据chartType生成柱状图，折线图，饼图", @@ -68,7 +68,7 @@ "canEdit": true, "key": "yAxis", "label": "yAxis", - "description": "y轴数据，例如：['1', '2', '3']", + "description": "y轴数据，例如：[1,2,3]", "defaultValue": "", "list": [ { @@ -77,7 +77,7 @@ } ], "required": true, - "toolDescription": "y轴数据，例如：['1', '2', '3']" + "toolDescription": "y轴数据，例如：[1,2,3]" }, { "renderTypeList": ["select", "reference"], @@ -145,8 +145,8 @@ "flowNodeType": "pluginOutput", "showStatus": false, "position": { - "x": 2122.252754006148, - "y": -63.5218674613718 + "x": 2128.8138851197145, + "y": -63.52186746137181 }, "version": "481", "inputs": [ @@ -154,10 +154,12 @@ "renderTypeList": ["reference"], "valueType": "string", "canEdit": true, - "key": "相对路径URL", - "label": "相对路径URL", + "key": "图表 url", + "label": "图表 url", "description": "可用使用markdown格式展示图片，如：![图片](url)", - "value": ["ws0DFKJnCPhk", "bzaYjKyQFOw2"] + "value": ["ws0DFKJnCPhk", "bzaYjKyQFOw2"], + "isToolOutput": true, + "required": true } ], "outputs": [] @@ -170,8 +172,8 @@ "flowNodeType": "httpRequest468", "showStatus": true, "position": { - "x": 1216.5166647574395, - "y": -206.30162946606856 + "x": 1264.2009472531117, + "y": -455.0773486762623 }, "version": "481", "inputs": [ @@ -275,7 +277,7 @@ "key": "system_httpJsonBody", "renderTypeList": ["hidden"], "valueType": "any", - "value": "{\r\n \"title\": \"{{title-plugin}}\",\r\n \"xAxis\": \"{{xAxis-plugin}}\",\r\n \"yAxis\": \"{{yAxis-plugin}}\",\r\n \"chartType\": \"{{chartType-plugin}}\"\r\n}", + "value": "{\r\n \"title\": \"{{$pluginInput.title$}}\",\r\n \"xAxis\": {{$pluginInput.xAxis$}},\r\n \"yAxis\": {{$pluginInput.yAxis$}},\r\n \"chartType\": \"{{$pluginInput.chartType$}}\"\r\n}", "label": "", "required": false, "valueDesc": "", @@ -306,126 +308,6 @@ "description": "", "debugLabel": "", "toolDescription": "" - }, - { - "renderTypeList": ["reference"], - "valueType": "string", - "canEdit": true, - "key": "title-plugin", - "label": "title-plugin", - "customInputConfig": { - "selectValueTypeList": [ - "string", - "number", - "boolean", - "object", - "arrayString", - "arrayNumber", - "arrayBoolean", - "arrayObject", - "arrayAny", - "any", - "chatHistory", - "datasetQuote", - "dynamic", - "selectApp", - "selectDataset" - ], - "showDescription": false, - "showDefaultValue": true - }, - "required": true, - "value": ["pluginInput", "title"] - }, - { - "renderTypeList": ["reference"], - "valueType": "string", - "canEdit": true, - "key": "xAxis-plugin", - "label": "xAxis-plugin", - "customInputConfig": { - "selectValueTypeList": [ - "string", - "number", - "boolean", - "object", - "arrayString", - "arrayNumber", - "arrayBoolean", - "arrayObject", - "arrayAny", - "any", - "chatHistory", - "datasetQuote", - "dynamic", - "selectApp", - "selectDataset" - ], - "showDescription": false, - "showDefaultValue": true - }, - "required": true, - "value": ["pluginInput", "xAxis"] - }, - { - "renderTypeList": ["reference"], - "valueType": "string", - "canEdit": true, - "key": "yAxis-plugin", - "label": "yAxis-plugin", - "customInputConfig": { - "selectValueTypeList": [ - "string", - "number", - "boolean", - "object", - "arrayString", - "arrayNumber", - "arrayBoolean", - "arrayObject", - "arrayAny", - "any", - "chatHistory", - "datasetQuote", - "dynamic", - "selectApp", - "selectDataset" - ], - "showDescription": false, - "showDefaultValue": true - }, - "required": true, - "value": ["pluginInput", "yAxis"] - }, - { - "renderTypeList": ["reference"], - "valueType": "string", - "canEdit": true, - "key": "chartType-plugin", - "label": "chartType-plugin", - "customInputConfig": { - "selectValueTypeList": [ - "string", - "number", - "boolean", - "object", - "arrayString", - "arrayNumber", - "arrayBoolean", - "arrayObject", - "arrayAny", - "any", - "chatHistory", - "datasetQuote", - "dynamic", - "selectApp", - "selectDataset" - ], - "showDescription": false, - "showDefaultValue": true - }, - "required": true, - "value": ["pluginInput", "chartType"] } ], "outputs": [ @@ -502,7 +384,9 @@ "chatConfig": { "welcomeText": "", "variables": [], - "questionGuide": false, + "questionGuide": { + "open": false + }, "ttsConfig": { "type": "web" }, diff --git a/packages/plugins/src/feishu/template.json b/packages/plugins/src/feishu/template.json index 947eb4793..3fca64acc 100644 --- a/packages/plugins/src/feishu/template.json +++ b/packages/plugins/src/feishu/template.json @@ -1,8 +1,8 @@ { "author": "", "version": "488", - "name": "飞书机器人 webhook", - "avatar": "/appMarketTemplates/plugin-feishu/avatar.svg", + "name": "飞书 webhook", + "avatar": "core/app/templates/plugin-feishu", "intro": "向飞书机器人发起 webhook 请求。", "courseUrl": "https://open.feishu.cn/document/client-docs/bot-v3/add-custom-bot#f62e72d5", "showStatus": false, @@ -27,7 +27,7 @@ "version": "481", "inputs": [ { - "renderTypeList": ["reference"], + "renderTypeList": ["input", "reference"], "selectedTypeIndex": 0, "valueType": "string", "canEdit": true, @@ -35,7 +35,9 @@ "label": "content", "description": "需要发送的消息", "required": true, - "toolDescription": "需要发送的消息" + "toolDescription": "需要发送的消息", + "list": [], + "defaultValue": "" }, { "renderTypeList": ["input"], @@ -46,7 +48,8 @@ "label": "hook_url", "description": "飞书机器人地址", "required": true, - "defaultValue": "" + "defaultValue": "", + "list": [] } ], "outputs": [ @@ -508,6 +511,30 @@ "sourceHandle": "qcJpBBVtXsGd-source-right", "targetHandle": "vzreK6vHrPvZ-target-left" } - ] + ], + "chatConfig": { + "welcomeText": "", + "variables": [], + "questionGuide": false, + "ttsConfig": { + "type": "web" + }, + "whisperConfig": { + "open": false, + "autoSend": false, + "autoTTSResponse": false + }, + "chatInputGuide": { + "open": false, + "textList": [], + "customUrl": "" + }, + "instruction": "", + "autoExecute": { + "open": false, + "defaultPrompt": "" + }, + "_id": "6710a5619c45325525326719" + } } } diff --git a/packages/plugins/src/google/template.json b/packages/plugins/src/google/template.json index 9ceab6be2..7009049ae 100644 --- a/packages/plugins/src/google/template.json +++ b/packages/plugins/src/google/template.json @@ -538,7 +538,9 @@ "chatConfig": { "welcomeText": "", "variables": [], - "questionGuide": false, + "questionGuide": { + "open": false + }, "ttsConfig": { "type": "web" }, diff --git a/packages/plugins/src/searchXNG/index.ts b/packages/plugins/src/searchXNG/index.ts new file mode 100644 index 000000000..eebe87adc --- /dev/null +++ b/packages/plugins/src/searchXNG/index.ts @@ -0,0 +1,76 @@ +import { delay } from '@fastgpt/global/common/system/utils'; +import * as cheerio from 'cheerio'; +import { i18nT } from '../../../web/i18n/utils'; + +type Props = { + query: string; + url: string; +}; + +interface SearchResult { + title: string; + link: string; + snippet: string; +} + +type Response = Promise<{ + result: string; + error?: Record; +}>; + +const main = async (props: Props, retry = 3): Response => { + const { query, url } = props; + + if (!query) { + return Promise.reject(i18nT('chat:not_query')); + } + + if (!url) { + return Promise.reject('Can not find url'); + } + + try { + const response = await fetch(`${url}?q=${encodeURIComponent(query)}&language=auto`); + const html = await response.text(); + const $ = cheerio.load(html, { + xml: false, + decodeEntities: true + }); + + const results: SearchResult[] = []; + + $('.result').each((_: number, element: cheerio.Element) => { + const $element = $(element); + results.push({ + title: $element.find('h3').text().trim(), + link: $element.find('a').first().attr('href') || '', + snippet: $element.find('.content').text().trim() + }); + }); + + if (results.length === 0) { + return { + result: JSON.stringify([]), + error: { + message: 'No search results', + code: 500 + } + }; + } + + return { + result: JSON.stringify(results.slice(0, 10)) + }; + } catch (error) { + console.log(error); + if (retry <= 0) { + console.log('Search XNG error', { error }); + return Promise.reject('Failed to fetch data from Search XNG'); + } + + await delay(Math.random() * 2000); + return main(props, retry - 1); + } +}; + +export default main; diff --git a/packages/plugins/src/searchXNG/template.json b/packages/plugins/src/searchXNG/template.json new file mode 100644 index 000000000..639c19813 --- /dev/null +++ b/packages/plugins/src/searchXNG/template.json @@ -0,0 +1,413 @@ +{ + "author": "", + "version": "4816", + "name": "Search XNG 搜索", + "avatar": "core/workflow/template/searxng", + "intro": "使用 Search XNG 服务进行搜索。", + "courseUrl": "/docs/guide/plugins/searxng_plugin_guide/", + "showStatus": true, + "weight": 10, + + "isTool": true, + "templateType": "search", + + "workflow": { + "nodes": [ + { + "nodeId": "pluginInput", + "name": "插件开始", + "intro": "可以配置插件需要哪些输入，利用这些输入来运行插件", + "avatar": "core/workflow/template/workflowStart", + "flowNodeType": "pluginInput", + "showStatus": false, + "position": { + "x": 482.8986144681427, + "y": -77.56127656499825 + }, + "version": "481", + "inputs": [ + { + "renderTypeList": ["input", "reference"], + "selectedTypeIndex": 0, + "valueType": "string", + "canEdit": true, + "key": "query", + "label": "query", + "description": "检索词", + "required": true, + "toolDescription": "检索词" + }, + { + "renderTypeList": ["input", "reference"], + "selectedTypeIndex": 0, + "valueType": "string", + "canEdit": true, + "key": "url", + "label": "url", + "description": "部署的searXNG服务的链接", + "defaultValue": "", + "list": [ + { + "label": "", + "value": "" + } + ], + "maxFiles": 5, + "canSelectFile": true, + "canSelectImg": true, + "required": true + } + ], + "outputs": [ + { + "id": "query", + "valueType": "string", + "key": "query", + "label": "query", + "type": "hidden" + }, + { + "id": "url", + "valueType": "string", + "key": "url", + "label": "url", + "type": "hidden" + } + ] + }, + { + "nodeId": "pluginOutput", + "name": "插件输出", + "intro": "自定义配置外部输出，使用插件时，仅暴露自定义配置的输出", + "avatar": "core/workflow/template/pluginOutput", + "flowNodeType": "pluginOutput", + "showStatus": false, + "position": { + "x": 1763.7518709731714, + "y": -55.56127656499825 + }, + "version": "481", + "inputs": [ + { + "renderTypeList": ["reference"], + "valueType": "string", + "canEdit": true, + "key": "result", + "label": "result", + "description": " 检索结果", + "value": ["hjnVuJAOwyXV", "lEyy5QqyIBrK"] + }, + { + "renderTypeList": ["reference"], + "valueType": "object", + "canEdit": true, + "key": "error", + "label": "error", + "isToolOutput": true, + "description": "请求错误", + "required": true, + "value": ["hjnVuJAOwyXV", "error"] + } + ], + "outputs": [] + }, + { + "nodeId": "hjnVuJAOwyXV", + "name": "HTTP 请求", + "intro": "可以发出一个 HTTP 请求，实现更为复杂的操作（联网搜索、数据库查询等）", + "avatar": "core/workflow/template/httpRequest", + "flowNodeType": "httpRequest468", + "showStatus": true, + "position": { + "x": 1054.6774638324207, + "y": -403.06127656499825 + }, + "version": "481", + "inputs": [ + { + "key": "system_addInputParam", + "renderTypeList": ["addInputParam"], + "valueType": "dynamic", + "label": "", + "required": false, + "description": "common:core.module.input.description.HTTP Dynamic Input", + "customInputConfig": { + "selectValueTypeList": [ + "string", + "number", + "boolean", + "object", + "arrayString", + "arrayNumber", + "arrayBoolean", + "arrayObject", + "arrayAny", + "any", + "chatHistory", + "datasetQuote", + "dynamic", + "selectApp", + "selectDataset" + ], + "showDescription": false, + "showDefaultValue": true + }, + "debugLabel": "", + "toolDescription": "" + }, + { + "key": "system_httpMethod", + "renderTypeList": ["custom"], + "valueType": "string", + "label": "", + "value": "POST", + "required": true, + "debugLabel": "", + "toolDescription": "" + }, + { + "key": "system_httpTimeout", + "renderTypeList": ["custom"], + "valueType": "number", + "label": "", + "value": 30, + "min": 5, + "max": 600, + "required": true, + "debugLabel": "", + "toolDescription": "" + }, + { + "key": "system_httpReqUrl", + "renderTypeList": ["hidden"], + "valueType": "string", + "label": "", + "description": "common:core.module.input.description.Http Request Url", + "placeholder": "https://api.ai.com/getInventory", + "required": false, + "value": "searchXNG", + "debugLabel": "", + "toolDescription": "" + }, + { + "key": "system_httpHeader", + "renderTypeList": ["custom"], + "valueType": "any", + "value": [], + "label": "", + "description": "common:core.module.input.description.Http Request Header", + "placeholder": "common:core.module.input.description.Http Request Header", + "required": false, + "debugLabel": "", + "toolDescription": "" + }, + { + "key": "system_httpParams", + "renderTypeList": ["hidden"], + "valueType": "any", + "value": [], + "label": "", + "required": false, + "debugLabel": "", + "toolDescription": "" + }, + { + "key": "system_httpJsonBody", + "renderTypeList": ["hidden"], + "valueType": "any", + "value": "{\n \"query\": \"{{query}}\",\n \"url\": \"{{url}}\"\n}", + "label": "", + "required": false, + "debugLabel": "", + "toolDescription": "" + }, + { + "key": "system_httpFormBody", + "renderTypeList": ["hidden"], + "valueType": "any", + "value": [], + "label": "", + "required": false, + "debugLabel": "", + "toolDescription": "" + }, + { + "key": "system_httpContentType", + "renderTypeList": ["hidden"], + "valueType": "string", + "value": "json", + "label": "", + "required": false, + "debugLabel": "", + "toolDescription": "" + }, + { + "renderTypeList": ["reference"], + "valueType": "string", + "canEdit": true, + "key": "query", + "label": "query", + "customInputConfig": { + "selectValueTypeList": [ + "string", + "number", + "boolean", + "object", + "arrayString", + "arrayNumber", + "arrayBoolean", + "arrayObject", + "arrayAny", + "any", + "chatHistory", + "datasetQuote", + "dynamic", + "selectApp", + "selectDataset" + ], + "showDescription": false, + "showDefaultValue": true + }, + "required": true, + "value": ["pluginInput", "query"] + }, + { + "renderTypeList": ["reference"], + "valueType": "string", + "canEdit": true, + "key": "url", + "label": "url", + "customInputConfig": { + "selectValueTypeList": [ + "string", + "number", + "boolean", + "object", + "arrayString", + "arrayNumber", + "arrayBoolean", + "arrayObject", + "arrayAny", + "any", + "chatHistory", + "datasetQuote", + "dynamic", + "selectApp", + "selectDataset" + ], + "showDescription": false, + "showDefaultValue": true + }, + "required": true, + "value": ["pluginInput", "url"] + } + ], + "outputs": [ + { + "id": "error", + "key": "error", + "label": "workflow:request_error", + "description": "HTTP请求错误信息，成功时返回空", + "valueType": "object", + "type": "static" + }, + { + "id": "httpRawResponse", + "key": "httpRawResponse", + "required": true, + "label": "workflow:raw_response", + "description": "HTTP请求的原始响应。只能接受字符串或JSON类型响应数据。", + "valueType": "any", + "type": "static" + }, + { + "id": "system_addOutputParam", + "key": "system_addOutputParam", + "type": "dynamic", + "valueType": "dynamic", + "label": "", + "customFieldConfig": { + "selectValueTypeList": [ + "string", + "number", + "boolean", + "object", + "arrayString", + "arrayNumber", + "arrayBoolean", + "arrayObject", + "any", + "chatHistory", + "datasetQuote", + "dynamic", + "selectApp", + "selectDataset" + ], + "showDescription": false, + "showDefaultValue": false + } + }, + { + "id": "lEyy5QqyIBrK", + "valueType": "string", + "type": "dynamic", + "key": "result", + "label": "result" + } + ] + }, + { + "nodeId": "f1mRh1D85H2D", + "name": "系统配置", + "intro": "", + "avatar": "core/workflow/template/systemConfig", + "flowNodeType": "pluginConfig", + "position": { + "x": -28.511358745511643, + "y": -103.56127656499825 + }, + "version": "4811", + "inputs": [], + "outputs": [] + } + ], + "edges": [ + { + "source": "pluginInput", + "target": "hjnVuJAOwyXV", + "sourceHandle": "pluginInput-source-right", + "targetHandle": "hjnVuJAOwyXV-target-left" + }, + { + "source": "hjnVuJAOwyXV", + "target": "pluginOutput", + "sourceHandle": "hjnVuJAOwyXV-source-right", + "targetHandle": "pluginOutput-target-left" + } + ], + "chatConfig": { + "welcomeText": "", + "variables": [], + "questionGuide": false, + "ttsConfig": { + "type": "web" + }, + "whisperConfig": { + "open": false, + "autoSend": false, + "autoTTSResponse": false + }, + "chatInputGuide": { + "open": false, + "textList": [], + "customUrl": "" + }, + "instruction": "", + "autoExecute": { + "open": false, + "defaultPrompt": "" + }, + "_id": "67611a5eed7efa452a6e50c5" + } + } +} diff --git a/packages/plugins/src/wiki/template.json b/packages/plugins/src/wiki/template.json index c9a98e427..c52da3d9c 100644 --- a/packages/plugins/src/wiki/template.json +++ b/packages/plugins/src/wiki/template.json @@ -320,7 +320,9 @@ "chatConfig": { "welcomeText": "", "variables": [], - "questionGuide": false, + "questionGuide": { + "open": false + }, "ttsConfig": { "type": "web" }, diff --git a/packages/service/common/file/constants.ts b/packages/service/common/file/constants.ts index a3449c4be..3245e5f8d 100644 --- a/packages/service/common/file/constants.ts +++ b/packages/service/common/file/constants.ts @@ -1,6 +1,6 @@ import path from 'path'; +import { isProduction } from '@fastgpt/global/common/system/constants'; -export const tmpFileDirPath = - process.env.NODE_ENV === 'production' ? '/app/tmp' : path.join(process.cwd(), 'tmp'); +export const tmpFileDirPath = isProduction ? '/app/tmp' : path.join(process.cwd(), 'tmp'); export const previewMaxCharCount = 3000; diff --git a/packages/service/common/file/gridfs/controller.ts b/packages/service/common/file/gridfs/controller.ts index a5693e39b..af304714c 100644 --- a/packages/service/common/file/gridfs/controller.ts +++ b/packages/service/common/file/gridfs/controller.ts @@ -4,7 +4,7 @@ import fsp from 'fs/promises'; import fs from 'fs'; import { DatasetFileSchema } from '@fastgpt/global/core/dataset/type'; import { MongoChatFileSchema, MongoDatasetFileSchema } from './schema'; -import { detectFileEncoding } from '@fastgpt/global/common/file/tools'; +import { detectFileEncoding, detectFileEncodingByPath } from '@fastgpt/global/common/file/tools'; import { CommonErrEnum } from '@fastgpt/global/common/error/code/common'; import { MongoRawTextBuffer } from '../../buffer/rawText/schema'; import { readRawContentByFileBuffer } from '../read/utils'; @@ -36,7 +36,6 @@ export async function uploadFile({ path, filename, contentType, - encoding, metadata = {} }: { bucketName: `${BucketNameEnum}`; @@ -45,7 +44,6 @@ export async function uploadFile({ path: string; filename: string; contentType?: string; - encoding: string; metadata?: Record; }) { if (!path) return Promise.reject(`filePath is empty`); @@ -59,7 +57,7 @@ export async function uploadFile({ // Add default metadata metadata.teamId = teamId; metadata.uid = uid; - metadata.encoding = encoding; + metadata.encoding = await detectFileEncodingByPath(path); // create a gridfs bucket const bucket = getGridBucket(bucketName); diff --git a/packages/service/common/file/image/controller.ts b/packages/service/common/file/image/controller.ts index 0c9110882..27bc7ade5 100644 --- a/packages/service/common/file/image/controller.ts +++ b/packages/service/common/file/image/controller.ts @@ -1,43 +1,92 @@ import { UploadImgProps } from '@fastgpt/global/common/file/api'; import { imageBaseUrl } from '@fastgpt/global/common/file/image/constants'; import { MongoImage } from './schema'; -import { ClientSession } from '../../../common/mongo'; +import { ClientSession, Types } from '../../../common/mongo'; import { guessBase64ImageType } from '../utils'; import { readFromSecondary } from '../../mongo/utils'; +import { addHours } from 'date-fns'; export const maxImgSize = 1024 * 1024 * 12; const base64MimeRegex = /data:image\/([^\)]+);base64/; export async function uploadMongoImg({ - type, base64Img, teamId, - expiredTime, metadata, - shareId + shareId, + forever = false }: UploadImgProps & { teamId: string; + forever?: Boolean; }) { if (base64Img.length > maxImgSize) { return Promise.reject('Image too large'); } const [base64Mime, base64Data] = base64Img.split(','); + // Check if mime type is valid + if (!base64MimeRegex.test(base64Mime)) { + return Promise.reject('Invalid image mime type'); + } + const mime = `image/${base64Mime.match(base64MimeRegex)?.[1] ?? 'image/jpeg'}`; const binary = Buffer.from(base64Data, 'base64'); const extension = mime.split('/')[1]; const { _id } = await MongoImage.create({ - type, teamId, binary, - expiredTime, metadata: Object.assign({ mime }, metadata), - shareId + shareId, + expiredTime: forever ? undefined : addHours(new Date(), 1) }); - return `${process.env.FE_DOMAIN || ''}${imageBaseUrl}${String(_id)}.${extension}`; + return `${process.env.FE_DOMAIN || ''}${process.env.NEXT_PUBLIC_BASE_URL || ''}${imageBaseUrl}${String(_id)}.${extension}`; } +const getIdFromPath = (path?: string) => { + if (!path) return; + + const paths = path.split('/'); + const name = paths[paths.length - 1]; + + if (!name) return; + + const id = name.split('.')[0]; + if (!id || !Types.ObjectId.isValid(id)) return; + + return id; +}; +// 删除旧的头像，新的头像去除过期时间 +export const refreshSourceAvatar = async ( + path?: string, + oldPath?: string, + session?: ClientSession +) => { + const newId = getIdFromPath(path); + const oldId = getIdFromPath(oldPath); + + if (!newId) return; + + await MongoImage.updateOne({ _id: newId }, { $unset: { expiredTime: 1 } }, { session }); + + if (oldId) { + await MongoImage.deleteOne({ _id: oldId }, { session }); + } +}; +export const removeImageByPath = (path?: string, session?: ClientSession) => { + if (!path) return; + + const paths = path.split('/'); + const name = paths[paths.length - 1]; + + if (!name) return; + + const id = name.split('.')[0]; + if (!id || !Types.ObjectId.isValid(id)) return; + + return MongoImage.deleteOne({ _id: id }, { session }); +}; + export async function readMongoImg({ id }: { id: string }) { const formatId = id.replace(/\.[^/.]+$/, ''); diff --git a/packages/service/common/file/image/schema.ts b/packages/service/common/file/image/schema.ts index f5c62d5cd..1418fa479 100644 --- a/packages/service/common/file/image/schema.ts +++ b/packages/service/common/file/image/schema.ts @@ -1,8 +1,7 @@ import { TeamCollectionName } from '@fastgpt/global/support/user/team/constant'; -import { connectionMongo, getMongoModel, type Model } from '../../mongo'; +import { connectionMongo, getMongoModel } from '../../mongo'; import { MongoImageSchemaType } from '@fastgpt/global/common/file/image/type.d'; -import { mongoImageTypeMap } from '@fastgpt/global/common/file/image/constants'; -const { Schema, model, models } = connectionMongo; +const { Schema } = connectionMongo; const ImageSchema = new Schema({ teamId: { @@ -14,27 +13,15 @@ const ImageSchema = new Schema({ type: Date, default: () => new Date() }, - expiredTime: { - type: Date - }, - binary: { - type: Buffer - }, - type: { - type: String, - enum: Object.keys(mongoImageTypeMap), - required: true - }, - metadata: { - type: Object - } + expiredTime: Date, + binary: Buffer, + metadata: Object }); try { // tts expired（60 Minutes） ImageSchema.index({ expiredTime: 1 }, { expireAfterSeconds: 60 * 60 }); ImageSchema.index({ type: 1 }); - ImageSchema.index({ createTime: 1 }); // delete related img ImageSchema.index({ teamId: 1, 'metadata.relatedId': 1 }); } catch (error) { diff --git a/packages/service/common/file/read/utils.ts b/packages/service/common/file/read/utils.ts index 0c3cb1ba9..7f41f6c47 100644 --- a/packages/service/common/file/read/utils.ts +++ b/packages/service/common/file/read/utils.ts @@ -1,5 +1,4 @@ import { uploadMongoImg } from '../image/controller'; -import { MongoImageTypeEnum } from '@fastgpt/global/common/file/image/constants'; import FormData from 'form-data'; import { WorkerNameEnum, runWorker } from '../../../worker/utils'; @@ -8,7 +7,6 @@ import type { ReadFileResponse } from '../../../worker/readFile/type'; import axios from 'axios'; import { addLog } from '../../system/log'; import { batchRun } from '@fastgpt/global/common/fn/utils'; -import { addHours } from 'date-fns'; import { matchMdImgTextAndUpload } from '@fastgpt/global/common/string/markdown'; export type readRawTextByLocalFileParams = { @@ -22,7 +20,7 @@ export const readRawTextByLocalFile = async (params: readRawTextByLocalFileParam const extension = path?.split('.')?.pop()?.toLowerCase() || ''; - const buffer = fs.readFileSync(path); + const buffer = await fs.promises.readFile(path); const { rawText } = await readRawContentByFileBuffer({ extension, @@ -114,10 +112,9 @@ export const readRawContentByFileBuffer = async ({ if (imageList) { await batchRun(imageList, async (item) => { const src = await uploadMongoImg({ - type: MongoImageTypeEnum.collectionImage, base64Img: `data:${item.mime};base64,${item.base64}`, teamId, - expiredTime: addHours(new Date(), 1), + // expiredTime: addHours(new Date(), 1), metadata: { ...metadata, mime: item.mime diff --git a/packages/service/common/file/utils.ts b/packages/service/common/file/utils.ts index 938187f96..3a623e597 100644 --- a/packages/service/common/file/utils.ts +++ b/packages/service/common/file/utils.ts @@ -1,4 +1,4 @@ -import { isProduction } from '../system/constants'; +import { isProduction } from '@fastgpt/global/common/system/constants'; import fs from 'fs'; import path from 'path'; diff --git a/packages/service/common/middle/cors.ts b/packages/service/common/middle/cors.ts index 775dc31e1..73b294162 100644 --- a/packages/service/common/middle/cors.ts +++ b/packages/service/common/middle/cors.ts @@ -3,10 +3,13 @@ import NextCors from 'nextjs-cors'; export async function withNextCors(req: NextApiRequest, res: NextApiResponse) { const methods = ['GET', 'eHEAD', 'PUT', 'PATCH', 'POST', 'DELETE']; + + const allowedOrigins = process.env.ALLOWED_ORIGINS?.split(','); const origin = req.headers.origin; + await NextCors(req, res, { methods, - origin: origin, + origin: allowedOrigins || origin, optionsSuccessStatus: 200 }); } diff --git a/packages/service/common/middle/reqFrequencyLimit.ts b/packages/service/common/middle/reqFrequencyLimit.ts index 8f950c75d..de7249483 100644 --- a/packages/service/common/middle/reqFrequencyLimit.ts +++ b/packages/service/common/middle/reqFrequencyLimit.ts @@ -9,10 +9,10 @@ import { jsonRes } from '../response'; // unit: times/s // how to use? // export default NextAPI(useQPSLimit(10), handler); // limit 10 times per second for a ip -export function useReqFrequencyLimit(seconds: number, limit: number) { +export function useReqFrequencyLimit(seconds: number, limit: number, force = false) { return async (req: ApiRequestProps, res: NextApiResponse) => { const ip = requestIp.getClientIp(req); - if (!ip || process.env.USE_IP_LIMIT !== 'true') { + if (!ip || (process.env.USE_IP_LIMIT !== 'true' && !force)) { return; } try { @@ -22,10 +22,9 @@ export function useReqFrequencyLimit(seconds: number, limit: number) { expiredTime: addSeconds(new Date(), seconds) }); } catch (_) { - res.status(429); jsonRes(res, { code: 429, - message: ERROR_ENUM.tooManyRequest + error: ERROR_ENUM.tooManyRequest }); } }; diff --git a/packages/service/common/middle/tracks/schema.ts b/packages/service/common/middle/tracks/schema.ts new file mode 100644 index 000000000..5d9b3adee --- /dev/null +++ b/packages/service/common/middle/tracks/schema.ts @@ -0,0 +1,20 @@ +import { TrackSchemaType } from '@fastgpt/global/common/middle/tracks/type'; +import { getMongoModel, Schema } from '../../mongo'; +import { TrackEnum } from '@fastgpt/global/common/middle/tracks/constants'; + +const TrackSchema = new Schema({ + event: { type: String, required: true, enum: Object.values(TrackEnum) }, + uid: String, + teamId: String, + tmbId: String, + createTime: { type: Date, default: () => new Date() }, + data: Object +}); + +try { + TrackSchema.index({ event: 1 }); +} catch (error) { + console.log(error); +} + +export const TrackModel = getMongoModel('track', TrackSchema); diff --git a/packages/service/common/middle/tracks/utils.ts b/packages/service/common/middle/tracks/utils.ts new file mode 100644 index 000000000..e56cd77c9 --- /dev/null +++ b/packages/service/common/middle/tracks/utils.ts @@ -0,0 +1,62 @@ +import { PushTrackCommonType } from '@fastgpt/global/common/middle/tracks/type'; +import { TrackModel } from './schema'; +import { TrackEnum } from '@fastgpt/global/common/middle/tracks/constants'; +import { addLog } from '../../system/log'; +import { OAuthEnum } from '@fastgpt/global/support/user/constant'; +import { AppTypeEnum } from '@fastgpt/global/core/app/constants'; +import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constants'; +import { getAppLatestVersion } from '../../../core/app/version/controller'; + +const createTrack = ({ event, data }: { event: TrackEnum; data: Record }) => { + if (!global.feConfigs?.isPlus) return; + addLog.info('Push tracks', { + event, + ...data + }); + + const { uid, teamId, tmbId, ...props } = data; + + return TrackModel.create({ + event, + uid, + teamId, + tmbId, + data: props + }); +}; +export const pushTrack = { + login: (data: PushTrackCommonType & { type: `${OAuthEnum}` | 'password' }) => { + return createTrack({ + event: TrackEnum.login, + data + }); + }, + createApp: (data: PushTrackCommonType & { type: AppTypeEnum }) => { + return createTrack({ + event: TrackEnum.createApp, + data + }); + }, + createDataset: (data: PushTrackCommonType & { type: DatasetTypeEnum }) => { + return createTrack({ + event: TrackEnum.createDataset, + data + }); + }, + countAppNodes: async (data: PushTrackCommonType & { appId: string }) => { + try { + const { nodes } = await getAppLatestVersion(data.appId); + const nodeTypeList = nodes.map((node) => ({ + type: node.flowNodeType, + pluginId: node.pluginId + })); + return createTrack({ + event: TrackEnum.appNodes, + data: { + ...data, + nodeTypeList + } + }); + } catch (error) {} + } +}; diff --git a/packages/service/common/response/index.ts b/packages/service/common/response/index.ts index a6586d077..a6de2e38a 100644 --- a/packages/service/common/response/index.ts +++ b/packages/service/common/response/index.ts @@ -33,7 +33,7 @@ export const jsonRes = ( addLog.error(`Api response error: ${url}`, ERROR_RESPONSE[errResponseKey]); - return res.json(ERROR_RESPONSE[errResponseKey]); + return res.status(code).json(ERROR_RESPONSE[errResponseKey]); } // another error diff --git a/packages/service/common/system/config/controller.ts b/packages/service/common/system/config/controller.ts index 962be6b37..aa79f916d 100644 --- a/packages/service/common/system/config/controller.ts +++ b/packages/service/common/system/config/controller.ts @@ -4,7 +4,12 @@ import { FastGPTConfigFileType } from '@fastgpt/global/common/system/types'; import { FastGPTProUrl } from '../constants'; export const getFastGPTConfigFromDB = async () => { - if (!FastGPTProUrl) return {} as FastGPTConfigFileType; + if (!FastGPTProUrl) { + return { + config: {} as FastGPTConfigFileType, + configId: undefined + }; + } const res = await MongoSystemConfigs.findOne({ type: SystemConfigsTypeEnum.fastgpt @@ -14,5 +19,8 @@ export const getFastGPTConfigFromDB = async () => { const config = res?.value || {}; - return config as FastGPTConfigFileType; + return { + configId: res ? String(res._id) : undefined, + config: config as FastGPTConfigFileType + }; }; diff --git a/packages/service/common/system/constants.ts b/packages/service/common/system/constants.ts index af250cfdc..1fe15f292 100644 --- a/packages/service/common/system/constants.ts +++ b/packages/service/common/system/constants.ts @@ -1,3 +1 @@ export const FastGPTProUrl = process.env.PRO_URL ? `${process.env.PRO_URL}/api` : ''; - -export const isProduction = process.env.NODE_ENV === 'production'; diff --git a/packages/service/common/system/tools.ts b/packages/service/common/system/tools.ts index 75d29825e..4d195c8e1 100644 --- a/packages/service/common/system/tools.ts +++ b/packages/service/common/system/tools.ts @@ -15,6 +15,9 @@ export const initFastGPTConfig = (config?: FastGPTConfigFileType) => { global.subPlans = config.subPlans; global.llmModels = config.llmModels; + global.llmModelPriceType = global.llmModels.some((item) => typeof item.inputPrice === 'number') + ? 'IO' + : 'Tokens'; global.vectorModels = config.vectorModels; global.audioSpeechModels = config.audioSpeechModels; global.whisperModel = config.whisperModel; diff --git a/packages/service/core/ai/config.ts b/packages/service/core/ai/config.ts index c30f5cc51..0db12a4b0 100644 --- a/packages/service/core/ai/config.ts +++ b/packages/service/core/ai/config.ts @@ -1,4 +1,3 @@ -import type { UserModelSchema } from '@fastgpt/global/support/user/type'; import OpenAI from '@fastgpt/global/core/ai'; import { ChatCompletionCreateParamsNonStreaming, @@ -7,13 +6,11 @@ import { import { getErrText } from '@fastgpt/global/common/error/utils'; import { addLog } from '../../common/system/log'; import { i18nT } from '../../../web/i18n/utils'; +import { OpenaiAccountType } from '@fastgpt/global/support/user/team/type'; export const openaiBaseUrl = process.env.OPENAI_BASE_URL || 'https://api.openai.com/v1'; -export const getAIApi = (props?: { - userKey?: UserModelSchema['openaiAccount']; - timeout?: number; -}) => { +export const getAIApi = (props?: { userKey?: OpenaiAccountType; timeout?: number }) => { const { userKey, timeout } = props || {}; const baseUrl = @@ -29,7 +26,7 @@ export const getAIApi = (props?: { }); }; -export const getAxiosConfig = (props?: { userKey?: UserModelSchema['openaiAccount'] }) => { +export const getAxiosConfig = (props?: { userKey?: OpenaiAccountType }) => { const { userKey } = props || {}; const baseUrl = @@ -57,7 +54,7 @@ export const createChatCompletion = async ({ options }: { body: T; - userKey?: UserModelSchema['openaiAccount']; + userKey?: OpenaiAccountType; timeout?: number; options?: OpenAI.RequestOptions; }): Promise<{ diff --git a/packages/service/core/ai/config/embedding/text-embedding-ada-002.json b/packages/service/core/ai/config/embedding/text-embedding-ada-002.json new file mode 100644 index 000000000..4f6290f75 --- /dev/null +++ b/packages/service/core/ai/config/embedding/text-embedding-ada-002.json @@ -0,0 +1,11 @@ +{ + "provider": "OpenAI", + "model": "text-embedding-ada-002", + "name": "text-embedding-ada-002", + + "defaultToken": 512, // 默认分块 token + "maxToken": 3000, // 最大分块 token + "weight": 0, // 权重 + + "charsPointsPrice": 0 // 积分/1k token +} diff --git a/packages/service/core/ai/config/llm/gpt-4o-mini.json b/packages/service/core/ai/config/llm/gpt-4o-mini.json new file mode 100644 index 000000000..0bcf7f6de --- /dev/null +++ b/packages/service/core/ai/config/llm/gpt-4o-mini.json @@ -0,0 +1,33 @@ +{ + "provider": "OpenAI", + "model": "gpt-4o-mini", + "name": "GPT-4o-mini", // alias + + "maxContext": 125000, // 最大上下文 + "maxResponse": 16000, // 最大回复 + "quoteMaxToken": 60000, // 最大引用 + "maxTemperature": 1.2, // 最大温度 + "presencePenaltyRange": [-2, 2], // 惩罚系数范围 + "frequencyPenaltyRange": [-2, 2], // 频率惩罚系数范围 + "responseFormatList": ["text", "json_object", "json_schema"], // 响应格式 + "showStopSign": true, // 是否显示停止符号 + + "vision": true, // 是否支持图片识别 + "toolChoice": true, // 是否支持工具调用 + "functionCall": false, // 是否支持函数调用（一般都可以 false 了，基本不用了） + "defaultSystemChatPrompt": "", // 默认系统提示 + + "datasetProcess": true, // 用于知识库文本处理 + "usedInClassify": true, // 用于问题分类 + "customCQPrompt": "", // 自定义问题分类提示 + "usedInExtractFields": true, // 用于提取字段 + "customExtractPrompt": "", // 自定义提取提示 + "usedInToolCall": true, // 用于工具调用 + "usedInQueryExtension": true, // 用于问题优化 + + "defaultConfig": {}, // 额外的自定义 body + "fieldMap": {}, // body 字段映射 + + "censor": false, // 是否开启敏感词过滤 + "charsPointsPrice": 0 // n 积分/1k token +} diff --git a/packages/service/core/ai/config/rerank/bge-reranker-v2-m3.json b/packages/service/core/ai/config/rerank/bge-reranker-v2-m3.json new file mode 100644 index 000000000..3cc1a33b5 --- /dev/null +++ b/packages/service/core/ai/config/rerank/bge-reranker-v2-m3.json @@ -0,0 +1,6 @@ +{ + "provider": "BAAI", + "model": "bge-reranker-v2-m3", + "name": "bge-reranker-v2-m3", + "charsPointsPrice": 0 +} diff --git a/packages/service/core/ai/config/stt/whisper-1.json b/packages/service/core/ai/config/stt/whisper-1.json new file mode 100644 index 000000000..2d8639786 --- /dev/null +++ b/packages/service/core/ai/config/stt/whisper-1.json @@ -0,0 +1,6 @@ +{ + "provider": "OpenAI", + "model": "whisper-1", + "name": "whisper-1", + "charsPointsPrice": 0 +} diff --git a/packages/service/core/ai/config/tts/tts-1.json b/packages/service/core/ai/config/tts/tts-1.json new file mode 100644 index 000000000..80105c227 --- /dev/null +++ b/packages/service/core/ai/config/tts/tts-1.json @@ -0,0 +1,32 @@ +{ + "provider": "OpenAI", + "model": "tts-1", + "name": "TTS1", + "charsPointsPrice": 0, + "voices": [ + { + "label": "Alloy", + "value": "alloy" + }, + { + "label": "Echo", + "value": "echo" + }, + { + "label": "Fable", + "value": "fable" + }, + { + "label": "Onyx", + "value": "onyx" + }, + { + "label": "Nova", + "value": "nova" + }, + { + "label": "Shimmer", + "value": "shimmer" + } + ] +} diff --git a/packages/service/core/ai/functions/createQuestionGuide.ts b/packages/service/core/ai/functions/createQuestionGuide.ts index bb332180c..7ceb33e51 100644 --- a/packages/service/core/ai/functions/createQuestionGuide.ts +++ b/packages/service/core/ai/functions/createQuestionGuide.ts @@ -1,33 +1,39 @@ import type { ChatCompletionMessageParam } from '@fastgpt/global/core/ai/type.d'; import { createChatCompletion } from '../config'; -import { countGptMessagesTokens } from '../../../common/string/tiktoken/index'; +import { countGptMessagesTokens, countPromptTokens } from '../../../common/string/tiktoken/index'; import { loadRequestMessages } from '../../chat/utils'; import { llmCompletionsBodyFormat } from '../utils'; - -export const Prompt_QuestionGuide = `You are an AI assistant tasked with predicting the user's next question based on the conversation history. Your goal is to generate 3 potential questions that will guide the user to continue the conversation. When generating these questions, adhere to the following rules: - -1. Use the same language as the user's last question in the conversation history. -2. Keep each question under 20 characters in length. -3. Return the questions in JSON format: ["question1", "question2", "question3"]. - -Analyze the conversation history provided to you and use it as context to generate relevant and engaging follow-up questions. Your predictions should be logical extensions of the current topic or related areas that the user might be interested in exploring further. - -Remember to maintain consistency in tone and style with the existing conversation while providing diverse options for the user to choose from. Your goal is to keep the conversation flowing naturally and help the user delve deeper into the subject matter or explore related topics.`; +import { + PROMPT_QUESTION_GUIDE, + PROMPT_QUESTION_GUIDE_FOOTER +} from '@fastgpt/global/core/ai/prompt/agent'; +import { addLog } from '../../../common/system/log'; +import json5 from 'json5'; export async function createQuestionGuide({ messages, - model + model, + customPrompt }: { messages: ChatCompletionMessageParam[]; model: string; -}) { + customPrompt?: string; +}): Promise<{ + result: string[]; + inputTokens: number; + outputTokens: number; +}> { const concatMessages: ChatCompletionMessageParam[] = [ ...messages, { role: 'user', - content: Prompt_QuestionGuide + content: `${customPrompt || PROMPT_QUESTION_GUIDE}\n${PROMPT_QUESTION_GUIDE_FOOTER}` } ]; + const requestMessages = await loadRequestMessages({ + messages: concatMessages, + useVision: false + }); const { response: data } = await createChatCompletion({ body: llmCompletionsBodyFormat( @@ -35,10 +41,7 @@ export async function createQuestionGuide({ model, temperature: 0.1, max_tokens: 200, - messages: await loadRequestMessages({ - messages: concatMessages, - useVision: false - }), + messages: requestMessages, stream: false }, model @@ -50,12 +53,15 @@ export async function createQuestionGuide({ const start = answer.indexOf('['); const end = answer.lastIndexOf(']'); - const tokens = await countGptMessagesTokens(concatMessages); + const inputTokens = await countGptMessagesTokens(requestMessages); + const outputTokens = await countPromptTokens(answer); if (start === -1 || end === -1) { + addLog.warn('Create question guide error', { answer }); return { result: [], - tokens: 0 + inputTokens: 0, + outputTokens: 0 }; } @@ -66,13 +72,17 @@ export async function createQuestionGuide({ try { return { - result: JSON.parse(jsonStr), - tokens + result: json5.parse(jsonStr), + inputTokens, + outputTokens }; } catch (error) { + console.log(error); + return { result: [], - tokens: 0 + inputTokens: 0, + outputTokens: 0 }; } } diff --git a/packages/service/core/ai/functions/queryExtension.ts b/packages/service/core/ai/functions/queryExtension.ts index 37ae19a62..31bb9ee8d 100644 --- a/packages/service/core/ai/functions/queryExtension.ts +++ b/packages/service/core/ai/functions/queryExtension.ts @@ -1,7 +1,7 @@ import { replaceVariable } from '@fastgpt/global/common/string/tools'; import { createChatCompletion } from '../config'; import { ChatItemType } from '@fastgpt/global/core/chat/type'; -import { countGptMessagesTokens } from '../../../common/string/tiktoken/index'; +import { countGptMessagesTokens, countPromptTokens } from '../../../common/string/tiktoken/index'; import { chatValue2RuntimePrompt } from '@fastgpt/global/core/chat/adapt'; import { getLLMModel } from '../model'; import { llmCompletionsBodyFormat } from '../utils'; @@ -121,7 +121,8 @@ export const queryExtension = async ({ rawQuery: string; extensionQueries: string[]; model: string; - tokens: number; + inputTokens: number; + outputTokens: number; }> => { const systemFewShot = chatBg ? `Q: 对话背景。 @@ -166,7 +167,8 @@ A: ${chatBg} rawQuery: query, extensionQueries: [], model, - tokens: 0 + inputTokens: 0, + outputTokens: 0 }; } @@ -181,7 +183,8 @@ A: ${chatBg} rawQuery: query, extensionQueries: Array.isArray(queries) ? queries : [], model, - tokens: await countGptMessagesTokens(messages) + inputTokens: await countGptMessagesTokens(messages), + outputTokens: await countPromptTokens(answer) }; } catch (error) { addLog.error(`Query extension error`, error); @@ -189,7 +192,8 @@ A: ${chatBg} rawQuery: query, extensionQueries: [], model, - tokens: 0 + inputTokens: 0, + outputTokens: 0 }; } }; diff --git a/packages/service/core/ai/model.ts b/packages/service/core/ai/model.ts index 2d7ef0f98..4d5a8633e 100644 --- a/packages/service/core/ai/model.ts +++ b/packages/service/core/ai/model.ts @@ -4,6 +4,7 @@ export const getLLMModel = (model?: string) => { global.llmModels[0] ); }; + export const getDatasetModel = (model?: string) => { return ( global.llmModels diff --git a/packages/service/core/ai/utils.ts b/packages/service/core/ai/utils.ts index ec95b61a8..aad1cb1ef 100644 --- a/packages/service/core/ai/utils.ts +++ b/packages/service/core/ai/utils.ts @@ -12,10 +12,12 @@ export const computedMaxToken = async ({ model, filterMessages = [] }: { - maxToken: number; + maxToken?: number; model: LLMModelItemType; filterMessages: ChatCompletionMessageParam[]; }) => { + if (maxToken === undefined) return; + maxToken = Math.min(maxToken, model.maxResponse); const tokensLimit = model.maxContext; @@ -37,8 +39,6 @@ export const computedTemperature = ({ model: LLMModelItemType; temperature: number; }) => { - if (temperature < 1) return temperature; - temperature = +(model.maxTemperature * (temperature / 10)).toFixed(2); temperature = Math.max(temperature, 0.01); @@ -63,12 +63,13 @@ export const llmCompletionsBodyFormat = ( const requestBody: T = { ...body, - temperature: body.temperature - ? computedTemperature({ - model: modelData, - temperature: body.temperature - }) - : undefined, + temperature: + typeof body.temperature === 'number' + ? computedTemperature({ + model: modelData, + temperature: body.temperature + }) + : undefined, ...modelData?.defaultConfig }; diff --git a/packages/service/core/app/plugin/controller.ts b/packages/service/core/app/plugin/controller.ts index 00c2eea94..2d185686d 100644 --- a/packages/service/core/app/plugin/controller.ts +++ b/packages/service/core/app/plugin/controller.ts @@ -131,6 +131,7 @@ export async function getChildAppPreviewNode({ name: app.name, intro: app.intro, courseUrl: app.courseUrl, + userGuide: app.userGuide, showStatus: app.showStatus, isTool: true, version: app.version, diff --git a/packages/service/core/app/plugin/utils.ts b/packages/service/core/app/plugin/utils.ts index 3ec4ecc56..43e0db8aa 100644 --- a/packages/service/core/app/plugin/utils.ts +++ b/packages/service/core/app/plugin/utils.ts @@ -1,11 +1,15 @@ import { ChatNodeUsageType } from '@fastgpt/global/support/wallet/bill/type'; import { PluginRuntimeType } from '@fastgpt/global/core/plugin/type'; +import { splitCombinePluginId } from './controller'; +import { PluginSourceEnum } from '@fastgpt/global/core/plugin/constants'; /* Plugin points calculation: - 1. Return 0 if error - 2. Add configured points if commercial plugin - 3. Add sum of child nodes points + 1. 系统插件/商业版插件： + - 有错误：返回 0 + - 无错误：返回 单次积分 + 子流程积分（可配置） + 2. 个人插件 + - 返回 子流程积分 */ export const computedPluginUsage = async ({ plugin, @@ -16,13 +20,16 @@ export const computedPluginUsage = async ({ childrenUsage: ChatNodeUsageType[]; error?: boolean; }) => { - if (error) { - return 0; + const { source } = await splitCombinePluginId(plugin.id); + const childrenUsages = childrenUsage.reduce((sum, item) => sum + (item.totalPoints || 0), 0); + + if (source !== PluginSourceEnum.personal) { + if (error) return 0; + + const pluginCurrentCost = plugin.currentCost ?? 0; + + return plugin.hasTokenFee ? pluginCurrentCost + childrenUsages : pluginCurrentCost; } - const childrenIUsages = childrenUsage.reduce((sum, item) => sum + (item.totalPoints || 0), 0); - - const pluginCurrentCose = plugin.currentCost ?? 0; - - return plugin.hasTokenFee ? pluginCurrentCose + childrenIUsages : pluginCurrentCose; + return childrenUsages; }; diff --git a/packages/service/core/app/schema.ts b/packages/service/core/app/schema.ts index 4e396d40e..ce042a500 100644 --- a/packages/service/core/app/schema.ts +++ b/packages/service/core/app/schema.ts @@ -11,7 +11,7 @@ export const AppCollectionName = 'apps'; export const chatConfigType = { welcomeText: String, variables: Array, - questionGuide: Boolean, + questionGuide: Object, ttsConfig: Object, whisperConfig: Object, scheduledTriggerConfig: Object, diff --git a/packages/service/core/app/templates/templateSchema.ts b/packages/service/core/app/templates/templateSchema.ts new file mode 100644 index 000000000..7e61a1b86 --- /dev/null +++ b/packages/service/core/app/templates/templateSchema.ts @@ -0,0 +1,51 @@ +import { AppTemplateSchemaType } from '@fastgpt/global/core/app/type'; +import { connectionMongo, getMongoModel } from '../../../common/mongo/index'; +const { Schema } = connectionMongo; + +export const collectionName = 'app_templates'; + +const AppTemplateSchema = new Schema({ + templateId: { + type: String, + required: true + }, + name: { + type: String + }, + intro: { + type: String + }, + avatar: { + type: String + }, + tags: { + type: [String], + default: undefined + }, + type: { + type: String + }, + isActive: { + type: Boolean + }, + userGuide: { + type: Object + }, + isQuickTemplate: { + type: Boolean + }, + order: { + type: Number, + default: -1 + }, + workflow: { + type: Object + } +}); + +AppTemplateSchema.index({ templateId: 1 }); + +export const MongoAppTemplate = getMongoModel( + collectionName, + AppTemplateSchema +); diff --git a/packages/service/core/app/templates/templateTypeSchema.ts b/packages/service/core/app/templates/templateTypeSchema.ts new file mode 100644 index 000000000..0926e8c6c --- /dev/null +++ b/packages/service/core/app/templates/templateTypeSchema.ts @@ -0,0 +1,25 @@ +import { TemplateTypeSchemaType } from '@fastgpt/global/core/app/type'; +import { connectionMongo, getMongoModel } from '../../../common/mongo/index'; +const { Schema } = connectionMongo; + +export const collectionName = 'app_template_types'; + +const TemplateTypeSchema = new Schema({ + typeName: { + type: String, + required: true + }, + typeId: { + type: String, + required: true + }, + typeOrder: { + type: Number, + default: 0 + } +}); + +export const MongoTemplateTypes = getMongoModel( + collectionName, + TemplateTypeSchema +); diff --git a/packages/service/core/app/version/schema.ts b/packages/service/core/app/version/schema.ts index 4acf91094..0836b950f 100644 --- a/packages/service/core/app/version/schema.ts +++ b/packages/service/core/app/version/schema.ts @@ -2,10 +2,16 @@ import { connectionMongo, getMongoModel, type Model } from '../../../common/mong const { Schema, model, models } = connectionMongo; import { AppVersionSchemaType } from '@fastgpt/global/core/app/version'; import { chatConfigType } from '../schema'; +import { TeamMemberCollectionName } from '@fastgpt/global/support/user/team/constant'; export const AppVersionCollectionName = 'app_versions'; const AppVersionSchema = new Schema({ + tmbId: { + type: String, + ref: TeamMemberCollectionName, + required: true + }, appId: { type: Schema.Types.ObjectId, ref: AppVersionCollectionName, @@ -26,16 +32,8 @@ const AppVersionSchema = new Schema({ chatConfig: { type: chatConfigType }, - isPublish: { - type: Boolean - }, - versionName: { - type: String, - default: '' - }, - tmbId: { - type: String - } + isPublish: Boolean, + versionName: String }); try { diff --git a/packages/service/core/chat/chatItemSchema.ts b/packages/service/core/chat/chatItemSchema.ts index 237ae49c9..a18065330 100644 --- a/packages/service/core/chat/chatItemSchema.ts +++ b/packages/service/core/chat/chatItemSchema.ts @@ -86,18 +86,21 @@ const ChatItemSchema = new Schema({ }); try { - ChatItemSchema.index({ dataId: 1 }, { background: true }); + ChatItemSchema.index({ dataId: 1 }); /* delete by app; delete by chat id; get chat list; get chat logs; close custom feedback; */ - ChatItemSchema.index({ appId: 1, chatId: 1, dataId: 1 }, { background: true }); + ChatItemSchema.index({ appId: 1, chatId: 1, dataId: 1 }); // admin charts - ChatItemSchema.index({ time: -1, obj: 1 }, { background: true }); + ChatItemSchema.index({ time: -1, obj: 1 }); // timer, clear history - ChatItemSchema.index({ teamId: 1, time: -1 }, { background: true }); + ChatItemSchema.index({ teamId: 1, time: -1 }); + + // Admin charts + ChatItemSchema.index({ obj: 1, time: -1 }, { partialFilterExpression: { obj: 'Human' } }); } catch (error) { console.log(error); } diff --git a/packages/service/core/chat/chatSchema.ts b/packages/service/core/chat/chatSchema.ts index 4aae6eef2..53a6569ee 100644 --- a/packages/service/core/chat/chatSchema.ts +++ b/packages/service/core/chat/chatSchema.ts @@ -72,6 +72,7 @@ const ChatSchema = new Schema({ type: Object, default: {} }, + pluginInputs: Array, metadata: { //For special storage type: Object, @@ -80,19 +81,19 @@ const ChatSchema = new Schema({ }); try { - ChatSchema.index({ chatId: 1 }, { background: true }); + ChatSchema.index({ chatId: 1 }); // get user history - ChatSchema.index({ tmbId: 1, appId: 1, top: -1, updateTime: -1 }, { background: true }); + ChatSchema.index({ tmbId: 1, appId: 1, top: -1, updateTime: -1 }); // delete by appid; clear history; init chat; update chat; auth chat; get chat; - ChatSchema.index({ appId: 1, chatId: 1 }, { background: true }); + ChatSchema.index({ appId: 1, chatId: 1 }); // get chat logs; - ChatSchema.index({ teamId: 1, appId: 1, updateTime: -1 }, { background: true }); + ChatSchema.index({ teamId: 1, appId: 1, updateTime: -1 }); // get share chat history - ChatSchema.index({ shareId: 1, outLinkUid: 1, updateTime: -1 }, { background: true }); + ChatSchema.index({ shareId: 1, outLinkUid: 1, updateTime: -1 }); // timer, clear history - ChatSchema.index({ teamId: 1, updateTime: -1 }, { background: true }); + ChatSchema.index({ teamId: 1, updateTime: -1 }); } catch (error) { console.log(error); } diff --git a/packages/service/core/chat/saveChat.ts b/packages/service/core/chat/saveChat.ts index 4004a8a1c..c5f7b0708 100644 --- a/packages/service/core/chat/saveChat.ts +++ b/packages/service/core/chat/saveChat.ts @@ -10,6 +10,7 @@ import { getAppChatConfig, getGuideModule } from '@fastgpt/global/core/workflow/ import { AppChatConfigType } from '@fastgpt/global/core/app/type'; import { mergeChatResponseData } from '@fastgpt/global/core/chat/utils'; import { pushChatLog } from './pushChatLog'; +import { FlowNodeTypeEnum } from '@fastgpt/global/core/workflow/node/constant'; type Props = { chatId: string; @@ -62,6 +63,9 @@ export async function saveChat({ systemConfigNode: getGuideModule(nodes), isPublicFetch: false }); + const pluginInputs = nodes?.find( + (node) => node.flowNodeType === FlowNodeTypeEnum.pluginInput + )?.inputs; await mongoSessionRun(async (session) => { const [{ _id: chatItemIdHuman }, { _id: chatItemIdAi }] = await MongoChatItem.insertMany( @@ -89,6 +93,7 @@ export async function saveChat({ variableList, welcomeText, variables: variables || {}, + pluginInputs, title: newTitle, source, shareId, diff --git a/packages/service/core/chat/utils.ts b/packages/service/core/chat/utils.ts index cbca1b1b3..ed35466e8 100644 --- a/packages/service/core/chat/utils.ts +++ b/packages/service/core/chat/utils.ts @@ -104,9 +104,6 @@ export const loadRequestMessages = async ({ }) => { // Load image to base64 const loadImageToBase64 = async (messages: ChatCompletionContentPart[]) => { - if (process.env.MULTIPLE_DATA_TO_BASE64 === 'false') { - return messages; - } return Promise.all( messages.map(async (item) => { if (item.type === 'image_url') { @@ -125,7 +122,7 @@ export const loadRequestMessages = async ({ try { // If imgUrl is a local path, load image from local, and set url to base64 - if (imgUrl.startsWith('/')) { + if (imgUrl.startsWith('/') || process.env.MULTIPLE_DATA_TO_BASE64 === 'true') { addLog.debug('Load image from local server', { baseUrl: serverRequestBaseUrl, requestUrl: imgUrl diff --git a/packages/service/core/dataset/collection/controller.ts b/packages/service/core/dataset/collection/controller.ts index df040b87b..061543274 100644 --- a/packages/service/core/dataset/collection/controller.ts +++ b/packages/service/core/dataset/collection/controller.ts @@ -1,11 +1,10 @@ -import { TrainingModeEnum } from '@fastgpt/global/core/dataset/constants'; +import { + DatasetCollectionTypeEnum, + TrainingModeEnum +} from '@fastgpt/global/core/dataset/constants'; import type { CreateDatasetCollectionParams } from '@fastgpt/global/core/dataset/api.d'; import { MongoDatasetCollection } from './schema'; -import { - CollectionWithDatasetType, - DatasetCollectionSchemaType, - DatasetSchemaType -} from '@fastgpt/global/core/dataset/type'; +import { DatasetCollectionSchemaType, DatasetSchemaType } from '@fastgpt/global/core/dataset/type'; import { MongoDatasetTraining } from '../training/schema'; import { MongoDatasetData } from '../data/schema'; import { delImgByRelatedId } from '../../../common/file/image/controller'; @@ -24,6 +23,8 @@ import { getLLMModel, getVectorModel } from '../../ai/model'; import { pushDataListToTrainingQueue } from '../training/controller'; import { MongoImage } from '../../../common/file/image/schema'; import { hashStr } from '@fastgpt/global/common/string/tools'; +import { addDays } from 'date-fns'; +import { MongoDatasetDataText } from '../data/dataTextSchema'; export const createCollectionAndInsertData = async ({ dataset, @@ -72,6 +73,17 @@ export const createCollectionAndInsertData = async ({ hashRawText: hashStr(rawText), rawTextLength: rawText.length, + nextSyncTime: (() => { + if (!dataset.autoSync) return undefined; + if ( + [DatasetCollectionTypeEnum.link, DatasetCollectionTypeEnum.apiFile].includes( + createCollectionParams.type + ) + ) { + return addDays(new Date(), 1); + } + return undefined; + })(), session }); @@ -155,10 +167,8 @@ export async function createOneCollection({ fileId, rawLink, - externalFileId, externalFileUrl, - apiFileId, hashRawText, @@ -166,7 +176,10 @@ export async function createOneCollection({ metadata = {}, session, tags, - createTime + + createTime, + updateTime, + nextSyncTime }: CreateOneCollectionParams) { // Create collection tags const collectionTags = await createOrGetCollectionTags({ tags, teamId, datasetId, session }); @@ -197,7 +210,10 @@ export async function createOneCollection({ rawTextLength, hashRawText, tags: collectionTags, - createTime + + createTime, + updateTime, + nextSyncTime } ], { session } @@ -211,7 +227,7 @@ export const delCollectionRelatedSource = async ({ collections, session }: { - collections: (CollectionWithDatasetType | DatasetCollectionSchemaType)[]; + collections: DatasetCollectionSchemaType[]; session: ClientSession; }) => { if (collections.length === 0) return; @@ -225,12 +241,12 @@ export const delCollectionRelatedSource = async ({ .map((item) => item?.metadata?.relatedImgId || '') .filter(Boolean); - // delete files + // Delete files await delFileByFileIdList({ bucketName: BucketNameEnum.dataset, fileIdList }); - // delete images + // Delete images await delImgByRelatedId({ teamId, relateIds: relatedImageIds, @@ -245,7 +261,7 @@ export async function delCollection({ session, delRelatedSource }: { - collections: (CollectionWithDatasetType | DatasetCollectionSchemaType)[]; + collections: DatasetCollectionSchemaType[]; session: ClientSession; delRelatedSource: boolean; }) { @@ -255,19 +271,10 @@ export async function delCollection({ if (!teamId) return Promise.reject('teamId is not exist'); - const datasetIds = Array.from( - new Set( - collections.map((item) => { - if (typeof item.datasetId === 'string') { - return String(item.datasetId); - } - return String(item.datasetId._id); - }) - ) - ); + const datasetIds = Array.from(new Set(collections.map((item) => String(item.datasetId)))); const collectionIds = collections.map((item) => String(item._id)); - // delete training data + // Delete training data await MongoDatasetTraining.deleteMany({ teamId, datasetIds: { $in: datasetIds }, @@ -279,11 +286,16 @@ export async function delCollection({ await delCollectionRelatedSource({ collections, session }); } - // delete dataset.datas + // Delete dataset_datas await MongoDatasetData.deleteMany( { teamId, datasetIds: { $in: datasetIds }, collectionId: { $in: collectionIds } }, { session } ); + // Delete dataset_data_texts + await MongoDatasetDataText.deleteMany( + { teamId, datasetIds: { $in: datasetIds }, collectionId: { $in: collectionIds } }, + { session } + ); // delete collections await MongoDatasetCollection.deleteMany( @@ -305,7 +317,7 @@ export async function delOnlyCollection({ collections, session }: { - collections: (CollectionWithDatasetType | DatasetCollectionSchemaType)[]; + collections: DatasetCollectionSchemaType[]; session: ClientSession; }) { if (collections.length === 0) return; @@ -314,16 +326,7 @@ export async function delOnlyCollection({ if (!teamId) return Promise.reject('teamId is not exist'); - const datasetIds = Array.from( - new Set( - collections.map((item) => { - if (typeof item.datasetId === 'string') { - return String(item.datasetId); - } - return String(item.datasetId._id); - }) - ) - ); + const datasetIds = Array.from(new Set(collections.map((item) => String(item.datasetId)))); const collectionIds = collections.map((item) => String(item._id)); // delete training data diff --git a/packages/service/core/dataset/collection/schema.ts b/packages/service/core/dataset/collection/schema.ts index a6f762861..da13ed8ae 100644 --- a/packages/service/core/dataset/collection/schema.ts +++ b/packages/service/core/dataset/collection/schema.ts @@ -1,4 +1,4 @@ -import { connectionMongo, getMongoModel, type Model } from '../../../common/mongo'; +import { connectionMongo, getMongoModel } from '../../../common/mongo'; const { Schema, model, models } = connectionMongo; import { DatasetCollectionSchemaType } from '@fastgpt/global/core/dataset/type.d'; import { TrainingTypeMap, DatasetCollectionTypeMap } from '@fastgpt/global/core/dataset/constants'; @@ -10,100 +10,102 @@ import { export const DatasetColCollectionName = 'dataset_collections'; -const DatasetCollectionSchema = new Schema( - { - parentId: { - type: Schema.Types.ObjectId, - ref: DatasetColCollectionName, - default: null - }, - teamId: { - type: Schema.Types.ObjectId, - ref: TeamCollectionName, - required: true - }, - tmbId: { - type: Schema.Types.ObjectId, - ref: TeamMemberCollectionName, - required: true - }, - datasetId: { - type: Schema.Types.ObjectId, - ref: DatasetCollectionName, - required: true - }, - type: { - type: String, - enum: Object.keys(DatasetCollectionTypeMap), - required: true - }, - name: { - type: String, - required: true - }, - createTime: { - type: Date, - default: () => new Date() - }, - updateTime: { - type: Date, - default: () => new Date() - }, - forbid: { - type: Boolean, - default: false - }, - - // chunk filed - trainingType: { - type: String, - enum: Object.keys(TrainingTypeMap) - }, - chunkSize: { - type: Number, - required: true - }, - chunkSplitter: { - type: String - }, - qaPrompt: { - type: String - }, - ocrParse: Boolean, - - tags: { - type: [String], - default: [] - }, - - // local file collection - fileId: { - type: Schema.Types.ObjectId, - ref: 'dataset.files' - }, - // web link collection - rawLink: String, - // api collection - apiFileId: String, - // external collection - externalFileId: String, - externalFileUrl: String, // external import url - - // metadata - rawTextLength: Number, - hashRawText: String, - metadata: { - type: Object, - default: {} - } +const DatasetCollectionSchema = new Schema({ + parentId: { + type: Schema.Types.ObjectId, + ref: DatasetColCollectionName, + default: null }, - { - // Auto update updateTime - timestamps: { - updatedAt: 'updateTime' - } + teamId: { + type: Schema.Types.ObjectId, + ref: TeamCollectionName, + required: true + }, + tmbId: { + type: Schema.Types.ObjectId, + ref: TeamMemberCollectionName, + required: true + }, + datasetId: { + type: Schema.Types.ObjectId, + ref: DatasetCollectionName, + required: true + }, + type: { + type: String, + enum: Object.keys(DatasetCollectionTypeMap), + required: true + }, + name: { + type: String, + required: true + }, + createTime: { + type: Date, + default: () => new Date() + }, + updateTime: { + type: Date, + default: () => new Date() + }, + forbid: { + type: Boolean, + default: false + }, + + // chunk filed + trainingType: { + type: String, + enum: Object.keys(TrainingTypeMap) + }, + chunkSize: { + type: Number, + required: true + }, + chunkSplitter: { + type: String + }, + qaPrompt: { + type: String + }, + ocrParse: Boolean, + + tags: { + type: [String], + default: [] + }, + + // local file collection + fileId: { + type: Schema.Types.ObjectId, + ref: 'dataset.files' + }, + // web link collection + rawLink: String, + // api collection + apiFileId: String, + // external collection + externalFileId: String, + externalFileUrl: String, // external import url + + // next sync time + nextSyncTime: Date, + + // metadata + rawTextLength: Number, + hashRawText: String, + metadata: { + type: Object, + default: {} } -); +}); + +DatasetCollectionSchema.virtual('dataset', { + ref: DatasetCollectionName, + localField: 'datasetId', + foreignField: '_id', + justOne: true +}); try { // auth file @@ -122,6 +124,16 @@ try { // create time filter DatasetCollectionSchema.index({ teamId: 1, datasetId: 1, createTime: 1 }); + // next sync time filter + DatasetCollectionSchema.index( + { type: 1, nextSyncTime: -1 }, + { + partialFilterExpression: { + nextSyncTime: { $exists: true } + } + } + ); + // Get collection by external file id DatasetCollectionSchema.index( { datasetId: 1, externalFileId: 1 }, diff --git a/packages/service/core/dataset/collection/utils.ts b/packages/service/core/dataset/collection/utils.ts index 3a257c229..9bf9a2262 100644 --- a/packages/service/core/dataset/collection/utils.ts +++ b/packages/service/core/dataset/collection/utils.ts @@ -130,7 +130,7 @@ export const collectionTagsToTagLabel = async ({ }; export const syncCollection = async (collection: CollectionWithDatasetType) => { - const dataset = collection.datasetId; + const dataset = collection.dataset; if ( collection.type !== DatasetCollectionTypeEnum.link && @@ -163,6 +163,10 @@ export const syncCollection = async (collection: CollectionWithDatasetType) => { ...sourceReadType }); + if (!rawText) { + return DatasetCollectionSyncResultEnum.failed; + } + // Check if the original text is the same: skip if same const hashRawText = hashStr(rawText); if (collection.hashRawText && hashRawText === collection.hashRawText) { @@ -178,28 +182,30 @@ export const syncCollection = async (collection: CollectionWithDatasetType) => { createCollectionParams: { teamId: collection.teamId, tmbId: collection.tmbId, - datasetId: collection.datasetId._id, name: collection.name, + datasetId: collection.datasetId, + parentId: collection.parentId, type: collection.type, + trainingType: collection.trainingType, + chunkSize: collection.chunkSize, + chunkSplitter: collection.chunkSplitter, + qaPrompt: collection.qaPrompt, + fileId: collection.fileId, rawLink: collection.rawLink, externalFileId: collection.externalFileId, externalFileUrl: collection.externalFileUrl, apiFileId: collection.apiFileId, - rawTextLength: rawText.length, hashRawText, + rawTextLength: rawText.length, + + metadata: collection.metadata, tags: collection.tags, createTime: collection.createTime, - - parentId: collection.parentId, - trainingType: collection.trainingType, - chunkSize: collection.chunkSize, - chunkSplitter: collection.chunkSplitter, - qaPrompt: collection.qaPrompt, - metadata: collection.metadata + updateTime: new Date() } }); diff --git a/packages/service/core/dataset/controller.ts b/packages/service/core/dataset/controller.ts index 0f9ef5c6f..96f6523e7 100644 --- a/packages/service/core/dataset/controller.ts +++ b/packages/service/core/dataset/controller.ts @@ -1,4 +1,4 @@ -import { CollectionWithDatasetType, DatasetSchemaType } from '@fastgpt/global/core/dataset/type'; +import { DatasetSchemaType } from '@fastgpt/global/core/dataset/type'; import { MongoDatasetCollection } from './collection/schema'; import { MongoDataset } from './schema'; import { delCollectionRelatedSource } from './collection/controller'; @@ -6,6 +6,7 @@ import { ClientSession } from '../../common/mongo'; import { MongoDatasetTraining } from './training/schema'; import { MongoDatasetData } from './data/schema'; import { deleteDatasetDataVector } from '../../common/vectorStore/controller'; +import { MongoDatasetDataText } from './data/dataTextSchema'; /* ============= dataset ========== */ /* find all datasetId by top datasetId */ @@ -49,9 +50,9 @@ export async function findDatasetAndAllChildren({ } export async function getCollectionWithDataset(collectionId: string) { - const data = (await MongoDatasetCollection.findById(collectionId) - .populate('datasetId') - .lean()) as CollectionWithDatasetType; + const data = await MongoDatasetCollection.findById(collectionId) + .populate<{ dataset: DatasetSchemaType }>('dataset') + .lean(); if (!data) { return Promise.reject('Collection is not exist'); } @@ -71,19 +72,10 @@ export async function delDatasetRelevantData({ const teamId = datasets[0].teamId; if (!teamId) { - return Promise.reject('teamId is required'); + return Promise.reject('TeamId is required'); } - const datasetIds = datasets.map((item) => String(item._id)); - - // Get _id, teamId, fileId, metadata.relatedImgId for all collections - const collections = await MongoDatasetCollection.find( - { - teamId, - datasetId: { $in: datasetIds } - }, - '_id teamId datasetId fileId metadata' - ).lean(); + const datasetIds = datasets.map((item) => item._id); // delete training data await MongoDatasetTraining.deleteMany({ @@ -91,21 +83,34 @@ export async function delDatasetRelevantData({ datasetId: { $in: datasetIds } }); - // image and file - await delCollectionRelatedSource({ collections, session }); - - // delete dataset.datas - await MongoDatasetData.deleteMany({ teamId, datasetId: { $in: datasetIds } }, { session }); - - // delete collections - await MongoDatasetCollection.deleteMany( + // Get _id, teamId, fileId, metadata.relatedImgId for all collections + const collections = await MongoDatasetCollection.find( { teamId, datasetId: { $in: datasetIds } }, + '_id teamId datasetId fileId metadata', { session } - ); + ).lean(); - // no session delete: delete files, vector data + // Delete Image and file + await delCollectionRelatedSource({ collections, session }); + + // delete collections + await MongoDatasetCollection.deleteMany({ + teamId, + datasetId: { $in: datasetIds } + }).session(session); + + // No session delete: + // Delete dataset_data_texts + await MongoDatasetDataText.deleteMany({ + teamId, + datasetId: { $in: datasetIds } + }); + // delete dataset_datas + await MongoDatasetData.deleteMany({ teamId, datasetId: { $in: datasetIds } }); + + // Delete vector data await deleteDatasetDataVector({ teamId, datasetIds }); } diff --git a/packages/service/core/dataset/data/dataTextSchema.ts b/packages/service/core/dataset/data/dataTextSchema.ts new file mode 100644 index 000000000..e0564ed2b --- /dev/null +++ b/packages/service/core/dataset/data/dataTextSchema.ts @@ -0,0 +1,45 @@ +import { connectionMongo, getMongoModel } from '../../../common/mongo'; +const { Schema } = connectionMongo; +import { DatasetDataSchemaType } from '@fastgpt/global/core/dataset/type.d'; +import { TeamCollectionName } from '@fastgpt/global/support/user/team/constant'; +import { DatasetCollectionName } from '../schema'; +import { DatasetColCollectionName } from '../collection/schema'; +import { DatasetDataCollectionName } from './schema'; + +export const DatasetDataTextCollectionName = 'dataset_data_texts'; + +const DatasetDataTextSchema = new Schema({ + teamId: { + type: Schema.Types.ObjectId, + ref: TeamCollectionName, + required: true + }, + datasetId: { + type: Schema.Types.ObjectId, + ref: DatasetCollectionName, + required: true + }, + collectionId: { + type: Schema.Types.ObjectId, + ref: DatasetColCollectionName, + required: true + }, + dataId: { + type: Schema.Types.ObjectId, + ref: DatasetDataCollectionName, + required: true + }, + fullTextToken: String +}); + +try { + DatasetDataTextSchema.index({ teamId: 1, datasetId: 1, fullTextToken: 'text' }); + DatasetDataTextSchema.index({ dataId: 1 }, { unique: true }); +} catch (error) { + console.log(error); +} + +export const MongoDatasetDataText = getMongoModel( + DatasetDataTextCollectionName, + DatasetDataTextSchema +); diff --git a/packages/service/core/dataset/data/schema.ts b/packages/service/core/dataset/data/schema.ts index 4d5e6aa40..85dd8a7d2 100644 --- a/packages/service/core/dataset/data/schema.ts +++ b/packages/service/core/dataset/data/schema.ts @@ -1,4 +1,4 @@ -import { connectionMongo, getMongoModel, type Model } from '../../../common/mongo'; +import { connectionMongo, getMongoModel } from '../../../common/mongo'; const { Schema, model, models } = connectionMongo; import { DatasetDataSchemaType } from '@fastgpt/global/core/dataset/type.d'; import { @@ -39,10 +39,6 @@ const DatasetDataSchema = new Schema({ type: String, default: '' }, - fullTextToken: { - type: String, - default: '' - }, indexes: { type: [ { @@ -71,27 +67,34 @@ const DatasetDataSchema = new Schema({ type: Number, default: 0 }, - inited: { - type: Boolean - }, - rebuilding: Boolean + rebuilding: Boolean, + + // Abandon + fullTextToken: String, + initFullText: Boolean }); -// list collection and count data; list data; delete collection(relate data) -DatasetDataSchema.index({ - teamId: 1, - datasetId: 1, - collectionId: 1, - chunkIndex: 1, - updateTime: -1 -}); -// full text index -DatasetDataSchema.index({ teamId: 1, datasetId: 1, fullTextToken: 'text' }); -// Recall vectors after data matching -DatasetDataSchema.index({ teamId: 1, datasetId: 1, collectionId: 1, 'indexes.dataId': 1 }); -DatasetDataSchema.index({ updateTime: 1 }); -// rebuild data -DatasetDataSchema.index({ rebuilding: 1, teamId: 1, datasetId: 1 }); +try { + // list collection and count data; list data; delete collection(relate data) + DatasetDataSchema.index({ + teamId: 1, + datasetId: 1, + collectionId: 1, + chunkIndex: 1, + updateTime: -1 + }); + // FullText tmp full text index + // DatasetDataSchema.index({ teamId: 1, datasetId: 1, fullTextToken: 'text' }); + // Recall vectors after data matching + DatasetDataSchema.index({ teamId: 1, datasetId: 1, collectionId: 1, 'indexes.dataId': 1 }); + DatasetDataSchema.index({ updateTime: 1 }); + // rebuild data + DatasetDataSchema.index({ rebuilding: 1, teamId: 1, datasetId: 1 }); + + DatasetDataSchema.index({ initFullText: 1 }); +} catch (error) { + console.log(error); +} export const MongoDatasetData = getMongoModel( DatasetDataCollectionName, diff --git a/packages/service/core/dataset/read.ts b/packages/service/core/dataset/read.ts index d870f3923..5d84e065b 100644 --- a/packages/service/core/dataset/read.ts +++ b/packages/service/core/dataset/read.ts @@ -7,8 +7,9 @@ import { TextSplitProps, splitText2Chunks } from '@fastgpt/global/common/string/ import axios from 'axios'; import { readRawContentByFileBuffer } from '../../common/file/read/utils'; import { parseFileExtensionFromUrl } from '@fastgpt/global/common/string/tools'; -import { APIFileServer } from '@fastgpt/global/core/dataset/apiDataset'; +import { APIFileServer, FeishuServer, YuqueServer } from '@fastgpt/global/core/dataset/apiDataset'; import { useApiDatasetRequest } from './apiDataset/api'; +import { POST } from '../../common/api/plusRequest'; export const readFileRawTextByUrl = async ({ teamId, @@ -53,7 +54,9 @@ export const readDatasetSourceRawText = async ({ isQAImport, selector, externalFileId, - apiServer + apiServer, + feishuServer, + yuqueServer }: { teamId: string; type: DatasetSourceReadTypeEnum; @@ -63,6 +66,8 @@ export const readDatasetSourceRawText = async ({ selector?: string; // link selector externalFileId?: string; // external file dataset apiServer?: APIFileServer; // api dataset + feishuServer?: FeishuServer; // feishu dataset + yuqueServer?: YuqueServer; // yuque dataset }): Promise => { if (type === DatasetSourceReadTypeEnum.fileLocal) { const { rawText } = await readFileContentFromMongo({ @@ -88,28 +93,45 @@ export const readDatasetSourceRawText = async ({ }); return rawText; } else if (type === DatasetSourceReadTypeEnum.apiFile) { - if (!apiServer) return Promise.reject('apiServer not found'); const rawText = await readApiServerFileContent({ apiServer, + feishuServer, + yuqueServer, apiFileId: sourceId, teamId }); return rawText; } - return ''; }; export const readApiServerFileContent = async ({ apiServer, + feishuServer, + yuqueServer, apiFileId, teamId }: { - apiServer: APIFileServer; + apiServer?: APIFileServer; + feishuServer?: FeishuServer; + yuqueServer?: YuqueServer; apiFileId: string; teamId: string; }) => { - return useApiDatasetRequest({ apiServer }).getFileContent({ teamId, apiFileId }); + if (apiServer) { + return useApiDatasetRequest({ apiServer }).getFileContent({ teamId, apiFileId }); + } + + if (feishuServer || yuqueServer) { + return POST(`/core/dataset/systemApiDataset`, { + type: 'content', + feishuServer, + yuqueServer, + apiFileId + }); + } + + return Promise.reject('No apiServer or feishuServer or yuqueServer'); }; export const rawText2Chunks = ({ diff --git a/packages/service/core/dataset/schema.ts b/packages/service/core/dataset/schema.ts index 490703cd8..f8f80ef4d 100644 --- a/packages/service/core/dataset/schema.ts +++ b/packages/service/core/dataset/schema.ts @@ -90,18 +90,14 @@ const DatasetSchema = new Schema({ apiServer: { type: Object }, + feishuServer: { + type: Object + }, + yuqueServer: { + type: Object + }, - syncSchedule: { - cronString: { - type: String - }, - timezone: { - type: String - } - }, - syncNextTime: { - type: Date - }, + autoSync: Boolean, // abandoned externalReadUrl: { @@ -112,7 +108,6 @@ const DatasetSchema = new Schema({ try { DatasetSchema.index({ teamId: 1 }); - DatasetSchema.index({ syncSchedule: 1, syncNextTime: -1 }); } catch (error) { console.log(error); } diff --git a/packages/service/core/dataset/search/controller.ts b/packages/service/core/dataset/search/controller.ts index 0667a2b1e..0de4c2884 100644 --- a/packages/service/core/dataset/search/controller.ts +++ b/packages/service/core/dataset/search/controller.ts @@ -9,7 +9,7 @@ import { getVectorModel } from '../../ai/model'; import { MongoDatasetData } from '../data/schema'; import { DatasetDataSchemaType, - DatasetDataWithCollectionType, + DatasetDataTextSchemaType, SearchDataResponseItemType } from '@fastgpt/global/core/dataset/type'; import { MongoDatasetCollection } from '../collection/schema'; @@ -23,6 +23,7 @@ import { Types } from '../../../common/mongo'; import json5 from 'json5'; import { MongoDatasetCollectionTags } from '../tag/schema'; import { readFromSecondary } from '../../../common/mongo/utils'; +import { MongoDatasetDataText } from '../data/dataTextSchema'; type SearchDatasetDataProps = { teamId: string; @@ -266,54 +267,60 @@ export async function searchDatasetData(props: SearchDatasetDataProps) { filterCollectionIdList }); - // get q and a - const dataList = (await MongoDatasetData.find( - { - teamId, - datasetId: { $in: datasetIds }, - collectionId: { $in: Array.from(new Set(results.map((item) => item.collectionId))) }, - 'indexes.dataId': { $in: results.map((item) => item.id?.trim()) } - }, - 'datasetId collectionId updateTime q a chunkIndex indexes' - ) - .populate('collectionId', 'name fileId rawLink externalFileId externalFileUrl') - .lean()) as DatasetDataWithCollectionType[]; + // Get data and collections + const collectionIdList = Array.from(new Set(results.map((item) => item.collectionId))); + const [dataList, collections] = await Promise.all([ + MongoDatasetData.find( + { + teamId, + datasetId: { $in: datasetIds }, + collectionId: { $in: collectionIdList }, + 'indexes.dataId': { $in: results.map((item) => item.id?.trim()) } + }, + '_id datasetId collectionId updateTime q a chunkIndex indexes', + { ...readFromSecondary } + ).lean(), + MongoDatasetCollection.find( + { + _id: { $in: collectionIdList } + }, + '_id name fileId rawLink externalFileId externalFileUrl', + { ...readFromSecondary } + ).lean() + ]); - // add score to data(It's already sorted. The first one is the one with the most points) - const concatResults = dataList.map((data) => { - const dataIdList = data.indexes.map((item) => item.dataId); + const formatResult = results + .map((item, index) => { + const collection = collections.find((col) => String(col._id) === String(item.collectionId)); + if (!collection) { + console.log('Collection is not found', item); + return; + } + const data = dataList.find((data) => + data.indexes.some((index) => index.dataId === item.id) + ); + if (!data) { + console.log('Data is not found', item); + return; + } - const maxScoreResult = results.find((item) => { - return dataIdList.includes(item.id); - }); + const score = item?.score || 0; - return { - ...data, - score: maxScoreResult?.score || 0 - }; - }); + const result: SearchDataResponseItemType = { + id: String(data._id), + updateTime: data.updateTime, + q: data.q, + a: data.a, + chunkIndex: data.chunkIndex, + datasetId: String(data.datasetId), + collectionId: String(data.collectionId), + ...getCollectionSourceData(collection), + score: [{ type: SearchScoreTypeEnum.embedding, value: score, index }] + }; - concatResults.sort((a, b) => b.score - a.score); - - const formatResult = concatResults.map((data, index) => { - if (!data.collectionId) { - console.log('Collection is not found', data); - } - - const result: SearchDataResponseItemType = { - id: String(data._id), - updateTime: data.updateTime, - q: data.q, - a: data.a, - chunkIndex: data.chunkIndex, - datasetId: String(data.datasetId), - collectionId: String(data.collectionId?._id), - ...getCollectionSourceData(data.collectionId), - score: [{ type: SearchScoreTypeEnum.embedding, value: data.score, index }] - }; - - return result; - }); + return result; + }) + .filter(Boolean) as SearchDataResponseItemType[]; return { embeddingRecallResults: formatResult, @@ -341,88 +348,224 @@ export async function searchDatasetData(props: SearchDatasetDataProps) { }; } - let searchResults = ( + const searchResults = ( await Promise.all( datasetIds.map(async (id) => { - return MongoDatasetData.aggregate([ - { - $match: { - teamId: new Types.ObjectId(teamId), - datasetId: new Types.ObjectId(id), - $text: { $search: jiebaSplit({ text: query }) }, - ...(filterCollectionIdList - ? { - collectionId: { - $in: filterCollectionIdList.map((id) => new Types.ObjectId(id)) + return MongoDatasetData.aggregate( + [ + { + $match: { + teamId: new Types.ObjectId(teamId), + datasetId: new Types.ObjectId(id), + $text: { $search: jiebaSplit({ text: query }) }, + ...(filterCollectionIdList + ? { + collectionId: { + $in: filterCollectionIdList.map((id) => new Types.ObjectId(id)) + } } - } - : {}), - ...(forbidCollectionIdList && forbidCollectionIdList.length > 0 - ? { - collectionId: { - $nin: forbidCollectionIdList.map((id) => new Types.ObjectId(id)) + : {}), + ...(forbidCollectionIdList && forbidCollectionIdList.length > 0 + ? { + collectionId: { + $nin: forbidCollectionIdList.map((id) => new Types.ObjectId(id)) + } } - } - : {}) + : {}) + } + }, + { + $sort: { + score: { $meta: 'textScore' } + } + }, + { + $limit: limit + }, + { + $project: { + _id: 1, + datasetId: 1, + collectionId: 1, + updateTime: 1, + q: 1, + a: 1, + chunkIndex: 1, + score: { $meta: 'textScore' } + } } - }, + ], { - $addFields: { - score: { $meta: 'textScore' } - } - }, - { - $sort: { - score: { $meta: 'textScore' } - } - }, - { - $limit: limit - }, - { - $project: { - _id: 1, - datasetId: 1, - collectionId: 1, - updateTime: 1, - q: 1, - a: 1, - chunkIndex: 1, - score: 1 - } + ...readFromSecondary } - ]); + ); }) ) ).flat() as (DatasetDataSchemaType & { score: number })[]; - // resort - searchResults.sort((a, b) => b.score - a.score); - searchResults.slice(0, limit); - + // Get data and collections const collections = await MongoDatasetCollection.find( { _id: { $in: searchResults.map((item) => item.collectionId) } }, - '_id name fileId rawLink' - ); + '_id name fileId rawLink externalFileId externalFileUrl', + { ...readFromSecondary } + ).lean(); return { - fullTextRecallResults: searchResults.map((item, index) => { - const collection = collections.find((col) => String(col._id) === String(item.collectionId)); - return { - id: String(item._id), - datasetId: String(item.datasetId), - collectionId: String(item.collectionId), - updateTime: item.updateTime, - ...getCollectionSourceData(collection), - q: item.q, - a: item.a, - chunkIndex: item.chunkIndex, - indexes: item.indexes, - score: [{ type: SearchScoreTypeEnum.fullText, value: item.score, index }] - }; - }), + fullTextRecallResults: searchResults + .map((data, index) => { + const collection = collections.find( + (col) => String(col._id) === String(data.collectionId) + ); + if (!collection) { + console.log('Collection is not found', data); + return; + } + + return { + id: String(data._id), + datasetId: String(data.datasetId), + collectionId: String(data.collectionId), + updateTime: data.updateTime, + q: data.q, + a: data.a, + chunkIndex: data.chunkIndex, + indexes: data.indexes, + ...getCollectionSourceData(collection), + score: [{ type: SearchScoreTypeEnum.fullText, value: data.score ?? 0, index }] + }; + }) + .filter(Boolean) as SearchDataResponseItemType[], + tokenLen: 0 + }; + }; + const fullTextRecall2 = async ({ + query, + limit, + filterCollectionIdList, + forbidCollectionIdList + }: { + query: string; + limit: number; + filterCollectionIdList?: string[]; + forbidCollectionIdList: string[]; + }): Promise<{ + fullTextRecallResults: SearchDataResponseItemType[]; + tokenLen: number; + }> => { + if (limit === 0) { + return { + fullTextRecallResults: [], + tokenLen: 0 + }; + } + + const searchResults = ( + await Promise.all( + datasetIds.map(async (id) => { + return MongoDatasetDataText.aggregate( + [ + { + $match: { + teamId: new Types.ObjectId(teamId), + datasetId: new Types.ObjectId(id), + $text: { $search: jiebaSplit({ text: query }) }, + ...(filterCollectionIdList + ? { + collectionId: { + $in: filterCollectionIdList.map((id) => new Types.ObjectId(id)) + } + } + : {}), + ...(forbidCollectionIdList && forbidCollectionIdList.length > 0 + ? { + collectionId: { + $nin: forbidCollectionIdList.map((id) => new Types.ObjectId(id)) + } + } + : {}) + } + }, + { + $sort: { + score: { $meta: 'textScore' } + } + }, + { + $limit: limit + }, + { + $project: { + _id: 1, + collectionId: 1, + dataId: 1, + score: { $meta: 'textScore' } + } + } + ], + { + ...readFromSecondary + } + ); + }) + ) + ).flat() as (DatasetDataTextSchemaType & { score: number })[]; + + // Get data and collections + const [dataList, collections] = await Promise.all([ + MongoDatasetData.find( + { + _id: { $in: searchResults.map((item) => item.dataId) } + }, + '_id datasetId collectionId updateTime q a chunkIndex indexes', + { ...readFromSecondary } + ).lean(), + MongoDatasetCollection.find( + { + _id: { $in: searchResults.map((item) => item.collectionId) } + }, + '_id name fileId rawLink externalFileId externalFileUrl', + { ...readFromSecondary } + ).lean() + ]); + + return { + fullTextRecallResults: searchResults + .map((item, index) => { + const collection = collections.find( + (col) => String(col._id) === String(item.collectionId) + ); + if (!collection) { + console.log('Collection is not found', item); + return; + } + const data = dataList.find((data) => String(data._id) === String(item.dataId)); + if (!data) { + console.log('Data is not found', item); + return; + } + + return { + id: String(data._id), + datasetId: String(data.datasetId), + collectionId: String(data.collectionId), + updateTime: data.updateTime, + q: data.q, + a: data.a, + chunkIndex: data.chunkIndex, + indexes: data.indexes, + ...getCollectionSourceData(collection), + score: [ + { + type: SearchScoreTypeEnum.fullText, + value: item.score || 0, + index + } + ] + }; + }) + .filter(Boolean) as SearchDataResponseItemType[], tokenLen: 0 }; }; @@ -493,7 +636,8 @@ export async function searchDatasetData(props: SearchDatasetDataProps) { forbidCollectionIdList, filterCollectionIdList }), - fullTextRecall({ + // FullText tmp + fullTextRecall2({ query, limit: fullTextLimit, filterCollectionIdList, diff --git a/packages/service/core/dataset/training/controller.ts b/packages/service/core/dataset/training/controller.ts index fb01e709c..fdae75d42 100644 --- a/packages/service/core/dataset/training/controller.ts +++ b/packages/service/core/dataset/training/controller.ts @@ -34,7 +34,7 @@ export const pushDataListToTrainingQueueByCollectionId = async ({ session?: ClientSession; } & PushDatasetDataProps) => { const { - datasetId: { _id: datasetId, agentModel, vectorModel } + dataset: { _id: datasetId, agentModel, vectorModel } } = await getCollectionWithDataset(collectionId); return pushDataListToTrainingQueue({ ...props, @@ -165,7 +165,8 @@ export async function pushDataListToTrainingQueue({ a: item.a, chunkIndex: item.chunkIndex ?? 0, weight: weight ?? 0, - indexes: item.indexes + indexes: item.indexes, + retryCount: 5 })), { session, diff --git a/packages/service/core/plugin/type.d.ts b/packages/service/core/plugin/type.d.ts index 3c66d0c85..d018bea6e 100644 --- a/packages/service/core/plugin/type.d.ts +++ b/packages/service/core/plugin/type.d.ts @@ -2,6 +2,5 @@ import { PluginTemplateType } from '@fastgpt/global/core/plugin/type.d'; import { SystemPluginTemplateItemType } from '@fastgpt/global/core/workflow/type'; declare global { - var communityPluginsV1: PluginTemplateType[]; var communityPlugins: SystemPluginTemplateItemType[]; } diff --git a/packages/service/core/workflow/dispatch/agent/classifyQuestion.ts b/packages/service/core/workflow/dispatch/agent/classifyQuestion.ts index 1558ee704..12091d3e2 100644 --- a/packages/service/core/workflow/dispatch/agent/classifyQuestion.ts +++ b/packages/service/core/workflow/dispatch/agent/classifyQuestion.ts @@ -1,5 +1,8 @@ import { chats2GPTMessages } from '@fastgpt/global/core/chat/adapt'; -import { countMessagesTokens } from '../../../../common/string/tiktoken/index'; +import { + countGptMessagesTokens, + countPromptTokens +} from '../../../../common/string/tiktoken/index'; import type { ChatItemType } from '@fastgpt/global/core/chat/type.d'; import { ChatItemValueTypeEnum, ChatRoleEnum } from '@fastgpt/global/core/chat/constants'; import { createChatCompletion } from '../../../ai/config'; @@ -35,7 +38,7 @@ type ActionProps = Props & { cqModel: LLMModelItemType }; /* request openai chat */ export const dispatchClassifyQuestion = async (props: Props): Promise => { const { - user, + externalProvider, node: { nodeId, name }, histories, params: { model, history = 6, agents, userChatInput } @@ -49,7 +52,7 @@ export const dispatchClassifyQuestion = async (props: Props): Promise item.key !== result.key) .map((item) => getHandleId(nodeId, 'source', item.key)), [DispatchNodeResponseKeyEnum.nodeResponse]: { - totalPoints: user.openaiAccount?.key ? 0 : totalPoints, + totalPoints: externalProvider.openaiAccount?.key ? 0 : totalPoints, model: modelName, query: userChatInput, - tokens, + inputTokens: inputTokens, + outputTokens: outputTokens, cqList: agents, cqResult: result.value, contextTotalLen: chatHistories.length + 2 @@ -80,9 +85,10 @@ export const dispatchClassifyQuestion = async (props: Props): Promise { @@ -131,7 +137,7 @@ const completions = async ({ }, cqModel ), - userKey: user.openaiAccount + userKey: externalProvider.openaiAccount }); const answer = data.choices?.[0].message?.content || ''; @@ -148,7 +154,8 @@ const completions = async ({ } return { - tokens: await countMessagesTokens(messages), + inputTokens: await countGptMessagesTokens(requestMessages), + outputTokens: await countPromptTokens(answer), arg: { type: id } }; }; diff --git a/packages/service/core/workflow/dispatch/agent/extract.ts b/packages/service/core/workflow/dispatch/agent/extract.ts index 84e257b1f..b4067ed39 100644 --- a/packages/service/core/workflow/dispatch/agent/extract.ts +++ b/packages/service/core/workflow/dispatch/agent/extract.ts @@ -3,7 +3,8 @@ import { filterGPTMessageByMaxTokens, loadRequestMessages } from '../../../chat/ import type { ChatItemType } from '@fastgpt/global/core/chat/type.d'; import { countMessagesTokens, - countGptMessagesTokens + countGptMessagesTokens, + countPromptTokens } from '../../../../common/string/tiktoken/index'; import { ChatItemValueTypeEnum, ChatRoleEnum } from '@fastgpt/global/core/chat/constants'; import { createChatCompletion } from '../../../ai/config'; @@ -46,7 +47,7 @@ const agentFunName = 'request_function'; export async function dispatchContentExtract(props: Props): Promise { const { - user, + externalProvider, node: { name }, histories, params: { content, history = 6, model, description, extractKeys } @@ -59,7 +60,7 @@ export async function dispatchContentExtract(props: Props): Promise { const extractModel = getLLMModel(model); const chatHistories = getHistories(history, histories); - const { arg, tokens } = await (async () => { + const { arg, inputTokens, outputTokens } = await (async () => { if (extractModel.toolChoice) { return toolChoice({ ...props, @@ -114,7 +115,8 @@ export async function dispatchContentExtract(props: Props): Promise { const { totalPoints, modelName } = formatModelChars2Points({ model: extractModel.model, - tokens, + inputTokens: inputTokens, + outputTokens: outputTokens, modelType: ModelTypeEnum.llm }); @@ -123,10 +125,11 @@ export async function dispatchContentExtract(props: Props): Promise { [NodeOutputKeyEnum.contextExtractFields]: JSON.stringify(arg), ...arg, [DispatchNodeResponseKeyEnum.nodeResponse]: { - totalPoints: user.openaiAccount?.key ? 0 : totalPoints, + totalPoints: externalProvider.openaiAccount?.key ? 0 : totalPoints, model: modelName, query: content, - tokens, + inputTokens, + outputTokens, extractDescription: description, extractResult: arg, contextTotalLen: chatHistories.length + 2 @@ -134,9 +137,10 @@ export async function dispatchContentExtract(props: Props): Promise { [DispatchNodeResponseKeyEnum.nodeDispatchUsages]: [ { moduleName: name, - totalPoints: user.openaiAccount?.key ? 0 : totalPoints, + totalPoints: externalProvider.openaiAccount?.key ? 0 : totalPoints, model: modelName, - tokens + inputTokens, + outputTokens } ] }; @@ -211,7 +215,7 @@ ${description ? `- ${description}` : ''} }; const toolChoice = async (props: ActionProps) => { - const { user, extractModel } = props; + const { externalProvider, extractModel } = props; const { filterMessages, agentFunction } = await getFunctionCallSchema(props); @@ -233,7 +237,7 @@ const toolChoice = async (props: ActionProps) => { }, extractModel ), - userKey: user.openaiAccount + userKey: externalProvider.openaiAccount }); const arg: Record = (() => { @@ -249,21 +253,24 @@ const toolChoice = async (props: ActionProps) => { } })(); - const completeMessages: ChatCompletionMessageParam[] = [ - ...filterMessages, + const AIMessages: ChatCompletionMessageParam[] = [ { role: ChatCompletionRequestMessageRoleEnum.Assistant, tool_calls: response.choices?.[0]?.message?.tool_calls } ]; + + const inputTokens = await countGptMessagesTokens(filterMessages, tools); + const outputTokens = await countGptMessagesTokens(AIMessages); return { - tokens: await countGptMessagesTokens(completeMessages, tools), + inputTokens, + outputTokens, arg }; }; const functionCall = async (props: ActionProps) => { - const { user, extractModel } = props; + const { externalProvider, extractModel } = props; const { agentFunction, filterMessages } = await getFunctionCallSchema(props); const functions: ChatCompletionCreateParams.Function[] = [agentFunction]; @@ -281,22 +288,26 @@ const functionCall = async (props: ActionProps) => { }, extractModel ), - userKey: user.openaiAccount + userKey: externalProvider.openaiAccount }); try { const arg = JSON.parse(response?.choices?.[0]?.message?.function_call?.arguments || ''); - const completeMessages: ChatCompletionMessageParam[] = [ - ...filterMessages, + + const AIMessages: ChatCompletionMessageParam[] = [ { role: ChatCompletionRequestMessageRoleEnum.Assistant, function_call: response.choices?.[0]?.message?.function_call } ]; + const inputTokens = await countGptMessagesTokens(filterMessages, undefined, functions); + const outputTokens = await countGptMessagesTokens(AIMessages); + return { arg, - tokens: await countGptMessagesTokens(completeMessages, undefined, functions) + inputTokens, + outputTokens }; } catch (error) { console.log(response.choices?.[0]?.message); @@ -305,14 +316,15 @@ const functionCall = async (props: ActionProps) => { return { arg: {}, - tokens: 0 + inputTokens: 0, + outputTokens: 0 }; } }; const completions = async ({ extractModel, - user, + externalProvider, histories, params: { content, extractKeys, description = 'No special requirements' } }: ActionProps) => { @@ -360,7 +372,7 @@ Human: ${content}` }, extractModel ), - userKey: user.openaiAccount + userKey: externalProvider.openaiAccount }); const answer = data.choices?.[0].message?.content || ''; @@ -370,7 +382,8 @@ Human: ${content}` if (!jsonStr) { return { rawResponse: answer, - tokens: await countMessagesTokens(messages), + inputTokens: await countMessagesTokens(messages), + outputTokens: await countPromptTokens(answer), arg: {} }; } @@ -378,7 +391,8 @@ Human: ${content}` try { return { rawResponse: answer, - tokens: await countMessagesTokens(messages), + inputTokens: await countMessagesTokens(messages), + outputTokens: await countPromptTokens(answer), arg: json5.parse(jsonStr) as Record }; } catch (error) { @@ -386,7 +400,8 @@ Human: ${content}` console.log(error); return { rawResponse: answer, - tokens: await countMessagesTokens(messages), + inputTokens: await countMessagesTokens(messages), + outputTokens: await countPromptTokens(answer), arg: {} }; } diff --git a/packages/service/core/workflow/dispatch/agent/runTool/functionCall.ts b/packages/service/core/workflow/dispatch/agent/runTool/functionCall.ts index daab1c0aa..92cdfa9cd 100644 --- a/packages/service/core/workflow/dispatch/agent/runTool/functionCall.ts +++ b/packages/service/core/workflow/dispatch/agent/runTool/functionCall.ts @@ -43,10 +43,10 @@ export const runToolWithFunctionCall = async ( requestOrigin, runtimeNodes, runtimeEdges, - user, + externalProvider, stream, workflowStreamResponse, - params: { temperature = 0, maxToken = 4000, aiChatVision } + params: { temperature, maxToken, aiChatVision } } = workflowProps; // Interactive @@ -109,7 +109,8 @@ export const runToolWithFunctionCall = async ( return { dispatchFlowResponse: [toolRunResponse], - toolNodeTokens: 0, + toolNodeInputTokens: 0, + toolNodeOutputTokens: 0, completeMessages: requestMessages, assistantResponses: toolRunResponse.assistantResponses, runTimes: toolRunResponse.runTimes, @@ -126,7 +127,8 @@ export const runToolWithFunctionCall = async ( }, { dispatchFlowResponse: [toolRunResponse], - toolNodeTokens: 0, + toolNodeInputTokens: 0, + toolNodeOutputTokens: 0, assistantResponses: toolRunResponse.assistantResponses, runTimes: toolRunResponse.runTimes } @@ -221,7 +223,7 @@ export const runToolWithFunctionCall = async ( getEmptyResponseTip } = await createChatCompletion({ body: requestBody, - userKey: user.openaiAccount, + userKey: externalProvider.openaiAccount, options: { headers: { Accept: 'application/json, text/plain, */*' @@ -340,7 +342,9 @@ export const runToolWithFunctionCall = async ( assistantToolMsgParams ] as ChatCompletionMessageParam[]; // Only toolCall tokens are counted here, Tool response tokens count towards the next reply - const tokens = await countGptMessagesTokens(concatToolMessages, undefined, functions); + // const tokens = await countGptMessagesTokens(concatToolMessages, undefined, functions); + const inputTokens = await countGptMessagesTokens(requestMessages, undefined, functions); + const outputTokens = await countGptMessagesTokens([assistantToolMsgParams]); /* ... user @@ -375,7 +379,12 @@ export const runToolWithFunctionCall = async ( const runTimes = (response?.runTimes || 0) + flatToolsResponseData.reduce((sum, item) => sum + item.runTimes, 0); - const toolNodeTokens = response?.toolNodeTokens ? response.toolNodeTokens + tokens : tokens; + const toolNodeInputTokens = response?.toolNodeInputTokens + ? response.toolNodeInputTokens + inputTokens + : inputTokens; + const toolNodeOutputTokens = response?.toolNodeOutputTokens + ? response.toolNodeOutputTokens + outputTokens + : outputTokens; // Check stop signal const hasStopSignal = flatToolsResponseData.some( @@ -408,7 +417,8 @@ export const runToolWithFunctionCall = async ( return { dispatchFlowResponse, - toolNodeTokens, + toolNodeInputTokens, + toolNodeOutputTokens, completeMessages, assistantResponses: toolNodeAssistants, runTimes, @@ -423,7 +433,8 @@ export const runToolWithFunctionCall = async ( }, { dispatchFlowResponse, - toolNodeTokens, + toolNodeInputTokens, + toolNodeOutputTokens, assistantResponses: toolNodeAssistants, runTimes } @@ -435,7 +446,8 @@ export const runToolWithFunctionCall = async ( content: answer }; const completeMessages = filterMessages.concat(gptAssistantResponse); - const tokens = await countGptMessagesTokens(completeMessages, undefined, functions); + const inputTokens = await countGptMessagesTokens(requestMessages, undefined, functions); + const outputTokens = await countGptMessagesTokens([gptAssistantResponse]); // console.log(tokens, 'response token'); // concat tool assistant @@ -443,7 +455,12 @@ export const runToolWithFunctionCall = async ( return { dispatchFlowResponse: response?.dispatchFlowResponse || [], - toolNodeTokens: response?.toolNodeTokens ? response.toolNodeTokens + tokens : tokens, + toolNodeInputTokens: response?.toolNodeInputTokens + ? response.toolNodeInputTokens + inputTokens + : inputTokens, + toolNodeOutputTokens: response?.toolNodeOutputTokens + ? response.toolNodeOutputTokens + outputTokens + : outputTokens, completeMessages, assistantResponses: [...assistantResponses, ...toolNodeAssistant.value], runTimes: (response?.runTimes || 0) + 1 diff --git a/packages/service/core/workflow/dispatch/agent/runTool/index.ts b/packages/service/core/workflow/dispatch/agent/runTool/index.ts index b70623189..590e9ab08 100644 --- a/packages/service/core/workflow/dispatch/agent/runTool/index.ts +++ b/packages/service/core/workflow/dispatch/agent/runTool/index.ts @@ -46,7 +46,7 @@ export const dispatchRunTools = async (props: DispatchToolModuleProps): Promise< requestOrigin, chatConfig, runningAppInfo: { teamId }, - user, + externalProvider, params: { model, systemPrompt, @@ -153,7 +153,7 @@ export const dispatchRunTools = async (props: DispatchToolModuleProps): Promise< })(); // censor model and system key - if (toolModel.censor && !user.openaiAccount?.key) { + if (toolModel.censor && !externalProvider.openaiAccount?.key) { await postTextCensor({ text: `${systemPrompt} ${userChatInput} @@ -165,6 +165,8 @@ export const dispatchRunTools = async (props: DispatchToolModuleProps): Promise< toolWorkflowInteractiveResponse, dispatchFlowResponse, // tool flow response toolNodeTokens, + toolNodeInputTokens, + toolNodeOutputTokens, completeMessages = [], // The actual message sent to AI(just save text) assistantResponses = [], // FastGPT system store assistant.value response runTimes @@ -225,10 +227,11 @@ export const dispatchRunTools = async (props: DispatchToolModuleProps): Promise< const { totalPoints, modelName } = formatModelChars2Points({ model, - tokens: toolNodeTokens, + inputTokens: toolNodeInputTokens, + outputTokens: toolNodeOutputTokens, modelType: ModelTypeEnum.llm }); - const toolAIUsage = user.openaiAccount?.key ? 0 : totalPoints; + const toolAIUsage = externalProvider.openaiAccount?.key ? 0 : totalPoints; // flat child tool response const childToolResponse = dispatchFlowResponse.map((item) => item.flowResponses).flat(); @@ -255,6 +258,8 @@ export const dispatchRunTools = async (props: DispatchToolModuleProps): Promise< // 展示的积分消耗 totalPoints: totalPointsUsage, toolCallTokens: toolNodeTokens, + toolCallInputTokens: toolNodeInputTokens, + toolCallOutputTokens: toolNodeOutputTokens, childTotalPoints: flatUsages.reduce((sum, item) => sum + item.totalPoints, 0), model: modelName, query: userChatInput, @@ -270,9 +275,10 @@ export const dispatchRunTools = async (props: DispatchToolModuleProps): Promise< // 工具调用本身的积分消耗 { moduleName: name, - totalPoints: toolAIUsage, model: modelName, - tokens: toolNodeTokens + totalPoints: toolAIUsage, + inputTokens: toolNodeInputTokens, + outputTokens: toolNodeOutputTokens }, // 工具的消耗 ...flatUsages diff --git a/packages/service/core/workflow/dispatch/agent/runTool/promptCall.ts b/packages/service/core/workflow/dispatch/agent/runTool/promptCall.ts index 0dc11e056..7f380b55d 100644 --- a/packages/service/core/workflow/dispatch/agent/runTool/promptCall.ts +++ b/packages/service/core/workflow/dispatch/agent/runTool/promptCall.ts @@ -51,10 +51,10 @@ export const runToolWithPromptCall = async ( requestOrigin, runtimeNodes, runtimeEdges, - user, + externalProvider, stream, workflowStreamResponse, - params: { temperature = 0, maxToken = 4000, aiChatVision } + params: { temperature, maxToken, aiChatVision } } = workflowProps; if (interactiveEntryToolParams) { @@ -115,7 +115,8 @@ export const runToolWithPromptCall = async ( return { dispatchFlowResponse: [toolRunResponse], - toolNodeTokens: 0, + toolNodeInputTokens: 0, + toolNodeOutputTokens: 0, completeMessages: concatMessages, assistantResponses: toolRunResponse.assistantResponses, runTimes: toolRunResponse.runTimes, @@ -131,7 +132,8 @@ export const runToolWithPromptCall = async ( }, { dispatchFlowResponse: [toolRunResponse], - toolNodeTokens: 0, + toolNodeInputTokens: 0, + toolNodeOutputTokens: 0, assistantResponses: toolRunResponse.assistantResponses, runTimes: toolRunResponse.runTimes } @@ -230,7 +232,7 @@ export const runToolWithPromptCall = async ( getEmptyResponseTip } = await createChatCompletion({ body: requestBody, - userKey: user.openaiAccount, + userKey: externalProvider.openaiAccount, options: { headers: { Accept: 'application/json, text/plain, */*' @@ -286,15 +288,20 @@ export const runToolWithPromptCall = async ( content: replaceAnswer }; const completeMessages = filterMessages.concat(gptAssistantResponse); - const tokens = await countGptMessagesTokens(completeMessages, undefined); - // console.log(tokens, 'response token'); + const inputTokens = await countGptMessagesTokens(requestMessages); + const outputTokens = await countGptMessagesTokens([gptAssistantResponse]); // concat tool assistant const toolNodeAssistant = GPTMessages2Chats([gptAssistantResponse])[0] as AIChatItemType; return { dispatchFlowResponse: response?.dispatchFlowResponse || [], - toolNodeTokens: response?.toolNodeTokens ? response.toolNodeTokens + tokens : tokens, + toolNodeInputTokens: response?.toolNodeInputTokens + ? response.toolNodeInputTokens + inputTokens + : inputTokens, + toolNodeOutputTokens: response?.toolNodeOutputTokens + ? response.toolNodeOutputTokens + outputTokens + : outputTokens, completeMessages, assistantResponses: [...assistantResponses, ...toolNodeAssistant.value], runTimes: (response?.runTimes || 0) + 1 @@ -366,17 +373,9 @@ export const runToolWithPromptCall = async ( function_call: toolJson }; - /* - ... - user - assistant: tool data - */ - const concatToolMessages = [ - ...requestMessages, - assistantToolMsgParams - ] as ChatCompletionMessageParam[]; // Only toolCall tokens are counted here, Tool response tokens count towards the next reply - const tokens = await countGptMessagesTokens(concatToolMessages, undefined); + const inputTokens = await countGptMessagesTokens(requestMessages); + const outputTokens = await countGptMessagesTokens([assistantToolMsgParams]); /* ... @@ -437,7 +436,12 @@ ANSWER: `; } const runTimes = (response?.runTimes || 0) + toolsRunResponse.toolResponse.runTimes; - const toolNodeTokens = response?.toolNodeTokens ? response.toolNodeTokens + tokens : tokens; + const toolNodeInputTokens = response?.toolNodeInputTokens + ? response.toolNodeInputTokens + inputTokens + : inputTokens; + const toolNodeOutputTokens = response?.toolNodeOutputTokens + ? response.toolNodeOutputTokens + outputTokens + : outputTokens; // Check stop signal const hasStopSignal = toolsRunResponse.toolResponse.flowResponses.some((item) => !!item.toolStop); @@ -460,7 +464,8 @@ ANSWER: `; return { dispatchFlowResponse, - toolNodeTokens, + toolNodeInputTokens, + toolNodeOutputTokens, completeMessages: filterMessages, assistantResponses: toolNodeAssistants, runTimes, @@ -475,7 +480,8 @@ ANSWER: `; }, { dispatchFlowResponse, - toolNodeTokens, + toolNodeInputTokens, + toolNodeOutputTokens, assistantResponses: toolNodeAssistants, runTimes } diff --git a/packages/service/core/workflow/dispatch/agent/runTool/toolChoice.ts b/packages/service/core/workflow/dispatch/agent/runTool/toolChoice.ts index 80bc641db..71d5f4021 100644 --- a/packages/service/core/workflow/dispatch/agent/runTool/toolChoice.ts +++ b/packages/service/core/workflow/dispatch/agent/runTool/toolChoice.ts @@ -24,14 +24,13 @@ import { AIChatItemType } from '@fastgpt/global/core/chat/type'; import { formatToolResponse, initToolCallEdges, initToolNodes } from './utils'; import { computedMaxToken, llmCompletionsBodyFormat } from '../../../../ai/utils'; import { getNanoid, sliceStrStartEnd } from '@fastgpt/global/common/string/tools'; -import { addLog } from '../../../../../common/system/log'; import { toolValueTypeList } from '@fastgpt/global/core/workflow/constants'; import { WorkflowInteractiveResponseType } from '@fastgpt/global/core/workflow/template/system/interactive/type'; import { ChatItemValueTypeEnum } from '@fastgpt/global/core/chat/constants'; -import { i18nT } from '../../../../../../web/i18n/utils'; +import { getErrText } from '@fastgpt/global/common/error/utils'; type ToolRunResponseType = { - toolRunResponse: DispatchFlowResponse; + toolRunResponse?: DispatchFlowResponse; toolMsgParams: ChatCompletionToolMessageParam; }[]; @@ -92,9 +91,9 @@ export const runToolWithToolChoice = async ( runtimeNodes, runtimeEdges, stream, - user, + externalProvider, workflowStreamResponse, - params: { temperature = 0, maxToken = 4000, aiChatVision } + params: { temperature, maxToken, aiChatVision } } = workflowProps; if (maxRunToolTimes <= 0 && response) { @@ -160,7 +159,8 @@ export const runToolWithToolChoice = async ( return { dispatchFlowResponse: [toolRunResponse], - toolNodeTokens: 0, + toolNodeInputTokens: 0, + toolNodeOutputTokens: 0, completeMessages: requestMessages, assistantResponses: toolRunResponse.assistantResponses, runTimes: toolRunResponse.runTimes, @@ -178,7 +178,8 @@ export const runToolWithToolChoice = async ( }, { dispatchFlowResponse: [toolRunResponse], - toolNodeTokens: 0, + toolNodeInputTokens: 0, + toolNodeOutputTokens: 0, assistantResponses: toolRunResponse.assistantResponses, runTimes: toolRunResponse.runTimes } @@ -209,7 +210,7 @@ export const runToolWithToolChoice = async ( properties[item.key] = { ...jsonSchema, description: item.toolDescription || '', - enum: item.enum?.split('\n').filter(Boolean) || [] + enum: item.enum?.split('\n').filter(Boolean) || undefined }; }); @@ -226,6 +227,7 @@ export const runToolWithToolChoice = async ( } }; }); + // Filter histories by maxToken const filterMessages = ( await filterGPTMessageByMaxTokens({ @@ -278,7 +280,7 @@ export const runToolWithToolChoice = async ( getEmptyResponseTip } = await createChatCompletion({ body: requestBody, - userKey: user.openaiAccount, + userKey: externalProvider.openaiAccount, options: { headers: { Accept: 'application/json, text/plain, */*' @@ -344,59 +346,87 @@ export const runToolWithToolChoice = async ( return Promise.reject(getEmptyResponseTip()); } - // Run the selected tool by LLM. - const toolsRunResponse = ( - await Promise.all( - toolCalls.map(async (tool) => { - const toolNode = toolNodes.find((item) => item.nodeId === tool.function?.name); + /* Run the selected tool by LLM. + Since only reference parameters are passed, if the same tool is run in parallel, it will get the same run parameters + */ + const toolsRunResponse: ToolRunResponseType = []; + for await (const tool of toolCalls) { + try { + const toolNode = toolNodes.find((item) => item.nodeId === tool.function?.name); - if (!toolNode) return; + if (!toolNode) continue; - const startParams = (() => { - try { - return json5.parse(tool.function.arguments); - } catch (error) { - return {}; + const startParams = (() => { + try { + return json5.parse(tool.function.arguments); + } catch (error) { + return {}; + } + })(); + + initToolNodes(runtimeNodes, [toolNode.nodeId], startParams); + const toolRunResponse = await dispatchWorkFlow({ + ...workflowProps, + isToolCall: true + }); + + const stringToolResponse = formatToolResponse(toolRunResponse.toolResponses); + + const toolMsgParams: ChatCompletionToolMessageParam = { + tool_call_id: tool.id, + role: ChatCompletionRequestMessageRoleEnum.Tool, + name: tool.function.name, + content: stringToolResponse + }; + + workflowStreamResponse?.({ + event: SseResponseEventEnum.toolResponse, + data: { + tool: { + id: tool.id, + toolName: '', + toolAvatar: '', + params: '', + response: sliceStrStartEnd(stringToolResponse, 5000, 5000) } - })(); + } + }); - initToolNodes(runtimeNodes, [toolNode.nodeId], startParams); - const toolRunResponse = await dispatchWorkFlow({ - ...workflowProps, - isToolCall: true - }); + toolsRunResponse.push({ + toolRunResponse, + toolMsgParams + }); + } catch (error) { + const err = getErrText(error); + workflowStreamResponse?.({ + event: SseResponseEventEnum.toolResponse, + data: { + tool: { + id: tool.id, + toolName: '', + toolAvatar: '', + params: '', + response: sliceStrStartEnd(err, 5000, 5000) + } + } + }); - const stringToolResponse = formatToolResponse(toolRunResponse.toolResponses); - - const toolMsgParams: ChatCompletionToolMessageParam = { + toolsRunResponse.push({ + toolRunResponse: undefined, + toolMsgParams: { tool_call_id: tool.id, role: ChatCompletionRequestMessageRoleEnum.Tool, name: tool.function.name, - content: stringToolResponse - }; + content: sliceStrStartEnd(err, 5000, 5000) + } + }); + } + } - workflowStreamResponse?.({ - event: SseResponseEventEnum.toolResponse, - data: { - tool: { - id: tool.id, - toolName: '', - toolAvatar: '', - params: '', - response: sliceStrStartEnd(stringToolResponse, 5000, 5000) - } - } - }); - - return { - toolRunResponse, - toolMsgParams - }; - }) - ) - ).filter(Boolean) as ToolRunResponseType; - - const flatToolsResponseData = toolsRunResponse.map((item) => item.toolRunResponse).flat(); + const flatToolsResponseData = toolsRunResponse + .map((item) => item.toolRunResponse) + .flat() + .filter(Boolean) as DispatchFlowResponse[]; // concat tool responses const dispatchFlowResponse = response ? response.dispatchFlowResponse.concat(flatToolsResponseData) @@ -430,24 +460,26 @@ export const runToolWithToolChoice = async ( ] as ChatCompletionMessageParam[]; // Only toolCall tokens are counted here, Tool response tokens count towards the next reply - const tokens = await countGptMessagesTokens(concatToolMessages, tools); + const inputTokens = await countGptMessagesTokens(requestMessages, tools); + const outputTokens = await countGptMessagesTokens(assistantToolMsgParams); + /* - ... - user - assistant: tool data - tool: tool response - */ + ... + user + assistant: tool data + tool: tool response + */ const completeMessages = [ ...concatToolMessages, ...toolsRunResponse.map((item) => item?.toolMsgParams) ]; /* - Get tool node assistant response - history assistant - current tool assistant - tool child assistant - */ + Get tool node assistant response + history assistant + current tool assistant + tool child assistant + */ const toolNodeAssistant = GPTMessages2Chats([ ...assistantToolMsgParams, ...toolsRunResponse.map((item) => item?.toolMsgParams) @@ -465,7 +497,10 @@ export const runToolWithToolChoice = async ( const runTimes = (response?.runTimes || 0) + flatToolsResponseData.reduce((sum, item) => sum + item.runTimes, 0); - const toolNodeTokens = response ? response.toolNodeTokens + tokens : tokens; + const toolNodeInputTokens = response ? response.toolNodeInputTokens + inputTokens : inputTokens; + const toolNodeOutputTokens = response + ? response.toolNodeOutputTokens + outputTokens + : outputTokens; // Check stop signal const hasStopSignal = flatToolsResponseData.some( @@ -473,12 +508,12 @@ export const runToolWithToolChoice = async ( ); // Check interactive response(Only 1 interaction is reserved) const workflowInteractiveResponseItem = toolsRunResponse.find( - (item) => item.toolRunResponse.workflowInteractiveResponse + (item) => item.toolRunResponse?.workflowInteractiveResponse ); if (hasStopSignal || workflowInteractiveResponseItem) { // Get interactive tool data const workflowInteractiveResponse = - workflowInteractiveResponseItem?.toolRunResponse.workflowInteractiveResponse; + workflowInteractiveResponseItem?.toolRunResponse?.workflowInteractiveResponse; // Flashback traverses completeMessages, intercepting messages that know the first user const firstUserIndex = completeMessages.findLastIndex((item) => item.role === 'user'); @@ -498,7 +533,8 @@ export const runToolWithToolChoice = async ( return { dispatchFlowResponse, - toolNodeTokens, + toolNodeInputTokens, + toolNodeOutputTokens, completeMessages, assistantResponses: toolNodeAssistants, runTimes, @@ -514,7 +550,8 @@ export const runToolWithToolChoice = async ( }, { dispatchFlowResponse, - toolNodeTokens, + toolNodeInputTokens, + toolNodeOutputTokens, assistantResponses: toolNodeAssistants, runTimes } @@ -526,14 +563,17 @@ export const runToolWithToolChoice = async ( content: answer }; const completeMessages = filterMessages.concat(gptAssistantResponse); - const tokens = await countGptMessagesTokens(completeMessages, tools); + const inputTokens = await countGptMessagesTokens(requestMessages, tools); + const outputTokens = await countGptMessagesTokens([gptAssistantResponse]); // concat tool assistant const toolNodeAssistant = GPTMessages2Chats([gptAssistantResponse])[0] as AIChatItemType; return { dispatchFlowResponse: response?.dispatchFlowResponse || [], - toolNodeTokens: response ? response.toolNodeTokens + tokens : tokens, + toolNodeInputTokens: response ? response.toolNodeInputTokens + inputTokens : inputTokens, + toolNodeOutputTokens: response ? response.toolNodeOutputTokens + outputTokens : outputTokens, + completeMessages, assistantResponses: [...assistantResponses, ...toolNodeAssistant.value], runTimes: (response?.runTimes || 0) + 1 @@ -580,7 +620,8 @@ async function streamResponse({ text: content }) }); - } else if (responseChoice?.tool_calls?.[0]) { + } + if (responseChoice?.tool_calls?.[0]) { const toolCall: ChatCompletionMessageToolCall = responseChoice.tool_calls[0]; // In a stream response, only one tool is returned at a time. If have id, description is executing a tool if (toolCall.id || callingTool) { diff --git a/packages/service/core/workflow/dispatch/agent/runTool/type.d.ts b/packages/service/core/workflow/dispatch/agent/runTool/type.d.ts index ad2866c13..8d70bc227 100644 --- a/packages/service/core/workflow/dispatch/agent/runTool/type.d.ts +++ b/packages/service/core/workflow/dispatch/agent/runTool/type.d.ts @@ -31,7 +31,9 @@ export type DispatchToolModuleProps = ModuleDispatchProps<{ export type RunToolResponse = { dispatchFlowResponse: DispatchFlowResponse[]; - toolNodeTokens: number; + toolNodeTokens?: number; // deprecated + toolNodeInputTokens: number; + toolNodeOutputTokens: number; completeMessages?: ChatCompletionMessageParam[]; assistantResponses?: AIChatItemValueItemType[]; toolWorkflowInteractiveResponse?: WorkflowInteractiveResponseType; diff --git a/packages/service/core/workflow/dispatch/chat/oneapi.ts b/packages/service/core/workflow/dispatch/chat/oneapi.ts index 59cfe7afb..d20d1bec7 100644 --- a/packages/service/core/workflow/dispatch/chat/oneapi.ts +++ b/packages/service/core/workflow/dispatch/chat/oneapi.ts @@ -5,13 +5,17 @@ import { ChatRoleEnum } from '@fastgpt/global/core/chat/constants'; import { SseResponseEventEnum } from '@fastgpt/global/core/workflow/runtime/constants'; import { textAdaptGptResponse } from '@fastgpt/global/core/workflow/runtime/utils'; import { createChatCompletion } from '../../../ai/config'; -import type { ChatCompletion, StreamChatType } from '@fastgpt/global/core/ai/type.d'; +import type { + ChatCompletion, + ChatCompletionMessageParam, + StreamChatType +} from '@fastgpt/global/core/ai/type.d'; import { formatModelChars2Points } from '../../../../support/wallet/usage/utils'; import type { LLMModelItemType } from '@fastgpt/global/core/ai/model.d'; import { postTextCensor } from '../../../../common/api/requestPlusApi'; import { ChatCompletionRequestMessageRoleEnum } from '@fastgpt/global/core/ai/constants'; import type { DispatchNodeResultType } from '@fastgpt/global/core/workflow/runtime/type'; -import { countMessagesTokens } from '../../../../common/string/tiktoken/index'; +import { countGptMessagesTokens } from '../../../../common/string/tiktoken/index'; import { chats2GPTMessages, chatValue2RuntimePrompt, @@ -62,7 +66,7 @@ export const dispatchChatCompletion = async (props: ChatProps): Promise { // censor model and system key - if (modelConstantsData.censor && !user.openaiAccount?.key) { + if (modelConstantsData.censor && !externalProvider.openaiAccount?.key) { return postTextCensor({ text: `${systemPrompt} ${userChatInput} @@ -170,7 +174,7 @@ export const dispatchChatCompletion = async (props: ChatProps): Promise = { [FlowNodeTypeEnum.workflowStart]: dispatchWorkflowStart, @@ -128,7 +127,8 @@ export async function dispatchWorkFlow(data: Props): Promise item.source === node.nodeId); const skipHandleIds = targetEdges.map((item) => item.sourceHandle); + toolRunResponse = getErrText(error); + // Skip all edges and return error return { [DispatchNodeResponseKeyEnum.nodeResponse]: { @@ -604,6 +606,11 @@ export async function dispatchWorkFlow(data: Props): Promise item.isEntry); // reset entry runtimeNodes.forEach((item) => { - // Interactive node is not the entry node, return interactive result + // Interactively nodes will use the "isEntry", which does not need to be updated if ( item.flowNodeType !== FlowNodeTypeEnum.userSelect && item.flowNodeType !== FlowNodeTypeEnum.formInput && @@ -679,7 +686,7 @@ export async function dispatchWorkFlow(data: Props): Promise => { params, runtimeEdges, runtimeNodes, - user, node: { name } } = props; const { loopInputArray = [], childrenNodeIdList = [] } = params; @@ -86,14 +85,14 @@ export const dispatchLoop = async (props: Props): Promise => { return { [DispatchNodeResponseKeyEnum.assistantResponses]: assistantResponses, [DispatchNodeResponseKeyEnum.nodeResponse]: { - totalPoints: totalPoints, + totalPoints, loopInput: loopInputArray, loopResult: outputValueArr, loopDetail: loopDetail }, [DispatchNodeResponseKeyEnum.nodeDispatchUsages]: [ { - totalPoints: user.openaiAccount?.key ? 0 : totalPoints, + totalPoints, moduleName: name } ], diff --git a/packages/service/core/workflow/dispatch/plugin/run.ts b/packages/service/core/workflow/dispatch/plugin/run.ts index cd75aee1d..f7eb09af5 100644 --- a/packages/service/core/workflow/dispatch/plugin/run.ts +++ b/packages/service/core/workflow/dispatch/plugin/run.ts @@ -130,8 +130,7 @@ export const dispatchRunPlugin = async (props: RunPluginProps): Promise => { [DispatchNodeResponseKeyEnum.nodeDispatchUsages]: [ { moduleName: appData.name, - totalPoints: usagePoints, - tokens: 0 + totalPoints: usagePoints } ], [DispatchNodeResponseKeyEnum.toolResponses]: text, diff --git a/packages/service/core/workflow/dispatch/tools/http468.ts b/packages/service/core/workflow/dispatch/tools/http468.ts index e1afb480a..da7b82192 100644 --- a/packages/service/core/workflow/dispatch/tools/http468.ts +++ b/packages/service/core/workflow/dispatch/tools/http468.ts @@ -2,6 +2,7 @@ import type { ModuleDispatchProps } from '@fastgpt/global/core/workflow/runtime/ import { NodeInputKeyEnum, NodeOutputKeyEnum, + VARIABLE_NODE_ID, WorkflowIOValueTypeEnum } from '@fastgpt/global/core/workflow/constants'; import { @@ -16,7 +17,9 @@ import { DispatchNodeResultType } from '@fastgpt/global/core/workflow/runtime/ty import { getErrText } from '@fastgpt/global/common/error/utils'; import { textAdaptGptResponse, - replaceEditorVariable + replaceEditorVariable, + formatVariableValByType, + getReferenceVariableValue } from '@fastgpt/global/core/workflow/runtime/utils'; import { ContentTypes } from '@fastgpt/global/core/workflow/constants'; import { uploadFileFromBase64Img } from '../../../../common/file/gridfs/controller'; @@ -104,15 +107,73 @@ export const dispatchHttp468Request = async (props: HttpRequestProps): Promise { - return replaceVariable( - replaceEditorVariable({ - text, - nodes: runtimeNodes, - variables: allVariables - }), - allVariables - ); + return replaceEditorVariable({ + text, + nodes: runtimeNodes, + variables: allVariables + }); + }; + /* Replace the JSON string to reduce parsing errors + 1. Replace undefined values with null + 2. Replace newline strings + */ + const replaceJsonBodyString = (text: string) => { + const valToStr = (val: any) => { + if (val === undefined) return 'null'; + if (val === null) return 'null'; + + if (typeof val === 'object') return JSON.stringify(val); + + if (typeof val === 'string') { + const str = JSON.stringify(val); + return str.startsWith('"') && str.endsWith('"') ? str.slice(1, -1) : str; + } + + return String(val); + }; + + // 1. Replace {{key}} variables + const regex1 = /{{([^}]+)}}/g; + const matches1 = text.match(regex1) || []; + const uniqueKeys1 = [...new Set(matches1.map((match) => match.slice(2, -2)))]; + for (const key of uniqueKeys1) { + text = text.replace(new RegExp(`{{(${key})}}`, 'g'), () => valToStr(variables[key])); + } + + // 2. Replace {{key.key}} variables + const regex2 = /\{\{\$([^.]+)\.([^$]+)\$\}\}/g; + const matches2 = [...text.matchAll(regex2)]; + if (matches2.length === 0) return text; + matches2.forEach((match) => { + const nodeId = match[1]; + const id = match[2]; + + const variableVal = (() => { + if (nodeId === VARIABLE_NODE_ID) { + return variables[id]; + } + // Find upstream node input/output + const node = runtimeNodes.find((node) => node.nodeId === nodeId); + if (!node) return; + + const output = node.outputs.find((output) => output.id === id); + if (output) return formatVariableValByType(output.value, output.valueType); + + const input = node.inputs.find((input) => input.key === id); + if (input) + return getReferenceVariableValue({ value: input.value, nodes: runtimeNodes, variables }); + })(); + + const formatVal = valToStr(variableVal); + + const regex = new RegExp(`\\{\\{\\$(${nodeId}\\.${id})\\$\\}\\}`, 'g'); + text = text.replace(regex, () => formatVal); + }); + + return text.replace(/(".*?")\s*:\s*undefined\b/g, '$1: null'); }; httpReqUrl = replaceStringVariables(httpReqUrl); @@ -176,18 +237,13 @@ export const dispatchHttp468Request = async (props: HttpRequestProps): Promise) { - for (const [key, value] of Object.entries(obj)) { - if (value === undefined) { - text = text.replace(new RegExp(`{{(${key})}}`, 'g'), UNDEFINED_SIGN); - } else { - const replacement = JSON.stringify(value); - const unquotedReplacement = - replacement.startsWith('"') && replacement.endsWith('"') - ? replacement.slice(1, -1) - : replacement; - text = text.replace(new RegExp(`{{(${key})}}`, 'g'), () => unquotedReplacement); - } - } - return text || ''; -} -function removeUndefinedSign(obj: Record) { - for (const key in obj) { - if (obj[key] === UNDEFINED_SIGN) { - obj[key] = undefined; - } else if (Array.isArray(obj[key])) { - obj[key] = obj[key].map((item: any) => { - if (item === UNDEFINED_SIGN) { - return undefined; - } else if (typeof item === 'object') { - removeUndefinedSign(item); - } - return item; - }); - } else if (typeof obj[key] === 'object') { - removeUndefinedSign(obj[key]); - } - } - return obj; -} +// function replaceVariable(text: string, obj: Record) { +// for (const [key, value] of Object.entries(obj)) { +// if (value === undefined) { +// text = text.replace(new RegExp(`{{(${key})}}`, 'g'), UNDEFINED_SIGN); +// } else { +// const replacement = JSON.stringify(value); +// const unquotedReplacement = +// replacement.startsWith('"') && replacement.endsWith('"') +// ? replacement.slice(1, -1) +// : replacement; +// text = text.replace(new RegExp(`{{(${key})}}`, 'g'), () => unquotedReplacement); +// } +// } +// return text || ''; +// } +// function removeUndefinedSign(obj: Record) { +// for (const key in obj) { +// if (obj[key] === UNDEFINED_SIGN) { +// obj[key] = undefined; +// } else if (Array.isArray(obj[key])) { +// obj[key] = obj[key].map((item: any) => { +// if (item === UNDEFINED_SIGN) { +// return undefined; +// } else if (typeof item === 'object') { +// removeUndefinedSign(item); +// } +// return item; +// }); +// } else if (typeof obj[key] === 'object') { +// removeUndefinedSign(obj[key]); +// } +// } +// return obj; +// } // Replace some special response from system plugin async function replaceSystemPluginResponse({ diff --git a/packages/service/core/workflow/dispatch/tools/queryExternsion.ts b/packages/service/core/workflow/dispatch/tools/queryExternsion.ts index e442f7656..48ce7dfee 100644 --- a/packages/service/core/workflow/dispatch/tools/queryExternsion.ts +++ b/packages/service/core/workflow/dispatch/tools/queryExternsion.ts @@ -31,7 +31,7 @@ export const dispatchQueryExtension = async ({ const queryExtensionModel = getLLMModel(model); const chatHistories = getHistories(history, histories); - const { extensionQueries, tokens } = await queryExtension({ + const { extensionQueries, inputTokens, outputTokens } = await queryExtension({ chatBg: systemPrompt, query: userChatInput, histories: chatHistories, @@ -42,7 +42,8 @@ export const dispatchQueryExtension = async ({ const { totalPoints, modelName } = formatModelChars2Points({ model: queryExtensionModel.model, - tokens, + inputTokens, + outputTokens, modelType: ModelTypeEnum.llm }); @@ -59,7 +60,8 @@ export const dispatchQueryExtension = async ({ [DispatchNodeResponseKeyEnum.nodeResponse]: { totalPoints, model: modelName, - tokens, + inputTokens, + outputTokens, query: userChatInput, textOutput: JSON.stringify(filterSameQueries) }, @@ -68,7 +70,8 @@ export const dispatchQueryExtension = async ({ moduleName: node.name, totalPoints, model: modelName, - tokens + inputTokens, + outputTokens } ], [NodeOutputKeyEnum.text]: JSON.stringify(filterSameQueries) diff --git a/packages/service/core/workflow/dispatch/tools/runIfElse.ts b/packages/service/core/workflow/dispatch/tools/runIfElse.ts index 3d10698a2..ee66bd226 100644 --- a/packages/service/core/workflow/dispatch/tools/runIfElse.ts +++ b/packages/service/core/workflow/dispatch/tools/runIfElse.ts @@ -53,8 +53,8 @@ function checkCondition(condition: VariableConditionEnum, inputValue: any, value [VariableConditionEnum.isEmpty]: () => isEmpty(inputValue), [VariableConditionEnum.isNotEmpty]: () => !isEmpty(inputValue), - [VariableConditionEnum.equalTo]: () => String(inputValue) === value, - [VariableConditionEnum.notEqual]: () => String(inputValue) !== value, + [VariableConditionEnum.equalTo]: () => String(inputValue).trim() === value.trim(), + [VariableConditionEnum.notEqual]: () => String(inputValue).trim() !== value.trim(), // number [VariableConditionEnum.greaterThan]: () => Number(inputValue) > Number(value), @@ -67,8 +67,8 @@ function checkCondition(condition: VariableConditionEnum, inputValue: any, value [VariableConditionEnum.notInclude]: () => !isInclude(inputValue, value), // string - [VariableConditionEnum.startWith]: () => inputValue?.startsWith(value), - [VariableConditionEnum.endWith]: () => inputValue?.endsWith(value), + [VariableConditionEnum.startWith]: () => inputValue?.trim()?.startsWith(value), + [VariableConditionEnum.endWith]: () => inputValue?.trim()?.endsWith(value), [VariableConditionEnum.reg]: () => { if (typeof inputValue !== 'string' || !value) return false; if (value.startsWith('/')) { @@ -79,7 +79,7 @@ function checkCondition(condition: VariableConditionEnum, inputValue: any, value } const reg = new RegExp(value, 'g'); - const result = reg.test(inputValue); + const result = reg.test(inputValue.trim()); return result; }, diff --git a/packages/service/core/workflow/dispatch/tools/runUpdateVar.ts b/packages/service/core/workflow/dispatch/tools/runUpdateVar.ts index 302c49774..d0f0d64f4 100644 --- a/packages/service/core/workflow/dispatch/tools/runUpdateVar.ts +++ b/packages/service/core/workflow/dispatch/tools/runUpdateVar.ts @@ -19,7 +19,7 @@ type Props = ModuleDispatchProps<{ type Response = DispatchNodeResultType<{}>; export const dispatchUpdateVariable = async (props: Props): Promise => { - const { params, variables, runtimeNodes, workflowStreamResponse, node } = props; + const { params, variables, runtimeNodes, workflowStreamResponse, externalProvider } = props; const { updateList } = params; const nodeIds = runtimeNodes.map((node) => node.nodeId); @@ -41,15 +41,15 @@ export const dispatchUpdateVariable = async (props: Props): Promise => const value = (() => { // If first item is empty, it means it is a input value if (!item.value?.[0]) { - const formatValue = valueTypeFormat(item.value?.[1], item.valueType); - - return typeof formatValue === 'string' - ? replaceEditorVariable({ - text: formatValue, - nodes: runtimeNodes, - variables - }) - : formatValue; + const val = + typeof item.value?.[1] === 'string' + ? replaceEditorVariable({ + text: item.value?.[1], + nodes: runtimeNodes, + variables + }) + : item.value?.[1]; + return valueTypeFormat(val, item.valueType); } else { return getReferenceVariableValue({ value: item.value, @@ -80,7 +80,7 @@ export const dispatchUpdateVariable = async (props: Props): Promise => workflowStreamResponse?.({ event: SseResponseEventEnum.updateVariables, - data: removeSystemVariable(variables) + data: removeSystemVariable(variables, externalProvider.externalWorkflowVariables) }); return { diff --git a/packages/service/core/workflow/dispatch/utils.ts b/packages/service/core/workflow/dispatch/utils.ts index 4186d89ac..55e68815c 100644 --- a/packages/service/core/workflow/dispatch/utils.ts +++ b/packages/service/core/workflow/dispatch/utils.ts @@ -14,6 +14,7 @@ import { NextApiResponse } from 'next'; import { SseResponseEventEnum } from '@fastgpt/global/core/workflow/runtime/constants'; import { getNanoid } from '@fastgpt/global/common/string/tools'; import { SearchDataResponseItemType } from '@fastgpt/global/core/dataset/type'; +import json5 from 'json5'; export const getWorkflowResponseWrite = ({ res, @@ -116,11 +117,18 @@ export const valueTypeFormat = (value: any, type?: WorkflowIOValueTypeEnum) => { return Boolean(value); } try { - if (type === WorkflowIOValueTypeEnum.datasetQuote && !Array.isArray(value)) { - return JSON.parse(value); - } - if (type === WorkflowIOValueTypeEnum.selectDataset && !Array.isArray(value)) { - return JSON.parse(value); + if ( + type && + [ + WorkflowIOValueTypeEnum.object, + WorkflowIOValueTypeEnum.chatHistory, + WorkflowIOValueTypeEnum.datasetQuote, + WorkflowIOValueTypeEnum.selectApp, + WorkflowIOValueTypeEnum.selectDataset + ].includes(type) && + typeof value !== 'object' + ) { + return json5.parse(value); } } catch (error) { return value; @@ -141,14 +149,23 @@ export const checkQuoteQAValue = (quoteQA?: SearchDataResponseItemType[]) => { }; /* remove system variable */ -export const removeSystemVariable = (variables: Record) => { +export const removeSystemVariable = ( + variables: Record, + removeObj: Record = {} +) => { const copyVariables = { ...variables }; + delete copyVariables.userId; delete copyVariables.appId; delete copyVariables.chatId; delete copyVariables.responseChatItemId; delete copyVariables.histories; delete copyVariables.cTime; + // delete external provider workflow variables + Object.keys(removeObj).forEach((key) => { + delete copyVariables[key]; + }); + return copyVariables; }; export const filterSystemVariables = (variables: Record): SystemVariablesType => { diff --git a/packages/service/support/outLink/schema.ts b/packages/service/support/outLink/schema.ts index 4323a808c..0c6d5ea4c 100644 --- a/packages/service/support/outLink/schema.ts +++ b/packages/service/support/outLink/schema.ts @@ -83,8 +83,16 @@ const OutLinkSchema = new Schema({ } }); +OutLinkSchema.virtual('associatedApp', { + ref: AppCollectionName, + localField: 'appId', + foreignField: '_id', + justOne: true +}); + try { OutLinkSchema.index({ shareId: -1 }); + OutLinkSchema.index({ teamId: 1, tmbId: 1, appId: 1 }); } catch (error) { console.log(error); } diff --git a/packages/service/support/permission/auth/org.ts b/packages/service/support/permission/auth/org.ts new file mode 100644 index 000000000..d864b1cb5 --- /dev/null +++ b/packages/service/support/permission/auth/org.ts @@ -0,0 +1,43 @@ +import { TeamPermission } from '@fastgpt/global/support/permission/user/controller'; +import { AuthModeType, AuthResponseType } from '../type'; +import { TeamErrEnum } from '@fastgpt/global/common/error/code/team'; +import { authUserPer } from '../user/auth'; +import { ManagePermissionVal } from '@fastgpt/global/support/permission/constant'; + +/* + Team manager can control org +*/ +export const authOrgMember = async ({ + orgIds, + ...props +}: { + orgIds: string | string[]; +} & AuthModeType): Promise => { + const result = await authUserPer({ + ...props, + per: ManagePermissionVal + }); + const { teamId, tmbId, isRoot, tmb } = result; + + if (isRoot) { + return { + teamId, + tmbId, + userId: result.userId, + appId: result.appId, + apikey: result.apikey, + isRoot, + authType: result.authType, + permission: new TeamPermission({ isOwner: true }) + }; + } + + if (tmb.permission.hasManagePer) { + return { + ...result, + permission: tmb.permission + }; + } + + return Promise.reject(TeamErrEnum.unAuthTeam); +}; diff --git a/packages/service/support/permission/auth/team.ts b/packages/service/support/permission/auth/team.ts index 0a5589649..48d676ed4 100644 --- a/packages/service/support/permission/auth/team.ts +++ b/packages/service/support/permission/auth/team.ts @@ -1,18 +1,32 @@ -import { TeamMemberWithUserSchema } from '@fastgpt/global/support/user/team/type'; import { MongoTeamMember } from '../../user/team/teamMemberSchema'; import { checkTeamAIPoints } from '../teamLimit'; -import { UserErrEnum } from '@fastgpt/global/common/error/code/user'; +import { UserModelSchema } from '@fastgpt/global/support/user/type'; +import { TeamSchema } from '@fastgpt/global/support/user/team/type'; +import { TeamErrEnum } from '@fastgpt/global/common/error/code/team'; export async function getUserChatInfoAndAuthTeamPoints(tmbId: string) { - const tmb = (await MongoTeamMember.findById(tmbId, 'teamId userId').populate( - 'userId', - 'timezone openaiAccount' - )) as TeamMemberWithUserSchema; - if (!tmb) return Promise.reject(UserErrEnum.unAuthUser); + const tmb = await MongoTeamMember.findById(tmbId, 'userId teamId') + .populate<{ user: UserModelSchema; team: TeamSchema }>([ + { + path: 'user', + select: 'timezone' + }, + { + path: 'team', + select: 'openaiAccount externalWorkflowVariables' + } + ]) + .lean(); - await checkTeamAIPoints(tmb.teamId); + if (!tmb) return Promise.reject(TeamErrEnum.notUser); + + await checkTeamAIPoints(tmb.team._id); return { - user: tmb.userId + timezone: tmb.user.timezone, + externalProvider: { + openaiAccount: tmb.team.openaiAccount, + externalWorkflowVariables: tmb.team.externalWorkflowVariables + } }; } diff --git a/packages/service/support/permission/controller.ts b/packages/service/support/permission/controller.ts index c68f07d1b..f233c66b1 100644 --- a/packages/service/support/permission/controller.ts +++ b/packages/service/support/permission/controller.ts @@ -8,19 +8,18 @@ import { authOpenApiKey } from '../openapi/auth'; import { FileTokenQuery } from '@fastgpt/global/common/file/type'; import { MongoResourcePermission } from './schema'; import { ClientSession } from 'mongoose'; -import { - PermissionValueType, - ResourcePermissionType, - ResourcePerWithGroup, - ResourcePerWithTmbWithUser -} from '@fastgpt/global/support/permission/type'; +import { PermissionValueType } from '@fastgpt/global/support/permission/type'; import { bucketNameMap } from '@fastgpt/global/common/file/constants'; import { addMinutes } from 'date-fns'; import { getGroupsByTmbId } from './memberGroup/controllers'; import { Permission } from '@fastgpt/global/support/permission/controller'; import { ParentIdType } from '@fastgpt/global/common/parentFolder/type'; -import { RequireOnlyOne } from '@fastgpt/global/common/type/utils'; import { CommonErrEnum } from '@fastgpt/global/common/error/code/common'; +import { MemberGroupSchemaType } from '@fastgpt/global/support/permission/memberGroup/type'; +import { TeamMemberSchema } from '@fastgpt/global/support/user/team/type'; +import { UserModelSchema } from '@fastgpt/global/support/user/type'; +import { OrgSchemaType } from '@fastgpt/global/support/user/team/org/type'; +import { getOrgIdSetWithParentByTmbId } from './org/controllers'; /** get resource permission for a team member * If there is no permission for the team member, it will return undefined @@ -67,67 +66,44 @@ export const getResourcePermission = async ({ } // If there is no personal permission, get the group permission - const groupIdList = (await getGroupsByTmbId({ tmbId, teamId })).map((item) => item._id); + const [groupPers, orgPers] = await Promise.all([ + getGroupsByTmbId({ tmbId, teamId }) + .then((res) => res.map((item) => item._id)) + .then((groupIdList) => + MongoResourcePermission.find( + { + teamId, + resourceType, + groupId: { + $in: groupIdList + }, + resourceId + }, + 'permission' + ).lean() + ) + .then((perList) => perList.map((item) => item.permission)), + getOrgIdSetWithParentByTmbId({ tmbId, teamId }) + .then((item) => Array.from(item)) + .then((orgIds) => + MongoResourcePermission.find( + { + teamId, + resourceType, + orgId: { + $in: Array.from(orgIds) + }, + resourceId + }, + 'permission' + ).lean() + ) + .then((perList) => perList.map((item) => item.permission)) + ]); - if (groupIdList.length === 0) { - return undefined; - } - - // get the maximum permission of the group - const pers = ( - await MongoResourcePermission.find( - { - teamId, - resourceType, - groupId: { - $in: groupIdList - }, - resourceId - }, - 'permission' - ).lean() - ).map((item) => item.permission); - - const groupPer = getGroupPer(pers); - - return groupPer; + return concatPer([...groupPers, ...orgPers]); }; -/* 仅取 members 不取 groups */ -export async function getResourceAllClbs({ - resourceId, - teamId, - resourceType, - session -}: { - teamId: string; - session?: ClientSession; -} & ( - | { - resourceType: 'team'; - resourceId?: undefined; - } - | { - resourceType: Omit; - resourceId?: string | null; - } -)): Promise { - return MongoResourcePermission.find( - { - resourceType: resourceType, - teamId: teamId, - resourceId, - groupId: { - $exists: false - } - }, - null, - { - session - } - ).lean(); -} - export async function getResourceClbsAndGroups({ resourceId, resourceType, @@ -155,37 +131,55 @@ export const getClbsAndGroupsWithInfo = async ({ resourceType, teamId }: { - resourceId: ParentIdType; - resourceType: Omit<`${PerResourceTypeEnum}`, 'team'>; teamId: string; -}) => +} & ( + | { + resourceId: ParentIdType; + resourceType: Omit<`${PerResourceTypeEnum}`, 'team'>; + } + | { + resourceType: 'team'; + resourceId?: undefined; + } +)) => Promise.all([ - (await MongoResourcePermission.find({ + MongoResourcePermission.find({ teamId, resourceId, resourceType, tmbId: { $exists: true } - }).populate({ - path: 'tmbId', - select: 'name userId', - populate: { - path: 'userId', - select: 'avatar' - } - })) as ResourcePerWithTmbWithUser[], - (await MongoResourcePermission.find({ + }) + .populate<{ tmb: TeamMemberSchema & { user: UserModelSchema } }>({ + path: 'tmb', + select: 'name userId role', + populate: { + path: 'user', + select: 'avatar' + } + }) + .lean(), + MongoResourcePermission.find({ teamId, resourceId, resourceType, groupId: { $exists: true } - }).populate({ - path: 'groupId', - select: 'name avatar' - })) as ResourcePerWithGroup[] + }) + .populate<{ group: MemberGroupSchemaType }>('group', 'name avatar') + .lean(), + MongoResourcePermission.find({ + teamId, + resourceId, + resourceType, + orgId: { + $exists: true + } + }) + .populate<{ org: OrgSchemaType }>({ path: 'org', select: 'name avatar' }) + .lean() ]); export const delResourcePermissionById = (id: string) => { @@ -195,6 +189,7 @@ export const delResourcePermission = ({ session, tmbId, groupId, + orgId, ...props }: { resourceType: PerResourceTypeEnum; @@ -203,15 +198,18 @@ export const delResourcePermission = ({ session?: ClientSession; tmbId?: string; groupId?: string; + orgId?: string; }) => { - // tmbId or groupId only one and not both - if (!!tmbId === !!groupId) { + // either tmbId or groupId or orgId must be provided + if (!tmbId && !groupId && !orgId) { return Promise.reject(CommonErrEnum.missingParams); } + return MongoResourcePermission.deleteOne( { ...(tmbId ? { tmbId } : {}), ...(groupId ? { groupId } : {}), + ...(orgId ? { orgId } : {}), ...props }, { session } @@ -249,7 +247,7 @@ export function authJWT(token: string) { }>((resolve, reject) => { const key = process.env.TOKEN_KEY as string; - jwt.verify(token, key, function (err, decoded: any) { + jwt.verify(token, key, (err, decoded: any) => { if (err || !decoded?.userId) { reject(ERROR_ENUM.unAuthorization); return; @@ -435,7 +433,7 @@ export const authFileToken = (token?: string) => } const key = (process.env.FILE_TOKEN_KEY as string) ?? 'filetoken'; - jwt.verify(token, key, function (err, decoded: any) { + jwt.verify(token, key, (err, decoded: any) => { if (err || !decoded.bucketName || !decoded?.teamId || !decoded?.fileId) { reject(ERROR_ENUM.unAuthFile); return; @@ -449,10 +447,10 @@ export const authFileToken = (token?: string) => }); }); -export const getGroupPer = (groups: PermissionValueType[] = []) => { - if (groups.length === 0) { +export const concatPer = (perList: PermissionValueType[] = []) => { + if (perList.length === 0) { return undefined; } - return new Permission().addPer(...groups).value; + return new Permission().addPer(...perList).value; }; diff --git a/packages/service/support/permission/dataset/auth.ts b/packages/service/support/permission/dataset/auth.ts index da58c7c9b..08829fcf6 100644 --- a/packages/service/support/permission/dataset/auth.ts +++ b/packages/service/support/permission/dataset/auth.ts @@ -3,7 +3,6 @@ import { getResourcePermission, parseHeaderCert } from '../controller'; import { CollectionWithDatasetType, DatasetDataItemType, - DatasetFileSchema, DatasetSchemaType } from '@fastgpt/global/core/dataset/type'; import { getTmbInfoByTmbId } from '../../user/team/controller'; @@ -12,10 +11,6 @@ import { NullPermission, PerResourceTypeEnum } from '@fastgpt/global/support/per import { DatasetErrEnum } from '@fastgpt/global/common/error/code/dataset'; import { DatasetPermission } from '@fastgpt/global/support/permission/dataset/controller'; import { getCollectionWithDataset } from '../../../core/dataset/controller'; -import { MongoDatasetCollection } from '../../../core/dataset/collection/schema'; -import { getFileById } from '../../../common/file/gridfs/controller'; -import { BucketNameEnum } from '@fastgpt/global/common/file/constants'; -import { CommonErrEnum } from '@fastgpt/global/common/error/code/common'; import { MongoDatasetData } from '../../../core/dataset/data/schema'; import { AuthModeType, AuthResponseType } from '../type'; import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constants'; @@ -172,7 +167,7 @@ export async function authDatasetCollection({ collection: CollectionWithDatasetType; } > { - const { teamId, tmbId, isRoot: isRootFromHeader } = await parseHeaderCert(props); + const { teamId, tmbId, userId, isRoot: isRootFromHeader } = await parseHeaderCert(props); const collection = await getCollectionWithDataset(collectionId); if (!collection) { @@ -181,12 +176,13 @@ export async function authDatasetCollection({ const { dataset } = await authDatasetByTmbId({ tmbId, - datasetId: collection.datasetId._id, + datasetId: collection.datasetId, per, isRoot: isRootFromHeader }); return { + userId, teamId, tmbId, collection, diff --git a/packages/service/support/permission/inheritPermission.ts b/packages/service/support/permission/inheritPermission.ts index c3ac42eb8..4f9993f5a 100644 --- a/packages/service/support/permission/inheritPermission.ts +++ b/packages/service/support/permission/inheritPermission.ts @@ -1,11 +1,11 @@ import { mongoSessionRun } from '../../common/mongo/sessionRun'; import { MongoResourcePermission } from './schema'; -import { ClientSession, Model } from 'mongoose'; -import { PerResourceTypeEnum } from '@fastgpt/global/support/permission/constant'; -import { PermissionValueType } from '@fastgpt/global/support/permission/type'; +import type { ClientSession, Model } from 'mongoose'; +import type { PerResourceTypeEnum } from '@fastgpt/global/support/permission/constant'; +import type { PermissionValueType } from '@fastgpt/global/support/permission/type'; import { getResourceClbsAndGroups } from './controller'; -import { RequireOnlyOne } from '@fastgpt/global/common/type/utils'; -import { ParentIdType } from '@fastgpt/global/common/parentFolder/type'; +import type { RequireOnlyOne } from '@fastgpt/global/common/type/utils'; +import type { ParentIdType } from '@fastgpt/global/common/parentFolder/type'; export type SyncChildrenPermissionResourceType = { _id: string; @@ -18,6 +18,7 @@ export type UpdateCollaboratorItem = { } & RequireOnlyOne<{ tmbId: string; groupId: string; + orgId: string; }>; // sync the permission to all children folders. @@ -161,7 +162,7 @@ export async function resumeInheritPermission({ } } -/* +/* Delete all the collaborators and then insert the new collaborators. */ export async function syncCollaborators({ diff --git a/packages/service/support/permission/memberGroup/controllers.ts b/packages/service/support/permission/memberGroup/controllers.ts index c6027ffb9..c34d2b4b3 100644 --- a/packages/service/support/permission/memberGroup/controllers.ts +++ b/packages/service/support/permission/memberGroup/controllers.ts @@ -1,9 +1,6 @@ import { MemberGroupSchemaType } from '@fastgpt/global/support/permission/memberGroup/type'; import { MongoGroupMemberModel } from './groupMemberSchema'; -import { TeamMemberSchema } from '@fastgpt/global/support/user/team/type'; -import { PerResourceTypeEnum } from '@fastgpt/global/support/permission/constant'; -import { MongoResourcePermission } from '../schema'; -import { getGroupPer, parseHeaderCert } from '../controller'; +import { parseHeaderCert } from '../controller'; import { MongoMemberGroupModel } from './memberGroupSchema'; import { DefaultGroupName } from '@fastgpt/global/support/user/team/group/constant'; import { ClientSession } from 'mongoose'; @@ -50,104 +47,41 @@ export const getTeamDefaultGroup = async ({ export const getGroupsByTmbId = async ({ tmbId, teamId, - role + role, + session }: { tmbId: string; teamId: string; role?: `${GroupMemberRole}`[]; + session?: ClientSession; }) => ( await Promise.all([ ( - await MongoGroupMemberModel.find({ - tmbId, - groupId: { - $exists: true + await MongoGroupMemberModel.find( + { + tmbId, + groupId: { + $exists: true + }, + ...(role ? { role: { $in: role } } : {}) }, - ...(role ? { role: { $in: role } } : {}) - }) - .populate('groupId') + undefined, + { session } + ) + .populate<{ group: MemberGroupSchemaType }>('group') .lean() - ).map((item) => { - return { - ...(item.groupId as any as MemberGroupSchemaType) - }; - }), - - role ? [] : getTeamDefaultGroup({ teamId }) + ).map((item) => item.group), + role ? [] : getTeamDefaultGroup({ teamId, session }) ]) ).flat(); -export const getTmbByGroupId = async (groupId: string) => { - return ( - await MongoGroupMemberModel.find({ - groupId - }) - .populate('tmbId') - .lean() - ).map((item) => { - return { - ...(item.tmbId as any as MemberGroupSchemaType) - }; - }); -}; - export const getGroupMembersByGroupId = async (groupId: string) => { return await MongoGroupMemberModel.find({ groupId }).lean(); }; -export const getGroupMembersWithInfoByGroupId = async (groupId: string) => { - return ( - await MongoGroupMemberModel.find({ - groupId - }) - .populate('tmbId') - .lean() - ).map((item) => item.tmbId) as any as TeamMemberSchema[]; // HACK: type casting -}; - -/** - * Get tmb's group permission: the maximum permission of the group - * @param tmbId - * @param resourceId - * @param resourceType - * @returns the maximum permission of the group - */ -export const getGroupPermission = async ({ - tmbId, - resourceId, - teamId, - resourceType -}: { - tmbId: string; - teamId: string; -} & ( - | { - resourceId?: undefined; - resourceType: 'team'; - } - | { - resourceId: string; - resourceType: Omit; - } -)) => { - const groupIds = (await getGroupsByTmbId({ tmbId, teamId })).map((item) => item._id); - const groupPermissions = ( - await MongoResourcePermission.find({ - groupId: { - $in: groupIds - }, - resourceType, - resourceId, - teamId - }) - ).map((item) => item.permission); - - return getGroupPer(groupPermissions); -}; - // auth group member role export const authGroupMemberRole = async ({ groupId, @@ -169,8 +103,12 @@ export const authGroupMemberRole = async ({ tmbId }; } - const groupMember = await MongoGroupMemberModel.findOne({ groupId, tmbId }); - const tmb = await getTmbInfoByTmbId({ tmbId }); + const [groupMember, tmb] = await Promise.all([ + MongoGroupMemberModel.findOne({ groupId, tmbId }), + getTmbInfoByTmbId({ tmbId }) + ]); + + // Team admin or role check if (tmb.permission.hasManagePer || (groupMember && role.includes(groupMember.role))) { return { ...result, diff --git a/packages/service/support/permission/memberGroup/groupMemberSchema.ts b/packages/service/support/permission/memberGroup/groupMemberSchema.ts index bfd0a8af3..8f30d42dd 100644 --- a/packages/service/support/permission/memberGroup/groupMemberSchema.ts +++ b/packages/service/support/permission/memberGroup/groupMemberSchema.ts @@ -26,6 +26,13 @@ export const GroupMemberSchema = new Schema({ } }); +GroupMemberSchema.virtual('group', { + ref: MemberGroupCollectionName, + localField: 'groupId', + foreignField: '_id', + justOne: true +}); + try { GroupMemberSchema.index({ groupId: 1 diff --git a/packages/service/support/permission/org/controllers.ts b/packages/service/support/permission/org/controllers.ts new file mode 100644 index 000000000..ce0746055 --- /dev/null +++ b/packages/service/support/permission/org/controllers.ts @@ -0,0 +1,95 @@ +import { TeamErrEnum } from '@fastgpt/global/common/error/code/team'; +import type { OrgSchemaType } from '@fastgpt/global/support/user/team/org/type'; +import type { ClientSession } from 'mongoose'; +import { MongoOrgModel } from './orgSchema'; +import { MongoOrgMemberModel } from './orgMemberSchema'; +import { getOrgChildrenPath } from '@fastgpt/global/support/user/team/org/constant'; + +export const getOrgsByTmbId = async ({ teamId, tmbId }: { teamId: string; tmbId: string }) => + MongoOrgMemberModel.find({ teamId, tmbId }, 'orgId').lean(); + +export const getOrgIdSetWithParentByTmbId = async ({ + teamId, + tmbId +}: { + teamId: string; + tmbId: string; +}) => { + const orgMembers = await MongoOrgMemberModel.find({ teamId, tmbId }, 'orgId').lean(); + + const orgIds = Array.from(new Set(orgMembers.map((item) => String(item.orgId)))); + const orgs = await MongoOrgModel.find({ _id: { $in: orgIds } }, 'path').lean(); + + const pathIdList = new Set( + orgs + .map((org) => { + const pathIdList = org.path.split('/').filter(Boolean); + return pathIdList; + }) + .flat() + ); + const parentOrgs = await MongoOrgModel.find( + { + teamId, + pathId: { $in: Array.from(pathIdList) } + }, + '_id' + ).lean(); + const parentOrgIds = parentOrgs.map((item) => String(item._id)); + + return new Set([...orgIds, ...parentOrgIds]); +}; + +export const getChildrenByOrg = async ({ + org, + teamId, + session +}: { + org: OrgSchemaType; + teamId: string; + session?: ClientSession; +}) => { + return MongoOrgModel.find( + { teamId, path: { $regex: `^${getOrgChildrenPath(org)}` } }, + undefined, + { + session + } + ).lean(); +}; + +export const getOrgAndChildren = async ({ + orgId, + teamId, + session +}: { + orgId: string; + teamId: string; + session?: ClientSession; +}) => { + const org = await MongoOrgModel.findOne({ _id: orgId, teamId }, undefined, { session }).lean(); + if (!org) { + return Promise.reject(TeamErrEnum.orgNotExist); + } + const children = await getChildrenByOrg({ org, teamId, session }); + return { org, children }; +}; + +export async function createRootOrg({ + teamId, + session +}: { + teamId: string; + session?: ClientSession; +}) { + return MongoOrgModel.create( + [ + { + teamId, + name: 'ROOT', + path: '' + } + ], + { session } + ); +} diff --git a/packages/service/support/permission/org/orgMemberSchema.ts b/packages/service/support/permission/org/orgMemberSchema.ts new file mode 100644 index 000000000..3abc5c233 --- /dev/null +++ b/packages/service/support/permission/org/orgMemberSchema.ts @@ -0,0 +1,65 @@ +import { OrgCollectionName } from '@fastgpt/global/support/user/team/org/constant'; +import { connectionMongo, getMongoModel } from '../../../common/mongo'; +import { + TeamCollectionName, + TeamMemberCollectionName +} from '@fastgpt/global/support/user/team/constant'; +import { OrgMemberSchemaType } from '@fastgpt/global/support/user/team/org/type'; +const { Schema } = connectionMongo; + +export const OrgMemberCollectionName = 'team_org_members'; + +export const OrgMemberSchema = new Schema({ + teamId: { + type: Schema.Types.ObjectId, + ref: TeamCollectionName, + required: true + }, + orgId: { + type: Schema.Types.ObjectId, + ref: OrgCollectionName, + required: true + }, + tmbId: { + type: Schema.Types.ObjectId, + ref: TeamMemberCollectionName, + required: true + } + // role: { + // type: String, + // enum: Object.values(OrgMemberRole), + // required: true, + // default: OrgMemberRole.member + // } +}); + +OrgMemberSchema.virtual('org', { + ref: OrgCollectionName, + localField: 'orgId', + foreignField: '_id', + justOne: true +}); + +try { + OrgMemberSchema.index( + { + teamId: 1, + orgId: 1, + tmbId: 1 + }, + { + unique: true + } + ); + OrgMemberSchema.index({ + teamId: 1, + tmbId: 1 + }); +} catch (error) { + console.log(error); +} + +export const MongoOrgMemberModel = getMongoModel( + OrgMemberCollectionName, + OrgMemberSchema +); diff --git a/packages/service/support/permission/org/orgSchema.ts b/packages/service/support/permission/org/orgSchema.ts new file mode 100644 index 000000000..45030c4a6 --- /dev/null +++ b/packages/service/support/permission/org/orgSchema.ts @@ -0,0 +1,77 @@ +import { TeamCollectionName } from '@fastgpt/global/support/user/team/constant'; +import { OrgCollectionName } from '@fastgpt/global/support/user/team/org/constant'; +import type { OrgSchemaType } from '@fastgpt/global/support/user/team/org/type'; +import { connectionMongo, getMongoModel } from '../../../common/mongo'; +import { OrgMemberCollectionName } from './orgMemberSchema'; +import { getNanoid } from '@fastgpt/global/common/string/tools'; +const { Schema } = connectionMongo; + +export const OrgSchema = new Schema( + { + teamId: { + type: Schema.Types.ObjectId, + ref: TeamCollectionName, + required: true + }, + pathId: { + // path id, only used for path + type: String, + required: true, + default: () => getNanoid() + }, + path: { + type: String, + required: function (this: OrgSchemaType) { + return typeof this.path !== 'string'; + } // allow empty string, but not null + }, + name: { + type: String, + required: true + }, + avatar: String, + description: String, + updateTime: { + type: Date, + default: () => new Date() + } + }, + { + // Auto update updateTime + timestamps: { + updatedAt: 'updateTime' + } + } +); + +OrgSchema.virtual('members', { + ref: OrgMemberCollectionName, + localField: '_id', + foreignField: 'orgId' +}); +// OrgSchema.virtual('permission', { +// ref: ResourcePermissionCollectionName, +// localField: '_id', +// foreignField: 'orgId', +// justOne: true +// }); + +try { + OrgSchema.index({ + teamId: 1, + path: 1 + }); + OrgSchema.index( + { + teamId: 1, + pathId: 1 + }, + { + unique: true + } + ); +} catch (error) { + console.log(error); +} + +export const MongoOrgModel = getMongoModel(OrgCollectionName, OrgSchema); diff --git a/packages/service/support/permission/schema.ts b/packages/service/support/permission/schema.ts index c2cf90a5a..10b4a0cdb 100644 --- a/packages/service/support/permission/schema.ts +++ b/packages/service/support/permission/schema.ts @@ -6,6 +6,7 @@ import { connectionMongo, getMongoModel } from '../../common/mongo'; import type { ResourcePermissionType } from '@fastgpt/global/support/permission/type'; import { PerResourceTypeEnum } from '@fastgpt/global/support/permission/constant'; import { MemberGroupCollectionName } from './memberGroup/memberGroupSchema'; +import { OrgCollectionName } from '@fastgpt/global/support/user/team/org/constant'; const { Schema } = connectionMongo; export const ResourcePermissionCollectionName = 'resource_permissions'; @@ -23,6 +24,10 @@ export const ResourcePermissionSchema = new Schema({ type: Schema.Types.ObjectId, ref: MemberGroupCollectionName }, + orgId: { + type: Schema.Types.ObjectId, + ref: OrgCollectionName + }, resourceType: { type: String, enum: Object.values(PerResourceTypeEnum), @@ -39,6 +44,25 @@ export const ResourcePermissionSchema = new Schema({ } }); +ResourcePermissionSchema.virtual('tmb', { + ref: TeamMemberCollectionName, + localField: 'tmbId', + foreignField: '_id', + justOne: true +}); +ResourcePermissionSchema.virtual('group', { + ref: MemberGroupCollectionName, + localField: 'groupId', + foreignField: '_id', + justOne: true +}); +ResourcePermissionSchema.virtual('org', { + ref: OrgCollectionName, + localField: 'orgId', + foreignField: '_id', + justOne: true +}); + try { ResourcePermissionSchema.index( { @@ -57,6 +81,23 @@ try { } ); + ResourcePermissionSchema.index( + { + resourceType: 1, + teamId: 1, + resourceId: 1, + orgId: 1 + }, + { + unique: true, + partialFilterExpression: { + orgId: { + $exists: true + } + } + } + ); + ResourcePermissionSchema.index( { resourceType: 1, @@ -74,6 +115,7 @@ try { } ); + // Delete tmb permission ResourcePermissionSchema.index({ resourceType: 1, teamId: 1, diff --git a/packages/service/support/permission/teamLimit.ts b/packages/service/support/permission/teamLimit.ts index 814480ccf..1a383f85b 100644 --- a/packages/service/support/permission/teamLimit.ts +++ b/packages/service/support/permission/teamLimit.ts @@ -19,9 +19,7 @@ export const checkDatasetLimit = async ({ if (!standardConstants) return; if (usedDatasetSize + insertLen >= datasetMaxSize) { - return Promise.reject( - `您的知识库容量为: ${datasetMaxSize}组，已使用: ${usedDatasetSize}组，导入当前文件需要: ${insertLen}组，请增加知识库容量后导入。` - ); + return Promise.reject(TeamErrEnum.datasetSizeNotEnough); } if (usedPoints >= totalPoints) { diff --git a/packages/service/support/permission/type.d.ts b/packages/service/support/permission/type.d.ts index 22b79d68e..91d03c6da 100644 --- a/packages/service/support/permission/type.d.ts +++ b/packages/service/support/permission/type.d.ts @@ -24,6 +24,7 @@ type authModeType = { export type AuthModeType = RequireAtLeastOne; export type AuthResponseType = { + userId: string; teamId: string; tmbId: string; authType?: `${AuthUserTypeEnum}`; diff --git a/packages/service/support/tmpData/controller.ts b/packages/service/support/tmpData/controller.ts index 09558837d..da499195c 100644 --- a/packages/service/support/tmpData/controller.ts +++ b/packages/service/support/tmpData/controller.ts @@ -24,7 +24,7 @@ export async function getTmpData({ }).lean()) as TmpDataSchema> | null; } -export async function setTmpData({ +export function setTmpData({ type, metadata, data @@ -33,7 +33,7 @@ export async function setTmpData({ metadata: TmpDataMetadata; data: TmpDataType; }) { - return await MongoTmpData.updateOne( + return MongoTmpData.updateOne( { dataId: getDataId(type, metadata) }, @@ -43,7 +43,8 @@ export async function setTmpData({ expireAt: addMilliseconds(Date.now(), TmpDataExpireTime[type]) }, { - upsert: true + upsert: true, + new: true } ); } diff --git a/packages/service/support/user/controller.ts b/packages/service/support/user/controller.ts index 3594cfe5e..cf432b214 100644 --- a/packages/service/support/user/controller.ts +++ b/packages/service/support/user/controller.ts @@ -44,7 +44,6 @@ export async function getUserDetail({ avatar: user.avatar, timezone: user.timezone, promotionRate: user.promotionRate, - openaiAccount: user.openaiAccount, team: tmb, notificationAccount: tmb.notificationAccount, permission: tmb.permission diff --git a/packages/service/support/user/schema.ts b/packages/service/support/user/schema.ts index c3dcc6852..10094ee6f 100644 --- a/packages/service/support/user/schema.ts +++ b/packages/service/support/user/schema.ts @@ -3,22 +3,10 @@ const { Schema } = connectionMongo; import { hashStr } from '@fastgpt/global/common/string/tools'; import type { UserModelSchema } from '@fastgpt/global/support/user/type'; import { UserStatusEnum, userStatusMap } from '@fastgpt/global/support/user/constant'; +import { getRandomUserAvatar } from '@fastgpt/global/support/user/utils'; export const userCollectionName = 'users'; -const defaultAvatars = [ - '/imgs/avatar/RoyalBlueAvatar.svg', - '/imgs/avatar/PurpleAvatar.svg', - '/imgs/avatar/AdoraAvatar.svg', - '/imgs/avatar/OrangeAvatar.svg', - '/imgs/avatar/RedAvatar.svg', - '/imgs/avatar/GrayModernAvatar.svg', - '/imgs/avatar/TealAvatar.svg', - '/imgs/avatar/GreenAvatar.svg', - '/imgs/avatar/BrightBlueAvatar.svg', - '/imgs/avatar/BlueAvatar.svg' -]; - const UserSchema = new Schema({ status: { type: String, @@ -47,13 +35,9 @@ const UserSchema = new Schema({ }, avatar: { type: String, - default: defaultAvatars[Math.floor(Math.random() * defaultAvatars.length)] - }, - inviterId: { - // 谁邀请注册的 - type: Schema.Types.ObjectId, - ref: userCollectionName + default: () => getRandomUserAvatar() }, + promotionRate: { type: Number, default: 15 @@ -71,14 +55,18 @@ const UserSchema = new Schema({ lastLoginTmbId: { type: Schema.Types.ObjectId }, - fastgpt_sem: { - type: Object - } + + inviterId: { + // 谁邀请注册的 + type: Schema.Types.ObjectId, + ref: userCollectionName + }, + fastgpt_sem: Object, + sourceDomain: String }); try { - // login - UserSchema.index({ username: 1, password: 1 }); + // Admin charts UserSchema.index({ createTime: -1 }); } catch (error) { console.log(error); diff --git a/packages/service/support/user/team/controller.ts b/packages/service/support/user/team/controller.ts index bc72a9089..451e02f5b 100644 --- a/packages/service/support/user/team/controller.ts +++ b/packages/service/support/user/team/controller.ts @@ -1,4 +1,4 @@ -import { TeamTmbItemType, TeamMemberWithTeamSchema } from '@fastgpt/global/support/user/team/type'; +import { TeamSchema, TeamTmbItemType } from '@fastgpt/global/support/user/team/type'; import { ClientSession, Types } from '../../../common/mongo'; import { TeamMemberRoleEnum, @@ -15,37 +15,44 @@ import { TeamDefaultPermissionVal } from '@fastgpt/global/support/permission/use import { MongoMemberGroupModel } from '../../permission/memberGroup/memberGroupSchema'; import { mongoSessionRun } from '../../../common/mongo/sessionRun'; import { DefaultGroupName } from '@fastgpt/global/support/user/team/group/constant'; +import { getAIApi, openaiBaseUrl } from '../../../core/ai/config'; +import { createRootOrg } from '../../permission/org/controllers'; +import { refreshSourceAvatar } from '../../../common/file/image/controller'; async function getTeamMember(match: Record): Promise { - const tmb = (await MongoTeamMember.findOne(match).populate('teamId')) as TeamMemberWithTeamSchema; + const tmb = await MongoTeamMember.findOne(match).populate<{ team: TeamSchema }>('team').lean(); if (!tmb) { return Promise.reject('member not exist'); } const Per = await getResourcePermission({ resourceType: PerResourceTypeEnum.team, - teamId: tmb.teamId._id, + teamId: tmb.teamId, tmbId: tmb._id }); return { userId: String(tmb.userId), - teamId: String(tmb.teamId._id), - teamName: tmb.teamId.name, + teamId: String(tmb.teamId), + teamAvatar: tmb.team.avatar, + teamName: tmb.team.name, memberName: tmb.name, - avatar: tmb.teamId.avatar, - balance: tmb.teamId.balance, + avatar: tmb.team.avatar, + balance: tmb.team.balance, tmbId: String(tmb._id), - teamDomain: tmb.teamId?.teamDomain, + teamDomain: tmb.team?.teamDomain, role: tmb.role, status: tmb.status, defaultTeam: tmb.defaultTeam, - lafAccount: tmb.teamId.lafAccount, permission: new TeamPermission({ per: Per ?? TeamDefaultPermissionVal, isOwner: tmb.role === TeamMemberRoleEnum.owner }), - notificationAccount: tmb.teamId.notificationAccount + notificationAccount: tmb.team.notificationAccount, + + lafAccount: tmb.team.lafAccount, + openaiAccount: tmb.team.openaiAccount, + externalWorkflowVariables: tmb.team.externalWorkflowVariables }; } @@ -73,13 +80,11 @@ export async function createDefaultTeam({ userId, teamName = 'My Team', avatar = '/icon/logo.svg', - balance, session }: { userId: string; teamName?: string; avatar?: string; - balance?: number; session: ClientSession; }) { // auth default team @@ -96,7 +101,6 @@ export async function createDefaultTeam({ ownerId: userId, name: teamName, avatar, - balance, createTime: new Date() } ], @@ -128,15 +132,11 @@ export async function createDefaultTeam({ ], { session } ); - console.log('create default team and group', userId); + await createRootOrg({ teamId: tmb.teamId, session }); + console.log('create default team, group and root org', userId); return tmb; } else { console.log('default team exist', userId); - await MongoTeam.findByIdAndUpdate(tmb.teamId, { - $set: { - ...(balance !== undefined && { balance }) - } - }); } } @@ -145,21 +145,88 @@ export async function updateTeam({ name, avatar, teamDomain, - lafAccount + lafAccount, + openaiAccount, + externalWorkflowVariable }: UpdateTeamProps & { teamId: string }) { + // auth openai key + if (openaiAccount?.key) { + console.log('auth user openai key', openaiAccount?.key); + const baseUrl = openaiAccount?.baseUrl || openaiBaseUrl; + openaiAccount.baseUrl = baseUrl; + + const ai = getAIApi({ + userKey: openaiAccount + }); + + const response = await ai.chat.completions.create({ + model: 'gpt-4o-mini', + max_tokens: 1, + messages: [{ role: 'user', content: 'hi' }] + }); + if (response?.choices?.[0]?.message?.content === undefined) { + return Promise.reject('Key response is empty'); + } + } + return mongoSessionRun(async (session) => { - await MongoTeam.findByIdAndUpdate( + const unsetObj = (() => { + const obj: Record = {}; + if (lafAccount?.pat === '') { + obj.lafAccount = 1; + } + if (openaiAccount?.key === '') { + obj.openaiAccount = 1; + } + if (externalWorkflowVariable) { + if (externalWorkflowVariable.value === '') { + obj[`externalWorkflowVariables.${externalWorkflowVariable.key}`] = 1; + } + } + + if (Object.keys(obj).length === 0) { + return undefined; + } + return { + $unset: obj + }; + })(); + const setObj = (() => { + const obj: Record = {}; + if (lafAccount?.pat && lafAccount?.appid) { + obj.lafAccount = lafAccount; + } + if (openaiAccount?.key && openaiAccount?.baseUrl) { + obj.openaiAccount = openaiAccount; + } + if (externalWorkflowVariable) { + if (externalWorkflowVariable.value !== '') { + obj[`externalWorkflowVariables.${externalWorkflowVariable.key}`] = + externalWorkflowVariable.value; + } + } + if (Object.keys(obj).length === 0) { + return undefined; + } + return obj; + })(); + + // This is where we get the old team + const team = await MongoTeam.findByIdAndUpdate( teamId, { - name, - avatar, - teamDomain, - lafAccount + $set: { + ...(name ? { name } : {}), + ...(avatar ? { avatar } : {}), + ...(teamDomain ? { teamDomain } : {}), + ...setObj + }, + ...unsetObj }, { session } ); - // update default group + // Update member group avatar if (avatar) { await MongoMemberGroupModel.updateOne( { @@ -171,6 +238,8 @@ export async function updateTeam({ }, { session } ); + + await refreshSourceAvatar(avatar, team?.avatar, session); } }); } diff --git a/packages/service/support/user/team/teamMemberSchema.ts b/packages/service/support/user/team/teamMemberSchema.ts index d14fb1b5b..117b8e6fe 100644 --- a/packages/service/support/user/team/teamMemberSchema.ts +++ b/packages/service/support/user/team/teamMemberSchema.ts @@ -3,7 +3,6 @@ const { Schema } = connectionMongo; import { TeamMemberSchema as TeamMemberType } from '@fastgpt/global/support/user/team/type.d'; import { userCollectionName } from '../../user/schema'; import { - TeamMemberRoleMap, TeamMemberStatusMap, TeamMemberCollectionName, TeamCollectionName @@ -24,10 +23,6 @@ const TeamMemberSchema = new Schema({ type: String, default: 'Member' }, - role: { - type: String - // enum: Object.keys(TeamMemberRoleMap) // disable enum validation for old data - }, status: { type: String, enum: Object.keys(TeamMemberStatusMap) @@ -39,9 +34,28 @@ const TeamMemberSchema = new Schema({ defaultTeam: { type: Boolean, default: false + }, + + // Abandoned + role: { + type: String + // enum: Object.keys(TeamMemberRoleMap) // disable enum validation for old data } }); +TeamMemberSchema.virtual('team', { + ref: TeamCollectionName, + localField: 'teamId', + foreignField: '_id', + justOne: true +}); +TeamMemberSchema.virtual('user', { + ref: userCollectionName, + localField: 'userId', + foreignField: '_id', + justOne: true +}); + try { TeamMemberSchema.index({ teamId: 1 }, { background: true }); TeamMemberSchema.index({ userId: 1 }, { background: true }); diff --git a/packages/service/support/user/team/teamSchema.ts b/packages/service/support/user/team/teamSchema.ts index 361a076b1..86a291cdd 100644 --- a/packages/service/support/user/team/teamSchema.ts +++ b/packages/service/support/user/team/teamSchema.ts @@ -21,10 +21,7 @@ const TeamSchema = new Schema({ type: Date, default: () => Date.now() }, - balance: { - type: Number, - default: 0 - }, + balance: Number, teamDomain: { type: String }, @@ -47,6 +44,16 @@ const TeamSchema = new Schema({ type: String } }, + openaiAccount: { + type: { + key: String, + baseUrl: String + } + }, + externalWorkflowVariables: { + type: Object, + default: {} + }, notificationAccount: { type: String, required: false diff --git a/packages/service/support/wallet/usage/controller.ts b/packages/service/support/wallet/usage/controller.ts index cc5ab1bdf..e948fb29e 100644 --- a/packages/service/support/wallet/usage/controller.ts +++ b/packages/service/support/wallet/usage/controller.ts @@ -31,20 +31,23 @@ export const createTrainingUsage = async ({ { moduleName: 'support.wallet.moduleName.index', model: vectorModel, - tokens: 0, - amount: 0 + amount: 0, + inputTokens: 0, + outputTokens: 0 }, { moduleName: 'support.wallet.moduleName.qa', model: agentModel, - tokens: 0, - amount: 0 + amount: 0, + inputTokens: 0, + outputTokens: 0 }, { moduleName: 'core.dataset.training.Auto mode', model: agentModel, - tokens: 0, - amount: 0 + amount: 0, + inputTokens: 0, + outputTokens: 0 } ] } diff --git a/packages/service/support/wallet/usage/schema.ts b/packages/service/support/wallet/usage/schema.ts index 9c243b4d2..7416d6b45 100644 --- a/packages/service/support/wallet/usage/schema.ts +++ b/packages/service/support/wallet/usage/schema.ts @@ -1,7 +1,7 @@ import { connectionMongo, getMongoModel, type Model } from '../../../common/mongo'; -const { Schema, model, models } = connectionMongo; +const { Schema } = connectionMongo; import { UsageSchemaType } from '@fastgpt/global/support/wallet/usage/type'; -import { UsageSourceMap } from '@fastgpt/global/support/wallet/usage/constants'; +import { UsageSourceEnum } from '@fastgpt/global/support/wallet/usage/constants'; import { TeamCollectionName, TeamMemberCollectionName @@ -22,7 +22,7 @@ const UsageSchema = new Schema({ }, source: { type: String, - enum: Object.keys(UsageSourceMap), + enum: Object.values(UsageSourceEnum), required: true }, appName: { @@ -61,11 +61,11 @@ const UsageSchema = new Schema({ }); try { - UsageSchema.index({ teamId: 1, tmbId: 1, source: 1, time: -1 }, { background: true }); + UsageSchema.index({ teamId: 1, tmbId: 1, source: 1, time: -1 }); // timer task. clear dead team - // UsageSchema.index({ teamId: 1, time: -1 }, { background: true }); + // UsageSchema.index({ teamId: 1, time: -1 }); - UsageSchema.index({ time: 1 }, { expireAfterSeconds: 180 * 24 * 60 * 60 }); + UsageSchema.index({ time: 1 }, { expireAfterSeconds: 360 * 24 * 60 * 60 }); } catch (error) { console.log(error); } diff --git a/packages/service/support/wallet/usage/utils.ts b/packages/service/support/wallet/usage/utils.ts index 003c98d7a..d4c21347b 100644 --- a/packages/service/support/wallet/usage/utils.ts +++ b/packages/service/support/wallet/usage/utils.ts @@ -1,17 +1,20 @@ +import { LLMModelItemType } from '@fastgpt/global/core/ai/model.d'; import { ModelTypeEnum, getModelMap } from '../../../core/ai/model'; export const formatModelChars2Points = ({ model, - tokens = 0, + inputTokens = 0, + outputTokens = 0, modelType, multiple = 1000 }: { model: string; - tokens: number; + inputTokens?: number; + outputTokens?: number; modelType: `${ModelTypeEnum}`; multiple?: number; }) => { - const modelData = getModelMap?.[modelType]?.(model); + const modelData = getModelMap?.[modelType]?.(model) as LLMModelItemType; if (!modelData) { return { totalPoints: 0, @@ -19,7 +22,12 @@ export const formatModelChars2Points = ({ }; } - const totalPoints = (modelData.charsPointsPrice || 0) * (tokens / multiple); + const isIOPriceType = typeof modelData.inputPrice === 'number'; + + const totalPoints = isIOPriceType + ? (modelData.inputPrice || 0) * (inputTokens / multiple) + + (modelData.outputPrice || 0) * (outputTokens / multiple) + : (modelData.charsPointsPrice || 0) * ((inputTokens + outputTokens) / multiple); return { modelName: modelData.name, diff --git a/packages/service/type.d.ts b/packages/service/type.d.ts index ad8e11399..11c7a6c69 100644 --- a/packages/service/type.d.ts +++ b/packages/service/type.d.ts @@ -2,27 +2,27 @@ import { FastGPTFeConfigsType, SystemEnvType } from '@fastgpt/global/common/syst import { AudioSpeechModelType, ReRankModelItemType, - WhisperModelType, + STTModelType, VectorModelItemType, LLMModelItemType } from '@fastgpt/global/core/ai/model.d'; import { SubPlanType } from '@fastgpt/global/support/wallet/sub/type'; import { WorkerNameEnum, WorkerPool } from './worker/utils'; import { Worker } from 'worker_threads'; -import { TemplateMarketItemType } from '@fastgpt/global/core/workflow/type'; declare global { + var systemInitBufferId: string | undefined; var systemVersion: string; var feConfigs: FastGPTFeConfigsType; var systemEnv: SystemEnvType; var subPlans: SubPlanType | undefined; var llmModels: LLMModelItemType[]; + var llmModelPriceType: 'IO' | 'Tokens'; var vectorModels: VectorModelItemType[]; var audioSpeechModels: AudioSpeechModelType[]; - var whisperModel: WhisperModelType; + var whisperModel: STTModelType; var reRankModels: ReRankModelItemType[]; var workerPoll: Record; - var appMarketTemplates: TemplateMarketItemType[]; } diff --git a/packages/service/worker/htmlStr2Md/utils.ts b/packages/service/worker/htmlStr2Md/utils.ts index 38112b92d..8384d005a 100644 --- a/packages/service/worker/htmlStr2Md/utils.ts +++ b/packages/service/worker/htmlStr2Md/utils.ts @@ -1,9 +1,27 @@ import TurndownService from 'turndown'; import { ImageType } from '../readFile/type'; import { matchMdImgTextAndUpload } from '@fastgpt/global/common/string/markdown'; +import { getNanoid } from '@fastgpt/global/common/string/tools'; // @ts-ignore const turndownPluginGfm = require('joplin-turndown-plugin-gfm'); +const processBase64Images = (htmlContent: string) => { + const base64Regex = /src="data:([^;]+);base64,([^"]+)"/g; + const images: ImageType[] = []; + + const processedHtml = htmlContent.replace(base64Regex, (match, mime, base64Data) => { + const uuid = `IMAGE_${getNanoid(12)}_IMAGE`; + images.push({ + uuid, + base64: base64Data, + mime + }); + return `src="${uuid}"`; + }); + + return { processedHtml, images }; +}; + export const html2md = ( html: string ): { @@ -25,11 +43,14 @@ export const html2md = ( turndownService.remove(['i', 'script', 'iframe', 'style']); turndownService.use(turndownPluginGfm.gfm); - const { text, imageList } = matchMdImgTextAndUpload(html); + // Base64 img to id, otherwise it will occupy memory when going to md + const { processedHtml, images } = processBase64Images(html); + const md = turndownService.turndown(processedHtml); + const { text, imageList } = matchMdImgTextAndUpload(md); return { - rawText: turndownService.turndown(text), - imageList + rawText: text, + imageList: [...images, ...imageList] }; } catch (error) { console.log('html 2 markdown error', error); diff --git a/packages/service/worker/readFile/extension/rawText.ts b/packages/service/worker/readFile/extension/rawText.ts index 15e0bed83..0f303f745 100644 --- a/packages/service/worker/readFile/extension/rawText.ts +++ b/packages/service/worker/readFile/extension/rawText.ts @@ -24,7 +24,11 @@ export const readFileRawText = ({ buffer, encoding }: ReadRawTextByBuffer): Read return buffer.toString(encoding as BufferEncoding); } - return iconv.decode(buffer, encoding); + if (encoding) { + return iconv.decode(buffer, encoding); + } + + return buffer.toString('utf-8'); } catch (error) { return buffer.toString('utf-8'); } diff --git a/packages/service/worker/readFile/parseOffice.ts b/packages/service/worker/readFile/parseOffice.ts index 05dabf9ec..177f5e2cd 100644 --- a/packages/service/worker/readFile/parseOffice.ts +++ b/packages/service/worker/readFile/parseOffice.ts @@ -44,13 +44,15 @@ const parsePowerPoint = async ({ } // Returning an array of all the xml contents read using fs.readFileSync - const xmlContentArray = files.map((file) => { - try { - return fs.readFileSync(`${decompressPath}/${file.path}`, encoding); - } catch (err) { - return fs.readFileSync(`${decompressPath}/${file.path}`, 'utf-8'); - } - }); + const xmlContentArray = await Promise.all( + files.map((file) => { + try { + return fs.promises.readFile(`${decompressPath}/${file.path}`, encoding); + } catch (err) { + return fs.promises.readFile(`${decompressPath}/${file.path}`, 'utf-8'); + } + }) + ); let responseArr: string[] = []; diff --git a/packages/templates/README.md b/packages/templates/README.md new file mode 100644 index 000000000..0ebd531dd --- /dev/null +++ b/packages/templates/README.md @@ -0,0 +1,3 @@ +# Package 说明 + +该 package 存放应用模板。 \ No newline at end of file diff --git a/packages/templates/package.json b/packages/templates/package.json new file mode 100644 index 000000000..447c38042 --- /dev/null +++ b/packages/templates/package.json @@ -0,0 +1,10 @@ +{ + "name": "@fastgpt/templates", + "version": "1.0.0", + "type": "module", + "dependencies": {}, + "devDependencies": { + "@fastgpt/global": "workspace:*", + "@fastgpt/service": "workspace:*" + } +} diff --git a/packages/templates/register.ts b/packages/templates/register.ts new file mode 100644 index 000000000..fae3334ea --- /dev/null +++ b/packages/templates/register.ts @@ -0,0 +1,83 @@ +import fs from 'fs'; +import path from 'path'; +import { isProduction } from '@fastgpt/global/common/system/constants'; +import { PluginSourceEnum } from '@fastgpt/global/core/plugin/constants'; +import { MongoAppTemplate } from '@fastgpt/service/core/app/templates/templateSchema'; +import { AppTemplateSchemaType } from '@fastgpt/global/core/app/type'; + +const getTemplateNameList = () => { + const currentFileUrl = new URL(import.meta.url); + const templatesPath = path.join(path.dirname(currentFileUrl.pathname), 'src'); + + return fs.readdirSync(templatesPath) as string[]; +}; + +const getFileTemplates = async (): Promise => { + const templateNames = getTemplateNameList(); + + return Promise.all( + templateNames.map>(async (name) => { + const fileContent = (await import(`./src/${name}/template.json`))?.default; + + return { + ...fileContent, + templateId: `${PluginSourceEnum.community}-${name}`, + isActive: true + }; + }) + ); +}; + +const getAppTemplates = async () => { + const communityTemplates = await getFileTemplates(); + + const dbTemplates = await MongoAppTemplate.find(); + + // Merge db data to community templates + const communityTemplateConfig = communityTemplates.map((template) => { + const config = dbTemplates.find((t) => t.templateId === template.templateId); + + if (config) { + return { + ...template, + isActive: config.isActive ?? template.isActive, + tags: config.tags ?? template.tags, + userGuide: config.userGuide ?? template.userGuide, + isQuickTemplate: config.isQuickTemplate ?? template.isQuickTemplate, + order: config.order ?? template.order + }; + } + + return template; + }); + + const res = [ + ...communityTemplateConfig, + ...dbTemplates.filter((t) => !isCommunityTemplate(t.templateId)) + ].sort((a, b) => (a.order ?? 0) - (b.order ?? 0)); + + return res; +}; + +export const getAppTemplatesAndLoadThem = async (refresh = false) => { + if (isProduction && global.appTemplates && global.appTemplates.length > 0 && !refresh) + return global.appTemplates; + + if (!global.appTemplates) { + global.appTemplates = []; + } + + try { + const appTemplates = await getAppTemplates(); + global.appTemplates = appTemplates; + return appTemplates; + } catch (error) { + // @ts-ignore + global.appTemplates = undefined; + return []; + } +}; + +export const isCommunityTemplate = (templateId: string) => { + return templateId.startsWith(PluginSourceEnum.community); +}; diff --git a/projects/app/public/appMarketTemplates/CQ/template.json b/packages/templates/src/CQ/template.json similarity index 64% rename from projects/app/public/appMarketTemplates/CQ/template.json rename to packages/templates/src/CQ/template.json index 08f9b94bf..6c04a80d5 100644 --- a/projects/app/public/appMarketTemplates/CQ/template.json +++ b/packages/templates/src/CQ/template.json @@ -18,56 +18,7 @@ "y": -486.7611729549753 }, "version": "481", - "inputs": [ - { - "key": "welcomeText", - "renderTypeList": ["FlowNodeInputTypeEnum.hidden"], - "valueType": "WorkflowIOValueTypeEnum.string", - "label": "core.app.Welcome Text", - "value": "你好，我是知识库助手，请不要忘记选择知识库噢~\n[你是谁]\n[如何使用]" - }, - { - "key": "variables", - "renderTypeList": ["FlowNodeInputTypeEnum.hidden"], - "valueType": "WorkflowIOValueTypeEnum.any", - "label": "core.app.Chat Variable", - "value": [] - }, - { - "key": "questionGuide", - "valueType": "WorkflowIOValueTypeEnum.boolean", - "renderTypeList": ["FlowNodeInputTypeEnum.hidden"], - "label": "core.app.Question Guide", - "value": true - }, - { - "key": "tts", - "renderTypeList": ["FlowNodeInputTypeEnum.hidden"], - "valueType": "WorkflowIOValueTypeEnum.any", - "label": "", - "value": { - "type": "web" - } - }, - { - "key": "whisper", - "renderTypeList": ["FlowNodeInputTypeEnum.hidden"], - "valueType": "WorkflowIOValueTypeEnum.any", - "label": "", - "value": { - "open": false, - "autoSend": false, - "autoTTSResponse": false - } - }, - { - "key": "scheduleTrigger", - "renderTypeList": ["FlowNodeInputTypeEnum.hidden"], - "valueType": "WorkflowIOValueTypeEnum.any", - "label": "", - "value": null - } - ], + "inputs": [], "outputs": [] }, { @@ -84,8 +35,8 @@ "inputs": [ { "key": "userChatInput", - "renderTypeList": ["FlowNodeInputTypeEnum.reference", "FlowNodeInputTypeEnum.textarea"], - "valueType": "WorkflowIOValueTypeEnum.string", + "renderTypeList": ["reference", "textarea"], + "valueType": "string", "label": "用户问题", "required": true, "toolDescription": "用户问题" @@ -96,7 +47,7 @@ "id": "userChatInput", "key": "userChatInput", "label": "core.module.input.label.user question", - "valueType": "WorkflowIOValueTypeEnum.string", + "valueType": "string", "type": "FlowNodeOutputTypeEnum.static" } ] @@ -116,58 +67,55 @@ "inputs": [ { "key": "model", - "renderTypeList": [ - "FlowNodeInputTypeEnum.settingLLMModel", - "FlowNodeInputTypeEnum.reference" - ], + "renderTypeList": ["settingLLMModel", "reference"], "label": "core.module.input.label.aiModel", - "valueType": "WorkflowIOValueTypeEnum.string", + "valueType": "string", "value": "gpt-4o-mini" }, { "key": "temperature", - "renderTypeList": ["FlowNodeInputTypeEnum.hidden"], + "renderTypeList": ["hidden"], "label": "", "value": 3, - "valueType": "WorkflowIOValueTypeEnum.number", + "valueType": "number", "min": 0, "max": 10, "step": 1 }, { "key": "maxToken", - "renderTypeList": ["FlowNodeInputTypeEnum.hidden"], + "renderTypeList": ["hidden"], "label": "", "value": 1950, - "valueType": "WorkflowIOValueTypeEnum.number", + "valueType": "number", "min": 100, "max": 4000, "step": 50 }, { "key": "isResponseAnswerText", - "renderTypeList": ["FlowNodeInputTypeEnum.hidden"], + "renderTypeList": ["hidden"], "label": "", "value": true, - "valueType": "WorkflowIOValueTypeEnum.boolean" + "valueType": "boolean" }, { "key": "quoteTemplate", - "renderTypeList": ["FlowNodeInputTypeEnum.hidden"], + "renderTypeList": ["hidden"], "label": "", - "valueType": "WorkflowIOValueTypeEnum.string" + "valueType": "string" }, { "key": "quotePrompt", - "renderTypeList": ["FlowNodeInputTypeEnum.hidden"], + "renderTypeList": ["hidden"], "label": "", - "valueType": "WorkflowIOValueTypeEnum.string" + "valueType": "string" }, { "key": "systemPrompt", - "renderTypeList": ["FlowNodeInputTypeEnum.textarea", "FlowNodeInputTypeEnum.reference"], + "renderTypeList": ["textarea", "reference"], "max": 3000, - "valueType": "WorkflowIOValueTypeEnum.string", + "valueType": "string", "label": "core.ai.Prompt", "description": "core.app.tip.systemPromptTip", "placeholder": "core.app.tip.chatNodeSystemPromptTip", @@ -175,11 +123,8 @@ }, { "key": "history", - "renderTypeList": [ - "FlowNodeInputTypeEnum.numberInput", - "FlowNodeInputTypeEnum.reference" - ], - "valueType": "WorkflowIOValueTypeEnum.chatHistory", + "renderTypeList": ["numberInput", "reference"], + "valueType": "chatHistory", "label": "core.module.input.label.chat history", "required": true, "min": 0, @@ -188,8 +133,8 @@ }, { "key": "userChatInput", - "renderTypeList": ["FlowNodeInputTypeEnum.reference", "FlowNodeInputTypeEnum.textarea"], - "valueType": "WorkflowIOValueTypeEnum.string", + "renderTypeList": ["reference", "textarea"], + "valueType": "string", "label": "用户问题", "required": true, "toolDescription": "用户问题", @@ -197,11 +142,11 @@ }, { "key": "quoteQA", - "renderTypeList": ["FlowNodeInputTypeEnum.settingDatasetQuotePrompt"], + "renderTypeList": ["settingDatasetQuotePrompt"], "label": "", "debugLabel": "知识库引用", "description": "", - "valueType": "WorkflowIOValueTypeEnum.datasetQuote", + "valueType": "datasetQuote", "value": ["MNMMMIjjWyMU", "quoteQA"] } ], @@ -211,7 +156,7 @@ "key": "history", "label": "core.module.output.label.New context", "description": "core.module.output.description.New context", - "valueType": "WorkflowIOValueTypeEnum.chatHistory", + "valueType": "chatHistory", "type": "FlowNodeOutputTypeEnum.static" }, { @@ -219,7 +164,7 @@ "key": "answerText", "label": "core.module.output.label.Ai response content", "description": "core.module.output.description.Ai response content", - "valueType": "WorkflowIOValueTypeEnum.string", + "valueType": "string", "type": "FlowNodeOutputTypeEnum.static" } ] @@ -239,21 +184,18 @@ "inputs": [ { "key": "model", - "renderTypeList": [ - "FlowNodeInputTypeEnum.selectLLMModel", - "FlowNodeInputTypeEnum.reference" - ], + "renderTypeList": ["selectLLMModel", "reference"], "label": "core.module.input.label.aiModel", "required": true, - "valueType": "WorkflowIOValueTypeEnum.string", + "valueType": "string", "llmModelType": "classify", "value": "gpt-4o-mini" }, { "key": "systemPrompt", - "renderTypeList": ["FlowNodeInputTypeEnum.textarea", "FlowNodeInputTypeEnum.reference"], + "renderTypeList": ["textarea", "reference"], "max": 3000, - "valueType": "WorkflowIOValueTypeEnum.string", + "valueType": "string", "label": "core.module.input.label.Background", "description": "core.module.input.description.Background", "placeholder": "core.module.input.placeholder.Classify background", @@ -261,11 +203,8 @@ }, { "key": "history", - "renderTypeList": [ - "FlowNodeInputTypeEnum.numberInput", - "FlowNodeInputTypeEnum.reference" - ], - "valueType": "WorkflowIOValueTypeEnum.chatHistory", + "renderTypeList": ["numberInput", "reference"], + "valueType": "chatHistory", "label": "core.module.input.label.chat history", "required": true, "min": 0, @@ -274,16 +213,16 @@ }, { "key": "userChatInput", - "renderTypeList": ["FlowNodeInputTypeEnum.reference", "FlowNodeInputTypeEnum.textarea"], - "valueType": "WorkflowIOValueTypeEnum.string", + "renderTypeList": ["reference", "textarea"], + "valueType": "string", "label": "用户问题", "required": true, "value": ["workflowStartNodeId", "userChatInput"] }, { "key": "agents", - "renderTypeList": ["FlowNodeInputTypeEnum.custom"], - "valueType": "WorkflowIOValueTypeEnum.any", + "renderTypeList": ["custom"], + "valueType": "any", "label": "", "value": [ { @@ -306,7 +245,7 @@ "id": "cqResult", "key": "cqResult", "label": "分类结果", - "valueType": "WorkflowIOValueTypeEnum.string", + "valueType": "string", "type": "FlowNodeOutputTypeEnum.static" } ] @@ -325,8 +264,8 @@ "inputs": [ { "key": "text", - "renderTypeList": ["FlowNodeInputTypeEnum.textarea", "FlowNodeInputTypeEnum.reference"], - "valueType": "WorkflowIOValueTypeEnum.string", + "renderTypeList": ["textarea", "reference"], + "valueType": "string", "label": "core.module.input.label.Response content", "description": "core.module.input.description.Response content", "placeholder": "core.module.input.description.Response content", @@ -351,68 +290,65 @@ "inputs": [ { "key": "datasets", - "renderTypeList": [ - "FlowNodeInputTypeEnum.selectDataset", - "FlowNodeInputTypeEnum.reference" - ], + "renderTypeList": ["selectDataset", "reference"], "label": "core.module.input.label.Select dataset", "value": [], - "valueType": "WorkflowIOValueTypeEnum.selectDataset", + "valueType": "selectDataset", "list": [], "required": true }, { "key": "similarity", - "renderTypeList": ["FlowNodeInputTypeEnum.selectDatasetParamsModal"], + "renderTypeList": ["selectDatasetParamsModal"], "label": "", "value": 0.4, - "valueType": "WorkflowIOValueTypeEnum.number" + "valueType": "number" }, { "key": "limit", - "renderTypeList": ["FlowNodeInputTypeEnum.hidden"], + "renderTypeList": ["hidden"], "label": "", "value": 1500, - "valueType": "WorkflowIOValueTypeEnum.number" + "valueType": "number" }, { "key": "searchMode", - "renderTypeList": ["FlowNodeInputTypeEnum.hidden"], + "renderTypeList": ["hidden"], "label": "", - "valueType": "WorkflowIOValueTypeEnum.string", + "valueType": "string", "value": "embedding" }, { "key": "usingReRank", - "renderTypeList": ["FlowNodeInputTypeEnum.hidden"], + "renderTypeList": ["hidden"], "label": "", - "valueType": "WorkflowIOValueTypeEnum.boolean", + "valueType": "boolean", "value": false }, { "key": "datasetSearchUsingExtensionQuery", - "renderTypeList": ["FlowNodeInputTypeEnum.hidden"], + "renderTypeList": ["hidden"], "label": "", - "valueType": "WorkflowIOValueTypeEnum.boolean", + "valueType": "boolean", "value": true }, { "key": "datasetSearchExtensionModel", - "renderTypeList": ["FlowNodeInputTypeEnum.hidden"], + "renderTypeList": ["hidden"], "label": "", - "valueType": "WorkflowIOValueTypeEnum.string" + "valueType": "string" }, { "key": "datasetSearchExtensionBg", - "renderTypeList": ["FlowNodeInputTypeEnum.hidden"], + "renderTypeList": ["hidden"], "label": "", - "valueType": "WorkflowIOValueTypeEnum.string", + "valueType": "string", "value": "" }, { "key": "userChatInput", - "renderTypeList": ["reference", "FlowNodeInputTypeEnum.textarea"], - "valueType": "WorkflowIOValueTypeEnum.string", + "renderTypeList": ["reference", "textarea"], + "valueType": "string", "label": "用户问题", "required": true, "toolDescription": "需要检索的内容", @@ -426,7 +362,7 @@ "label": "core.module.Dataset quote.label", "description": "特殊数组格式，搜索结果为空时，返回空数组。", "type": "FlowNodeOutputTypeEnum.static", - "valueType": "WorkflowIOValueTypeEnum.datasetQuote" + "valueType": "datasetQuote" } ] } diff --git a/projects/app/public/appMarketTemplates/Chinese/template.json b/packages/templates/src/Chinese/template.json similarity index 95% rename from projects/app/public/appMarketTemplates/Chinese/template.json rename to packages/templates/src/Chinese/template.json index bc6d87288..7930229bb 100644 --- a/projects/app/public/appMarketTemplates/Chinese/template.json +++ b/packages/templates/src/Chinese/template.json @@ -2,9 +2,13 @@ "name": "汉语新解", "intro": "生成汉语释义图", "author": "", - "avatar": "/appMarketTemplates/Chinese/avatar.svg", + "avatar": "core/app/templates/chinese", "tags": ["roleplay"], "type": "advanced", + "userGuide": { + "type": "link", + "content": "https://mp.weixin.qq.com/s/T90-uZqGovYR90fST0o80Q" + }, "workflow": { "nodes": [ { @@ -35,10 +39,12 @@ }, { "key": "questionGuide", - "valueType": "boolean", + "valueType": "any", "renderTypeList": ["hidden"], "label": "core.app.Question Guide", - "value": false + "value": { + "open": false + } }, { "key": "tts", diff --git a/projects/app/public/appMarketTemplates/TranslateRobot/template.json b/packages/templates/src/TranslateRobot/template.json similarity index 99% rename from projects/app/public/appMarketTemplates/TranslateRobot/template.json rename to packages/templates/src/TranslateRobot/template.json index d83756887..bd4eb1f3d 100644 --- a/projects/app/public/appMarketTemplates/TranslateRobot/template.json +++ b/packages/templates/src/TranslateRobot/template.json @@ -2,7 +2,7 @@ "name": "多轮翻译机器人", "intro": "通过 4 轮翻译，提高翻译英文的质量", "author": "", - "avatar": "/appMarketTemplates/TranslateRobot/avatar.svg", + "avatar": "core/app/templates/TranslateRobot", "tags": ["office-services"], "type": "advanced", "workflow": { @@ -35,10 +35,12 @@ }, { "key": "questionGuide", - "valueType": "boolean", + "valueType": "any", "renderTypeList": ["hidden"], "label": "core.app.Question Guide", - "value": false + "value": { + "open": false + } }, { "key": "tts", diff --git a/projects/app/public/appMarketTemplates/animalLife/template.json b/packages/templates/src/animalLife/template.json similarity index 97% rename from projects/app/public/appMarketTemplates/animalLife/template.json rename to packages/templates/src/animalLife/template.json index 896ae0245..4706c1dbf 100644 --- a/projects/app/public/appMarketTemplates/animalLife/template.json +++ b/packages/templates/src/animalLife/template.json @@ -2,7 +2,7 @@ "name": "动物的一生", "intro": "使用AI生成任何事物的 “人生图”", "author": "", - "avatar": "/appMarketTemplates/animalLife/avatar.svg", + "avatar": "core/app/templates/animalLife", "tags": ["roleplay"], "type": "advanced", "workflow": { @@ -35,10 +35,12 @@ }, { "key": "questionGuide", - "valueType": "boolean", + "valueType": "any", "renderTypeList": ["hidden"], "label": "core.app.Question Guide", - "value": false + "value": { + "open": false + } }, { "key": "tts", diff --git a/projects/app/public/appMarketTemplates/chatGuide/template.json b/packages/templates/src/chatGuide/template.json similarity index 65% rename from projects/app/public/appMarketTemplates/chatGuide/template.json rename to packages/templates/src/chatGuide/template.json index c682b8cc6..67dab9bcd 100644 --- a/projects/app/public/appMarketTemplates/chatGuide/template.json +++ b/packages/templates/src/chatGuide/template.json @@ -19,73 +19,7 @@ "y": -490.7611729549753 }, "version": "481", - "inputs": [ - { - "key": "welcomeText", - "renderTypeList": ["FlowNodeInputTypeEnum.hidden"], - "valueType": "WorkflowIOValueTypeEnum.string", - "label": "core.app.Welcome Text", - "value": "你好，我可以为你翻译各种语言，请告诉我你需要翻译成什么语言？" - }, - { - "key": "variables", - "renderTypeList": ["FlowNodeInputTypeEnum.hidden"], - "valueType": "WorkflowIOValueTypeEnum.any", - "label": "core.app.Chat Variable", - "value": [ - { - "id": "myb3xk", - "key": "language", - "label": "目标语言", - "type": "select", - "required": true, - "maxLen": 50, - "enums": [ - { - "value": "中文" - }, - { - "value": "英文" - } - ] - } - ] - }, - { - "key": "questionGuide", - "valueType": "WorkflowIOValueTypeEnum.boolean", - "renderTypeList": ["FlowNodeInputTypeEnum.hidden"], - "label": "core.app.Question Guide", - "value": false - }, - { - "key": "tts", - "renderTypeList": ["FlowNodeInputTypeEnum.hidden"], - "valueType": "WorkflowIOValueTypeEnum.any", - "label": "", - "value": { - "type": "web" - } - }, - { - "key": "whisper", - "renderTypeList": ["FlowNodeInputTypeEnum.hidden"], - "valueType": "WorkflowIOValueTypeEnum.any", - "label": "", - "value": { - "open": false, - "autoSend": false, - "autoTTSResponse": false - } - }, - { - "key": "scheduleTrigger", - "renderTypeList": ["FlowNodeInputTypeEnum.hidden"], - "valueType": "WorkflowIOValueTypeEnum.any", - "label": "", - "value": null - } - ], + "inputs": [], "outputs": [] }, { @@ -103,7 +37,7 @@ { "key": "userChatInput", "renderTypeList": ["FlowNodeInputTypeEnum.reference", "FlowNodeInputTypeEnum.textarea"], - "valueType": "WorkflowIOValueTypeEnum.string", + "valueType": "string", "label": "用户问题", "required": true, "toolDescription": "用户问题" @@ -114,7 +48,7 @@ "id": "userChatInput", "key": "userChatInput", "label": "core.module.input.label.user question", - "valueType": "WorkflowIOValueTypeEnum.string", + "valueType": "string", "type": "FlowNodeOutputTypeEnum.static" } ] @@ -139,7 +73,7 @@ "FlowNodeInputTypeEnum.reference" ], "label": "core.module.input.label.aiModel", - "valueType": "WorkflowIOValueTypeEnum.string", + "valueType": "string", "value": "gpt-3.5-turbo" }, { @@ -147,7 +81,7 @@ "renderTypeList": ["FlowNodeInputTypeEnum.hidden"], "label": "", "value": 0, - "valueType": "WorkflowIOValueTypeEnum.number", + "valueType": "number", "min": 0, "max": 10, "step": 1 @@ -157,7 +91,7 @@ "renderTypeList": ["FlowNodeInputTypeEnum.hidden"], "label": "", "value": 2000, - "valueType": "WorkflowIOValueTypeEnum.number", + "valueType": "number", "min": 100, "max": 4000, "step": 50 @@ -167,25 +101,25 @@ "renderTypeList": ["FlowNodeInputTypeEnum.hidden"], "label": "", "value": true, - "valueType": "WorkflowIOValueTypeEnum.boolean" + "valueType": "boolean" }, { "key": "quoteTemplate", "renderTypeList": ["FlowNodeInputTypeEnum.hidden"], "label": "", - "valueType": "WorkflowIOValueTypeEnum.string" + "valueType": "string" }, { "key": "quotePrompt", "renderTypeList": ["FlowNodeInputTypeEnum.hidden"], "label": "", - "valueType": "WorkflowIOValueTypeEnum.string" + "valueType": "string" }, { "key": "systemPrompt", "renderTypeList": ["FlowNodeInputTypeEnum.textarea", "FlowNodeInputTypeEnum.reference"], "max": 3000, - "valueType": "WorkflowIOValueTypeEnum.string", + "valueType": "string", "label": "core.ai.Prompt", "description": "core.app.tip.systemPromptTip", "placeholder": "core.app.tip.chatNodeSystemPromptTip", @@ -197,7 +131,7 @@ "FlowNodeInputTypeEnum.numberInput", "FlowNodeInputTypeEnum.reference" ], - "valueType": "WorkflowIOValueTypeEnum.chatHistory", + "valueType": "chatHistory", "label": "core.module.input.label.chat history", "required": true, "min": 0, @@ -207,7 +141,7 @@ { "key": "userChatInput", "renderTypeList": ["FlowNodeInputTypeEnum.reference", "FlowNodeInputTypeEnum.textarea"], - "valueType": "WorkflowIOValueTypeEnum.string", + "valueType": "string", "label": "用户问题", "required": true, "toolDescription": "用户问题", @@ -219,7 +153,7 @@ "label": "", "debugLabel": "知识库引用", "description": "", - "valueType": "WorkflowIOValueTypeEnum.datasetQuote" + "valueType": "datasetQuote" } ], "outputs": [ @@ -228,7 +162,7 @@ "key": "history", "label": "core.module.output.label.New context", "description": "core.module.output.description.New context", - "valueType": "WorkflowIOValueTypeEnum.chatHistory", + "valueType": "chatHistory", "type": "FlowNodeOutputTypeEnum.static" }, { @@ -236,7 +170,7 @@ "key": "answerText", "label": "core.module.output.label.Ai response content", "description": "core.module.output.description.Ai response content", - "valueType": "WorkflowIOValueTypeEnum.string", + "valueType": "string", "type": "FlowNodeOutputTypeEnum.static" } ] diff --git a/projects/app/public/appMarketTemplates/divination/template.json b/packages/templates/src/divination/template.json similarity index 97% rename from projects/app/public/appMarketTemplates/divination/template.json rename to packages/templates/src/divination/template.json index 62f2a6340..fcfdf647f 100644 --- a/projects/app/public/appMarketTemplates/divination/template.json +++ b/packages/templates/src/divination/template.json @@ -2,7 +2,7 @@ "name": "周易占卜", "intro": "AI占卜,快来算算你的运势~", "author": "", - "avatar": "/appMarketTemplates/divination/avatar.svg", + "avatar": "core/app/templates/divination", "tags": ["roleplay"], "type": "advanced", "workflow": { @@ -35,10 +35,12 @@ }, { "key": "questionGuide", - "valueType": "boolean", + "valueType": "any", "renderTypeList": ["hidden"], "label": "core.app.Question Guide", - "value": false + "value": { + "open": false + } }, { "key": "tts", diff --git a/projects/app/public/appMarketTemplates/flux/template.json b/packages/templates/src/flux/template.json similarity index 99% rename from projects/app/public/appMarketTemplates/flux/template.json rename to packages/templates/src/flux/template.json index b23100b4f..d31e03e7c 100644 --- a/projects/app/public/appMarketTemplates/flux/template.json +++ b/packages/templates/src/flux/template.json @@ -2,7 +2,7 @@ "name": "Flux 绘图", "intro": "通过请求 Flux 接口绘图，需要有 api key", "author": "", - "avatar": "/appMarketTemplates/flux/avatar.svg", + "avatar": "core/app/templates/flux", "type": "plugin", "tags": ["image-generation"], "workflow": { diff --git a/projects/app/public/appMarketTemplates/githubIssue/template.json b/packages/templates/src/githubIssue/template.json similarity index 96% rename from projects/app/public/appMarketTemplates/githubIssue/template.json rename to packages/templates/src/githubIssue/template.json index 3471f37d1..e53aec743 100644 --- a/projects/app/public/appMarketTemplates/githubIssue/template.json +++ b/packages/templates/src/githubIssue/template.json @@ -2,9 +2,13 @@ "name": "GitHub Issue 总结机器人", "intro": "定时获取GitHub Issue信息,使用AI进行总结,并推送到飞书群中", "author": "", - "avatar": "/appMarketTemplates/githubIssue/avatar.svg", + "avatar": "core/app/templates/githubIssue", "tags": ["office-services"], "type": "advanced", + "userGuide": { + "type": "link", + "content": "https://mp.weixin.qq.com/s/CBrwSn1jQZO7ybsMSx5GnQ" + }, "workflow": { "nodes": [ { @@ -35,10 +39,12 @@ }, { "key": "questionGuide", - "valueType": "boolean", + "valueType": "hidden", "renderTypeList": ["hidden"], "label": "core.app.Question Guide", - "value": false + "value": { + "open": false + } }, { "key": "tts", diff --git a/projects/app/public/appMarketTemplates/google/template.json b/packages/templates/src/google/template.json similarity index 98% rename from projects/app/public/appMarketTemplates/google/template.json rename to packages/templates/src/google/template.json index e93035c48..6097dee77 100644 --- a/projects/app/public/appMarketTemplates/google/template.json +++ b/packages/templates/src/google/template.json @@ -2,7 +2,7 @@ "name": "谷歌搜索", "intro": "通过请求谷歌搜索，查询相关内容作为模型的参考。", "author": "", - "avatar": "/appMarketTemplates/google/avatar.svg", + "avatar": "core/app/templates/google", "tags": ["recommendation", "web-search"], "type": "advanced", "workflow": { @@ -35,10 +35,12 @@ }, { "key": "questionGuide", - "valueType": "boolean", + "valueType": "any", "renderTypeList": ["hidden"], "label": "core.app.Question Guide", - "value": false + "value": { + "open": false + } }, { "key": "tts", @@ -178,7 +180,7 @@ "key": "NodeOutputKeyEnum.answerText", "label": "core.module.output.label.Ai response content", "description": "core.module.output.description.Ai response content", - "valueType": "WorkflowIOValueTypeEnum.string", + "valueType": "string", "type": "FlowNodeOutputTypeEnum.static" } ] diff --git a/projects/app/public/appMarketTemplates/longTranslate/template.json b/packages/templates/src/longTranslate/template.json similarity index 97% rename from projects/app/public/appMarketTemplates/longTranslate/template.json rename to packages/templates/src/longTranslate/template.json index 9cb6c98ac..9f9bcc913 100644 --- a/projects/app/public/appMarketTemplates/longTranslate/template.json +++ b/packages/templates/src/longTranslate/template.json @@ -1,10 +1,14 @@ { "name": "长文翻译专家", - "intro": "使用专有名称知识库协助翻译,更适合长文本的翻译机器人", + "intro": "使用专有名词知识库协助翻译,更适合长文本的翻译机器人", "author": "", - "avatar": "/appMarketTemplates/TranslateRobot/avatar.svg", + "avatar": "core/app/templates/TranslateRobot", "tags": ["office-services"], "type": "advanced", + "userGuide": { + "type": "link", + "content": "https://mp.weixin.qq.com/s/2kXdaSLOImhH1DTaFU8qXA" + }, "workflow": { "nodes": [ { @@ -35,10 +39,12 @@ }, { "key": "questionGuide", - "valueType": "boolean", + "valueType": "any", "renderTypeList": ["hidden"], "label": "core.app.Question Guide", - "value": false + "value": { + "open": false + } }, { "key": "tts", diff --git a/projects/app/public/appMarketTemplates/plugin-dalle/template.json b/packages/templates/src/plugin-dalle/template.json similarity index 99% rename from projects/app/public/appMarketTemplates/plugin-dalle/template.json rename to packages/templates/src/plugin-dalle/template.json index 53bb5533d..99261f7db 100644 --- a/projects/app/public/appMarketTemplates/plugin-dalle/template.json +++ b/packages/templates/src/plugin-dalle/template.json @@ -2,7 +2,7 @@ "name": "Dalle3 绘图", "intro": "通过请求 Dalle3 接口绘图，需要有 api key", "author": "", - "avatar": "/appMarketTemplates/plugin-dalle/avatar.svg", + "avatar": "core/app/templates/plugin-dalle", "type": "plugin", "tags": ["recommendation", "image-generation"], "workflow": { diff --git a/projects/app/public/appMarketTemplates/plugin-feishu/template.json b/packages/templates/src/plugin-feishu/template.json similarity index 99% rename from projects/app/public/appMarketTemplates/plugin-feishu/template.json rename to packages/templates/src/plugin-feishu/template.json index 4cca190ef..09cc21b55 100644 --- a/projects/app/public/appMarketTemplates/plugin-feishu/template.json +++ b/packages/templates/src/plugin-feishu/template.json @@ -2,7 +2,7 @@ "name": "飞书 webhook 插件", "intro": "通过 webhook 给飞书机器人发送一条消息", "author": "", - "avatar": "/appMarketTemplates/plugin-feishu/avatar.svg", + "avatar": "core/app/templates/plugin-feishu", "type": "plugin", "tags": ["recommendation", "office-services"], "workflow": { diff --git a/projects/app/public/appMarketTemplates/simpleDatasetChat/template.json b/packages/templates/src/simpleDatasetChat/template.json similarity index 72% rename from projects/app/public/appMarketTemplates/simpleDatasetChat/template.json rename to packages/templates/src/simpleDatasetChat/template.json index 44aa3de53..4c88b3086 100644 --- a/projects/app/public/appMarketTemplates/simpleDatasetChat/template.json +++ b/packages/templates/src/simpleDatasetChat/template.json @@ -18,56 +18,7 @@ "y": -486.7611729549753 }, "version": "481", - "inputs": [ - { - "key": "welcomeText", - "renderTypeList": ["FlowNodeInputTypeEnum.hidden"], - "valueType": "WorkflowIOValueTypeEnum.string", - "label": "core.app.Welcome Text", - "value": "你好，我是知识库助手，请不要忘记选择知识库噢~\n[你是谁]\n[如何使用]" - }, - { - "key": "variables", - "renderTypeList": ["FlowNodeInputTypeEnum.hidden"], - "valueType": "WorkflowIOValueTypeEnum.any", - "label": "core.app.Chat Variable", - "value": [] - }, - { - "key": "questionGuide", - "valueType": "WorkflowIOValueTypeEnum.boolean", - "renderTypeList": ["FlowNodeInputTypeEnum.hidden"], - "label": "core.app.Question Guide", - "value": false - }, - { - "key": "tts", - "renderTypeList": ["FlowNodeInputTypeEnum.hidden"], - "valueType": "WorkflowIOValueTypeEnum.any", - "label": "", - "value": { - "type": "web" - } - }, - { - "key": "whisper", - "renderTypeList": ["FlowNodeInputTypeEnum.hidden"], - "valueType": "WorkflowIOValueTypeEnum.any", - "label": "", - "value": { - "open": false, - "autoSend": false, - "autoTTSResponse": false - } - }, - { - "key": "scheduleTrigger", - "renderTypeList": ["FlowNodeInputTypeEnum.hidden"], - "valueType": "WorkflowIOValueTypeEnum.any", - "label": "", - "value": null - } - ], + "inputs": [], "outputs": [] }, { @@ -85,7 +36,7 @@ { "key": "userChatInput", "renderTypeList": ["FlowNodeInputTypeEnum.reference", "FlowNodeInputTypeEnum.textarea"], - "valueType": "WorkflowIOValueTypeEnum.string", + "valueType": "string", "label": "用户问题", "required": true, "toolDescription": "用户问题" @@ -96,7 +47,7 @@ "id": "userChatInput", "key": "userChatInput", "label": "core.module.input.label.user question", - "valueType": "WorkflowIOValueTypeEnum.string", + "valueType": "string", "type": "FlowNodeOutputTypeEnum.static" } ] @@ -121,7 +72,7 @@ "FlowNodeInputTypeEnum.reference" ], "label": "core.module.input.label.aiModel", - "valueType": "WorkflowIOValueTypeEnum.string", + "valueType": "string", "value": "gpt-4o-mini" }, { @@ -129,7 +80,7 @@ "renderTypeList": ["FlowNodeInputTypeEnum.hidden"], "label": "", "value": 3, - "valueType": "WorkflowIOValueTypeEnum.number", + "valueType": "number", "min": 0, "max": 10, "step": 1 @@ -139,7 +90,7 @@ "renderTypeList": ["FlowNodeInputTypeEnum.hidden"], "label": "", "value": 1950, - "valueType": "WorkflowIOValueTypeEnum.number", + "valueType": "number", "min": 100, "max": 4000, "step": 50 @@ -149,25 +100,25 @@ "renderTypeList": ["FlowNodeInputTypeEnum.hidden"], "label": "", "value": true, - "valueType": "WorkflowIOValueTypeEnum.boolean" + "valueType": "boolean" }, { "key": "quoteTemplate", "renderTypeList": ["FlowNodeInputTypeEnum.hidden"], "label": "", - "valueType": "WorkflowIOValueTypeEnum.string" + "valueType": "string" }, { "key": "quotePrompt", "renderTypeList": ["FlowNodeInputTypeEnum.hidden"], "label": "", - "valueType": "WorkflowIOValueTypeEnum.string" + "valueType": "string" }, { "key": "systemPrompt", "renderTypeList": ["FlowNodeInputTypeEnum.textarea", "FlowNodeInputTypeEnum.reference"], "max": 3000, - "valueType": "WorkflowIOValueTypeEnum.string", + "valueType": "string", "label": "core.ai.Prompt", "description": "core.app.tip.systemPromptTip", "placeholder": "core.app.tip.chatNodeSystemPromptTip", @@ -179,7 +130,7 @@ "FlowNodeInputTypeEnum.numberInput", "FlowNodeInputTypeEnum.reference" ], - "valueType": "WorkflowIOValueTypeEnum.chatHistory", + "valueType": "chatHistory", "label": "core.module.input.label.chat history", "required": true, "min": 0, @@ -189,7 +140,7 @@ { "key": "userChatInput", "renderTypeList": ["FlowNodeInputTypeEnum.reference", "FlowNodeInputTypeEnum.textarea"], - "valueType": "WorkflowIOValueTypeEnum.string", + "valueType": "string", "label": "用户问题", "required": true, "toolDescription": "用户问题", @@ -201,7 +152,7 @@ "label": "", "debugLabel": "知识库引用", "description": "", - "valueType": "WorkflowIOValueTypeEnum.datasetQuote", + "valueType": "datasetQuote", "value": ["iKBoX2vIzETU", "quoteQA"] } ], @@ -211,7 +162,7 @@ "key": "history", "label": "core.module.output.label.New context", "description": "core.module.output.description.New context", - "valueType": "WorkflowIOValueTypeEnum.chatHistory", + "valueType": "chatHistory", "type": "FlowNodeOutputTypeEnum.static" }, { @@ -219,7 +170,7 @@ "key": "answerText", "label": "core.module.output.label.Ai response content", "description": "core.module.output.description.Ai response content", - "valueType": "WorkflowIOValueTypeEnum.string", + "valueType": "string", "type": "FlowNodeOutputTypeEnum.static" } ] @@ -245,7 +196,7 @@ ], "label": "core.module.input.label.Select dataset", "value": [], - "valueType": "WorkflowIOValueTypeEnum.selectDataset", + "valueType": "selectDataset", "list": [], "required": true }, @@ -254,53 +205,53 @@ "renderTypeList": ["FlowNodeInputTypeEnum.selectDatasetParamsModal"], "label": "", "value": 0.4, - "valueType": "WorkflowIOValueTypeEnum.number" + "valueType": "number" }, { "key": "limit", "renderTypeList": ["hidden"], "label": "", "value": 1500, - "valueType": "WorkflowIOValueTypeEnum.number" + "valueType": "number" }, { "key": "searchMode", "renderTypeList": ["FlowNodeInputTypeEnum.hidden"], "label": "", - "valueType": "WorkflowIOValueTypeEnum.string", + "valueType": "string", "value": "embedding" }, { "key": "usingReRank", "renderTypeList": ["FlowNodeInputTypeEnum.hidden"], "label": "", - "valueType": "WorkflowIOValueTypeEnum.boolean", + "valueType": "boolean", "value": false }, { "key": "datasetSearchUsingExtensionQuery", "renderTypeList": ["FlowNodeInputTypeEnum.hidden"], "label": "", - "valueType": "WorkflowIOValueTypeEnum.boolean", + "valueType": "boolean", "value": true }, { "key": "datasetSearchExtensionModel", "renderTypeList": ["FlowNodeInputTypeEnum.hidden"], "label": "", - "valueType": "WorkflowIOValueTypeEnum.string" + "valueType": "string" }, { "key": "datasetSearchExtensionBg", "renderTypeList": ["FlowNodeInputTypeEnum.hidden"], "label": "", - "valueType": "WorkflowIOValueTypeEnum.string", + "valueType": "string", "value": "" }, { "key": "userChatInput", "renderTypeList": ["FlowNodeInputTypeEnum.reference", "FlowNodeInputTypeEnum.textarea"], - "valueType": "WorkflowIOValueTypeEnum.string", + "valueType": "string", "label": "用户问题", "required": true, "toolDescription": "需要检索的内容", @@ -313,7 +264,7 @@ "key": "quoteQA", "label": "core.module.Dataset quote.label", "type": "FlowNodeOutputTypeEnum.static", - "valueType": "WorkflowIOValueTypeEnum.datasetQuote", + "valueType": "datasetQuote", "description": "特殊数组格式，搜索结果为空时，返回空数组。" } ] diff --git a/projects/app/public/appMarketTemplates/srt-translate/template.json b/packages/templates/src/srt-translate/template.json similarity index 99% rename from projects/app/public/appMarketTemplates/srt-translate/template.json rename to packages/templates/src/srt-translate/template.json index f38c76c45..43258360c 100644 --- a/projects/app/public/appMarketTemplates/srt-translate/template.json +++ b/packages/templates/src/srt-translate/template.json @@ -2,9 +2,13 @@ "name": "长字幕反思翻译机器人", "intro": "利用 AI 自我反思提升翻译质量，同时循环迭代执行 AI 工作流来突破 LLM tokens 限制，实现一个高效的长字幕翻译机器人", "author": "", - "avatar": "/appMarketTemplates/TranslateRobot/avatar.svg", + "avatar": "core/app/templates/TranslateRobot", "tags": ["office-services"], "type": "advanced", + "userGuide": { + "type": "link", + "content": "https://mp.weixin.qq.com/s/uztciEVvumNWNlct0IeJ-w" + }, "workflow": { "nodes": [ { @@ -35,10 +39,12 @@ }, { "key": "questionGuide", - "valueType": "boolean", + "valueType": "any", "renderTypeList": ["hidden"], "label": "core.app.Question Guide", - "value": false + "value": { + "open": false + } }, { "key": "tts", diff --git a/projects/app/public/appMarketTemplates/stock/template.json b/packages/templates/src/stock/template.json similarity index 96% rename from projects/app/public/appMarketTemplates/stock/template.json rename to packages/templates/src/stock/template.json index 7c3e99685..4a03d6794 100644 --- a/projects/app/public/appMarketTemplates/stock/template.json +++ b/packages/templates/src/stock/template.json @@ -2,7 +2,7 @@ "name": "利好大A", "intro": "", "author": "", - "avatar": "/appMarketTemplates/stock/avatar.svg", + "avatar": "core/app/templates/stock", "tags": ["roleplay"], "type": "advanced", "workflow": { @@ -35,10 +35,12 @@ }, { "key": "questionGuide", - "valueType": "boolean", + "valueType": "any", "renderTypeList": ["hidden"], "label": "core.app.Question Guide", - "value": false + "value": { + "open": false + } }, { "key": "tts", diff --git a/projects/app/public/appMarketTemplates/timeBot/template.json b/packages/templates/src/timeBot/template.json similarity index 100% rename from projects/app/public/appMarketTemplates/timeBot/template.json rename to packages/templates/src/timeBot/template.json diff --git a/packages/templates/tsconfig.json b/packages/templates/tsconfig.json new file mode 100644 index 000000000..13c4daa3b --- /dev/null +++ b/packages/templates/tsconfig.json @@ -0,0 +1,21 @@ +{ + "compilerOptions": { + "target": "es2015", + "lib": ["dom", "dom.iterable", "esnext"], + "allowJs": true, + "skipLibCheck": true, + "strict": true, + "forceConsistentCasingInFileNames": true, + "noEmit": true, + "esModuleInterop": true, + "module": "esnext", + "moduleResolution": "node", + "resolveJsonModule": true, + "isolatedModules": true, + "jsx": "preserve", + "incremental": true, + "baseUrl": "." + }, + "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", "**/*.d.ts", "../**/*.d.ts"], + "exclude": ["node_modules"] +} diff --git a/packages/templates/type.d.ts b/packages/templates/type.d.ts new file mode 100644 index 000000000..8fd24c7f0 --- /dev/null +++ b/packages/templates/type.d.ts @@ -0,0 +1,5 @@ +import { AppTemplateSchemaType } from '@fastgpt/global/core/app/type'; + +declare global { + var appTemplates: AppTemplateSchemaType[]; +} diff --git a/packages/web/common/system/nextjs.ts b/packages/web/common/system/nextjs.ts new file mode 100644 index 000000000..54a4bb9c4 --- /dev/null +++ b/packages/web/common/system/nextjs.ts @@ -0,0 +1,15 @@ +import { I18nNsType } from '../../types/i18next'; +import { serverSideTranslations } from 'next-i18next/serverSideTranslations'; + +export const serviceSideProps = async (content: any, ns: I18nNsType = []) => { + const lang = content.req?.cookies?.NEXT_LOCALE || content.locale; + const extraLng = content.req?.cookies?.NEXT_LOCALE ? undefined : content.locales; + + // Device size + const deviceSize = content.req?.cookies?.NEXT_DEVICE_SIZE || null; + + return { + ...(await serverSideTranslations(lang, ['common', ...ns], null, extraLng)), + deviceSize + }; +}; diff --git a/packages/web/components/common/Avatar/index.tsx b/packages/web/components/common/Avatar/index.tsx index 9f398ac22..8b7abb30f 100644 --- a/packages/web/components/common/Avatar/index.tsx +++ b/packages/web/components/common/Avatar/index.tsx @@ -28,4 +28,4 @@ const Avatar = ({ w = '30px', src, ...props }: ImageProps) => { ); }; -export default Avatar; +export default React.memo(Avatar); diff --git a/packages/web/components/common/DndDrag/index.tsx b/packages/web/components/common/DndDrag/index.tsx index a072292ad..fc64f3a3e 100644 --- a/packages/web/components/common/DndDrag/index.tsx +++ b/packages/web/components/common/DndDrag/index.tsx @@ -57,4 +57,4 @@ function DndDrag({ children, renderClone, onDragEndCb, dataList }: Props) ); } -export default DndDrag; +export default React.memo(DndDrag) as (props: Props) => React.ReactElement; diff --git a/packages/web/components/common/Icon/button.tsx b/packages/web/components/common/Icon/button.tsx index 38cd50a39..9fde10ec4 100644 --- a/packages/web/components/common/Icon/button.tsx +++ b/packages/web/components/common/Icon/button.tsx @@ -18,7 +18,6 @@ const MyIconButton = ({ }: Props) => { return ( import('./icons/common/addUser.svg'), 'common/administrator': () => import('./icons/common/administrator.svg'), 'common/arrowLeft': () => import('./icons/common/arrowLeft.svg'), + 'common/arrowRight': () => import('./icons/common/arrowRight.svg'), 'common/backFill': () => import('./icons/common/backFill.svg'), 'common/backLight': () => import('./icons/common/backLight.svg'), 'common/billing': () => import('./icons/common/billing.svg'), @@ -29,6 +30,8 @@ export const iconPaths = { 'common/courseLight': () => import('./icons/common/courseLight.svg'), 'common/customTitleLight': () => import('./icons/common/customTitleLight.svg'), 'common/data': () => import('./icons/common/data.svg'), + 'common/dingtalkFill': () => import('./icons/common/dingtalkFill.svg'), + 'common/downArrowFill': () => import('./icons/common/downArrowFill.svg'), 'common/editor/resizer': () => import('./icons/common/editor/resizer.svg'), 'common/errorFill': () => import('./icons/common/errorFill.svg'), 'common/file/move': () => import('./icons/common/file/move.svg'), @@ -47,6 +50,7 @@ export const iconPaths = { 'common/language/China': () => import('./icons/common/language/China.svg'), 'common/language/en': () => import('./icons/common/language/en.svg'), 'common/language/zh': () => import('./icons/common/language/zh.svg'), + 'common/layer': () => import('./icons/common/layer.svg'), 'common/leftArrowLight': () => import('./icons/common/leftArrowLight.svg'), 'common/line': () => import('./icons/common/line.svg'), 'common/lineChange': () => import('./icons/common/lineChange.svg'), @@ -55,7 +59,7 @@ export const iconPaths = { 'common/loading': () => import('./icons/common/loading.svg'), 'common/logLight': () => import('./icons/common/logLight.svg'), 'common/microsoft': () => import('./icons/common/microsoft.svg'), - 'common/modal': () => import('./icons/common/modal.svg'), + 'common/model': () => import('./icons/common/model.svg'), 'common/monitor': () => import('./icons/common/monitor.svg'), 'common/more': () => import('./icons/common/more.svg'), 'common/moreFill': () => import('./icons/common/moreFill.svg'), @@ -80,17 +84,22 @@ export const iconPaths = { 'common/settingLight': () => import('./icons/common/settingLight.svg'), 'common/solidChevronRight': () => import('./icons/common/solidChevronRight.svg'), 'common/subtract': () => import('./icons/common/subtract.svg'), + 'common/templateMarket': () => import('./icons/common/templateMarket.svg'), 'common/text/t': () => import('./icons/common/text/t.svg'), + 'common/thirdParty': () => import('./icons/common/thirdParty.svg'), 'common/tickFill': () => import('./icons/common/tickFill.svg'), 'common/toolkit': () => import('./icons/common/toolkit.svg'), 'common/trash': () => import('./icons/common/trash.svg'), 'common/upRightArrowLight': () => import('./icons/common/upRightArrowLight.svg'), 'common/uploadFileFill': () => import('./icons/common/uploadFileFill.svg'), 'common/userInfo': () => import('./icons/common/userInfo.svg'), + 'common/variable': () => import('./icons/common/variable.svg'), 'common/viewLight': () => import('./icons/common/viewLight.svg'), 'common/voiceLight': () => import('./icons/common/voiceLight.svg'), + 'common/wallet': () => import('./icons/common/wallet.svg'), 'common/warn': () => import('./icons/common/warn.svg'), 'common/wechatFill': () => import('./icons/common/wechatFill.svg'), + 'common/wecom': () => import('./icons/common/wecom.svg'), configmap: () => import('./icons/configmap.svg'), copy: () => import('./icons/copy.svg'), 'core/app/aiFill': () => import('./icons/core/app/aiFill.svg'), @@ -118,10 +127,22 @@ export const iconPaths = { 'core/app/simpleMode/tts': () => import('./icons/core/app/simpleMode/tts.svg'), 'core/app/simpleMode/variable': () => import('./icons/core/app/simpleMode/variable.svg'), 'core/app/simpleMode/whisper': () => import('./icons/core/app/simpleMode/whisper.svg'), + 'core/app/templates/TranslateRobot': () => + import('./icons/core/app/templates/TranslateRobot.svg'), + 'core/app/templates/animalLife': () => import('./icons/core/app/templates/animalLife.svg'), + 'core/app/templates/chinese': () => import('./icons/core/app/templates/chinese.svg'), + 'core/app/templates/divination': () => import('./icons/core/app/templates/divination.svg'), + 'core/app/templates/flux': () => import('./icons/core/app/templates/flux.svg'), + 'core/app/templates/githubIssue': () => import('./icons/core/app/templates/githubIssue.svg'), + 'core/app/templates/google': () => import('./icons/core/app/templates/google.svg'), + 'core/app/templates/plugin-dalle': () => import('./icons/core/app/templates/plugin-dalle.svg'), + 'core/app/templates/plugin-feishu': () => import('./icons/core/app/templates/plugin-feishu.svg'), + 'core/app/templates/stock': () => import('./icons/core/app/templates/stock.svg'), 'core/app/toolCall': () => import('./icons/core/app/toolCall.svg'), 'core/app/ttsFill': () => import('./icons/core/app/ttsFill.svg'), 'core/app/type/httpPlugin': () => import('./icons/core/app/type/httpPlugin.svg'), 'core/app/type/httpPluginFill': () => import('./icons/core/app/type/httpPluginFill.svg'), + 'core/app/type/jsonImport': () => import('./icons/core/app/type/jsonImport.svg'), 'core/app/type/plugin': () => import('./icons/core/app/type/plugin.svg'), 'core/app/type/pluginFill': () => import('./icons/core/app/type/pluginFill.svg'), 'core/app/type/pluginLight': () => import('./icons/core/app/type/pluginLight.svg'), @@ -169,6 +190,9 @@ export const iconPaths = { import('./icons/core/dataset/externalDatasetColor.svg'), 'core/dataset/externalDatasetOutline': () => import('./icons/core/dataset/externalDatasetOutline.svg'), + 'core/dataset/feishuDatasetColor': () => import('./icons/core/dataset/feishuDatasetColor.svg'), + 'core/dataset/feishuDatasetOutline': () => + import('./icons/core/dataset/feishuDatasetOutline.svg'), 'core/dataset/fileCollection': () => import('./icons/core/dataset/fileCollection.svg'), 'core/dataset/fullTextRecall': () => import('./icons/core/dataset/fullTextRecall.svg'), 'core/dataset/manualCollection': () => import('./icons/core/dataset/manualCollection.svg'), @@ -182,6 +206,8 @@ export const iconPaths = { 'core/dataset/websiteDatasetColor': () => import('./icons/core/dataset/websiteDatasetColor.svg'), 'core/dataset/websiteDatasetOutline': () => import('./icons/core/dataset/websiteDatasetOutline.svg'), + 'core/dataset/yuqueDatasetColor': () => import('./icons/core/dataset/yuqueDatasetColor.svg'), + 'core/dataset/yuqueDatasetOutline': () => import('./icons/core/dataset/yuqueDatasetOutline.svg'), 'core/modules/basicNode': () => import('./icons/core/modules/basicNode.svg'), 'core/modules/fixview': () => import('./icons/core/modules/fixview.svg'), 'core/modules/flowLight': () => import('./icons/core/modules/flowLight.svg'), @@ -268,6 +294,7 @@ export const iconPaths = { 'core/workflow/template/readFiles': () => import('./icons/core/workflow/template/readFiles.svg'), 'core/workflow/template/reply': () => import('./icons/core/workflow/template/reply.svg'), 'core/workflow/template/runApp': () => import('./icons/core/workflow/template/runApp.svg'), + 'core/workflow/template/searxng': () => import('./icons/core/workflow/template/searxng.svg'), 'core/workflow/template/stopTool': () => import('./icons/core/workflow/template/stopTool.svg'), 'core/workflow/template/systemConfig': () => import('./icons/core/workflow/template/systemConfig.svg'), @@ -323,6 +350,8 @@ export const iconPaths = { history: () => import('./icons/history.svg'), infoRounded: () => import('./icons/infoRounded.svg'), kbTest: () => import('./icons/kbTest.svg'), + key: () => import('./icons/key.svg'), + keyPrimary: () => import('./icons/keyPrimary.svg'), menu: () => import('./icons/menu.svg'), minus: () => import('./icons/minus.svg'), 'modal/AddClb': () => import('./icons/modal/AddClb.svg'), @@ -334,6 +363,29 @@ export const iconPaths = { 'modal/selectSource': () => import('./icons/modal/selectSource.svg'), 'modal/setting': () => import('./icons/modal/setting.svg'), 'modal/teamPlans': () => import('./icons/modal/teamPlans.svg'), + 'model/BAAI': () => import('./icons/model/BAAI.svg'), + 'model/alicloud': () => import('./icons/model/alicloud.svg'), + 'model/baichuan': () => import('./icons/model/baichuan.svg'), + 'model/chatglm': () => import('./icons/model/chatglm.svg'), + 'model/claude': () => import('./icons/model/claude.svg'), + 'model/deepseek': () => import('./icons/model/deepseek.svg'), + 'model/doubao': () => import('./icons/model/doubao.svg'), + 'model/ernie': () => import('./icons/model/ernie.svg'), + 'model/fishaudio': () => import('./icons/model/fishaudio.svg'), + 'model/gemini': () => import('./icons/model/gemini.svg'), + 'model/groq': () => import('./icons/model/groq.svg'), + 'model/huggingface': () => import('./icons/model/huggingface.svg'), + 'model/hunyuan': () => import('./icons/model/hunyuan.svg'), + 'model/meta': () => import('./icons/model/meta.svg'), + 'model/minimax': () => import('./icons/model/minimax.svg'), + 'model/mistral': () => import('./icons/model/mistral.svg'), + 'model/moonshot': () => import('./icons/model/moonshot.svg'), + 'model/ollama': () => import('./icons/model/ollama.svg'), + 'model/openai': () => import('./icons/model/openai.svg'), + 'model/qwen': () => import('./icons/model/qwen.svg'), + 'model/sparkDesk': () => import('./icons/model/sparkDesk.svg'), + 'model/stepfun': () => import('./icons/model/stepfun.svg'), + 'model/yi': () => import('./icons/model/yi.svg'), more: () => import('./icons/more.svg'), moreLine: () => import('./icons/moreLine.svg'), out: () => import('./icons/out.svg'), @@ -341,7 +393,9 @@ export const iconPaths = { 'phoneTabbar/me': () => import('./icons/phoneTabbar/me.svg'), 'phoneTabbar/tool': () => import('./icons/phoneTabbar/tool.svg'), 'phoneTabbar/toolFill': () => import('./icons/phoneTabbar/toolFill.svg'), + 'plugins/dingding': () => import('./icons/plugins/dingding.svg'), 'plugins/doc2x': () => import('./icons/plugins/doc2x.svg'), + 'plugins/qiwei': () => import('./icons/plugins/qiwei.svg'), 'plugins/textEditor': () => import('./icons/plugins/textEditor.svg'), point: () => import('./icons/point.svg'), preview: () => import('./icons/preview.svg'), @@ -349,6 +403,7 @@ export const iconPaths = { 'price/right': () => import('./icons/price/right.svg'), save: () => import('./icons/save.svg'), stop: () => import('./icons/stop.svg'), + 'support/account/laf': () => import('./icons/support/account/laf.svg'), 'support/account/loginoutLight': () => import('./icons/support/account/loginoutLight.svg'), 'support/account/plans': () => import('./icons/support/account/plans.svg'), 'support/account/promotionLight': () => import('./icons/support/account/promotionLight.svg'), @@ -359,7 +414,6 @@ export const iconPaths = { 'support/bill/shoppingCart': () => import('./icons/support/bill/shoppingCart.svg'), 'support/bill/wallet': () => import('./icons/support/bill/wallet.svg'), 'support/outlink/apikeyFill': () => import('./icons/support/outlink/apikeyFill.svg'), - 'support/outlink/apikeyLight': () => import('./icons/support/outlink/apikeyLight.svg'), 'support/outlink/iframeLight': () => import('./icons/support/outlink/iframeLight.svg'), 'support/outlink/share': () => import('./icons/support/outlink/share.svg'), 'support/outlink/shareLight': () => import('./icons/support/outlink/shareLight.svg'), @@ -367,10 +421,8 @@ export const iconPaths = { 'support/permission/privateLight': () => import('./icons/support/permission/privateLight.svg'), 'support/permission/publicLight': () => import('./icons/support/permission/publicLight.svg'), 'support/team/group': () => import('./icons/support/team/group.svg'), - 'support/team/key': () => import('./icons/support/team/key.svg'), 'support/team/memberLight': () => import('./icons/support/team/memberLight.svg'), 'support/usage/usageRecordLight': () => import('./icons/support/usage/usageRecordLight.svg'), - 'support/user/individuation': () => import('./icons/support/user/individuation.svg'), 'support/user/informLight': () => import('./icons/support/user/informLight.svg'), 'support/user/userFill': () => import('./icons/support/user/userFill.svg'), 'support/user/userLight': () => import('./icons/support/user/userLight.svg'), diff --git a/packages/web/components/common/Icon/icons/common/arrowRight.svg b/packages/web/components/common/Icon/icons/common/arrowRight.svg new file mode 100644 index 000000000..05b639cbb --- /dev/null +++ b/packages/web/components/common/Icon/icons/common/arrowRight.svg @@ -0,0 +1,3 @@ + + + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/common/dingtalkFill.svg b/packages/web/components/common/Icon/icons/common/dingtalkFill.svg new file mode 100644 index 000000000..742a73f27 --- /dev/null +++ b/packages/web/components/common/Icon/icons/common/dingtalkFill.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/common/downArrowFill.svg b/packages/web/components/common/Icon/icons/common/downArrowFill.svg new file mode 100644 index 000000000..d0f5035e4 --- /dev/null +++ b/packages/web/components/common/Icon/icons/common/downArrowFill.svg @@ -0,0 +1,5 @@ + + + + + diff --git a/packages/web/components/common/Icon/icons/common/layer.svg b/packages/web/components/common/Icon/icons/common/layer.svg new file mode 100644 index 000000000..80d1998f5 --- /dev/null +++ b/packages/web/components/common/Icon/icons/common/layer.svg @@ -0,0 +1,3 @@ + + + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/common/modal.svg b/packages/web/components/common/Icon/icons/common/model.svg similarity index 100% rename from packages/web/components/common/Icon/icons/common/modal.svg rename to packages/web/components/common/Icon/icons/common/model.svg diff --git a/packages/web/components/common/Icon/icons/common/templateMarket.svg b/packages/web/components/common/Icon/icons/common/templateMarket.svg new file mode 100644 index 000000000..b09774b9c --- /dev/null +++ b/packages/web/components/common/Icon/icons/common/templateMarket.svg @@ -0,0 +1,5 @@ + + + + + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/common/thirdParty.svg b/packages/web/components/common/Icon/icons/common/thirdParty.svg new file mode 100644 index 000000000..f6324f43b --- /dev/null +++ b/packages/web/components/common/Icon/icons/common/thirdParty.svg @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/common/variable.svg b/packages/web/components/common/Icon/icons/common/variable.svg new file mode 100644 index 000000000..2d2d89c7f --- /dev/null +++ b/packages/web/components/common/Icon/icons/common/variable.svg @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/common/wallet.svg b/packages/web/components/common/Icon/icons/common/wallet.svg new file mode 100644 index 000000000..add3e9257 --- /dev/null +++ b/packages/web/components/common/Icon/icons/common/wallet.svg @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/common/wecom.svg b/packages/web/components/common/Icon/icons/common/wecom.svg new file mode 100644 index 000000000..1345e42ca --- /dev/null +++ b/packages/web/components/common/Icon/icons/common/wecom.svg @@ -0,0 +1,7 @@ + + + + + + + diff --git a/projects/app/public/appMarketTemplates/TranslateRobot/avatar.svg b/packages/web/components/common/Icon/icons/core/app/templates/TranslateRobot.svg similarity index 100% rename from projects/app/public/appMarketTemplates/TranslateRobot/avatar.svg rename to packages/web/components/common/Icon/icons/core/app/templates/TranslateRobot.svg diff --git a/projects/app/public/appMarketTemplates/animalLife/avatar.svg b/packages/web/components/common/Icon/icons/core/app/templates/animalLife.svg similarity index 100% rename from projects/app/public/appMarketTemplates/animalLife/avatar.svg rename to packages/web/components/common/Icon/icons/core/app/templates/animalLife.svg diff --git a/projects/app/public/appMarketTemplates/Chinese/avatar.svg b/packages/web/components/common/Icon/icons/core/app/templates/chinese.svg similarity index 100% rename from projects/app/public/appMarketTemplates/Chinese/avatar.svg rename to packages/web/components/common/Icon/icons/core/app/templates/chinese.svg diff --git a/projects/app/public/appMarketTemplates/divination/avatar.svg b/packages/web/components/common/Icon/icons/core/app/templates/divination.svg similarity index 100% rename from projects/app/public/appMarketTemplates/divination/avatar.svg rename to packages/web/components/common/Icon/icons/core/app/templates/divination.svg diff --git a/projects/app/public/appMarketTemplates/flux/avatar.svg b/packages/web/components/common/Icon/icons/core/app/templates/flux.svg similarity index 100% rename from projects/app/public/appMarketTemplates/flux/avatar.svg rename to packages/web/components/common/Icon/icons/core/app/templates/flux.svg diff --git a/projects/app/public/appMarketTemplates/githubIssue/avatar.svg b/packages/web/components/common/Icon/icons/core/app/templates/githubIssue.svg similarity index 100% rename from projects/app/public/appMarketTemplates/githubIssue/avatar.svg rename to packages/web/components/common/Icon/icons/core/app/templates/githubIssue.svg diff --git a/projects/app/public/appMarketTemplates/google/avatar.svg b/packages/web/components/common/Icon/icons/core/app/templates/google.svg similarity index 100% rename from projects/app/public/appMarketTemplates/google/avatar.svg rename to packages/web/components/common/Icon/icons/core/app/templates/google.svg diff --git a/projects/app/public/appMarketTemplates/plugin-dalle/avatar.svg b/packages/web/components/common/Icon/icons/core/app/templates/plugin-dalle.svg similarity index 100% rename from projects/app/public/appMarketTemplates/plugin-dalle/avatar.svg rename to packages/web/components/common/Icon/icons/core/app/templates/plugin-dalle.svg diff --git a/projects/app/public/appMarketTemplates/plugin-feishu/avatar.svg b/packages/web/components/common/Icon/icons/core/app/templates/plugin-feishu.svg similarity index 100% rename from projects/app/public/appMarketTemplates/plugin-feishu/avatar.svg rename to packages/web/components/common/Icon/icons/core/app/templates/plugin-feishu.svg diff --git a/projects/app/public/appMarketTemplates/stock/avatar.svg b/packages/web/components/common/Icon/icons/core/app/templates/stock.svg similarity index 100% rename from projects/app/public/appMarketTemplates/stock/avatar.svg rename to packages/web/components/common/Icon/icons/core/app/templates/stock.svg diff --git a/packages/web/components/common/Icon/icons/core/app/type/jsonImport.svg b/packages/web/components/common/Icon/icons/core/app/type/jsonImport.svg new file mode 100644 index 000000000..8ec2dc584 --- /dev/null +++ b/packages/web/components/common/Icon/icons/core/app/type/jsonImport.svg @@ -0,0 +1,19 @@ + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/core/chat/QGFill.svg b/packages/web/components/common/Icon/icons/core/chat/QGFill.svg index 1974fb398..3257f19ab 100644 --- a/packages/web/components/common/Icon/icons/core/chat/QGFill.svg +++ b/packages/web/components/common/Icon/icons/core/chat/QGFill.svg @@ -1 +1,10 @@ - \ No newline at end of file + + + + + + + + + + diff --git a/packages/web/components/common/Icon/icons/core/dataset/feishuDatasetColor.svg b/packages/web/components/common/Icon/icons/core/dataset/feishuDatasetColor.svg new file mode 100644 index 000000000..b52ad1bbf --- /dev/null +++ b/packages/web/components/common/Icon/icons/core/dataset/feishuDatasetColor.svg @@ -0,0 +1,14 @@ + + + + + + + + + + + + + + diff --git a/packages/web/components/common/Icon/icons/core/dataset/feishuDatasetOutline.svg b/packages/web/components/common/Icon/icons/core/dataset/feishuDatasetOutline.svg new file mode 100644 index 000000000..7e5b37385 --- /dev/null +++ b/packages/web/components/common/Icon/icons/core/dataset/feishuDatasetOutline.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/core/dataset/yuqueDatasetColor.svg b/packages/web/components/common/Icon/icons/core/dataset/yuqueDatasetColor.svg new file mode 100644 index 000000000..fb80f52a2 --- /dev/null +++ b/packages/web/components/common/Icon/icons/core/dataset/yuqueDatasetColor.svg @@ -0,0 +1,14 @@ + + + + + + + + + + + + + + diff --git a/packages/web/components/common/Icon/icons/core/dataset/yuqueDatasetOutline.svg b/packages/web/components/common/Icon/icons/core/dataset/yuqueDatasetOutline.svg new file mode 100644 index 000000000..97daba221 --- /dev/null +++ b/packages/web/components/common/Icon/icons/core/dataset/yuqueDatasetOutline.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/core/workflow/template/searxng.svg b/packages/web/components/common/Icon/icons/core/workflow/template/searxng.svg new file mode 100644 index 000000000..5b55fa275 --- /dev/null +++ b/packages/web/components/common/Icon/icons/core/workflow/template/searxng.svg @@ -0,0 +1,96 @@ + + + + + + + + image/svg+xml + + + + + + + + + + + + + + + + + diff --git a/packages/web/components/common/Icon/icons/support/team/key.svg b/packages/web/components/common/Icon/icons/key.svg similarity index 100% rename from packages/web/components/common/Icon/icons/support/team/key.svg rename to packages/web/components/common/Icon/icons/key.svg diff --git a/packages/web/components/common/Icon/icons/keyPrimary.svg b/packages/web/components/common/Icon/icons/keyPrimary.svg new file mode 100644 index 000000000..836c66079 --- /dev/null +++ b/packages/web/components/common/Icon/icons/keyPrimary.svg @@ -0,0 +1,9 @@ + + + + + + + + + diff --git a/packages/web/components/common/Icon/icons/model/BAAI.svg b/packages/web/components/common/Icon/icons/model/BAAI.svg new file mode 100644 index 000000000..938361b67 --- /dev/null +++ b/packages/web/components/common/Icon/icons/model/BAAI.svg @@ -0,0 +1,3 @@ + + + diff --git a/packages/web/components/common/Icon/icons/model/alicloud.svg b/packages/web/components/common/Icon/icons/model/alicloud.svg new file mode 100644 index 000000000..4a6104a70 --- /dev/null +++ b/packages/web/components/common/Icon/icons/model/alicloud.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/projects/app/public/imgs/model/baichuan.svg b/packages/web/components/common/Icon/icons/model/baichuan.svg similarity index 100% rename from projects/app/public/imgs/model/baichuan.svg rename to packages/web/components/common/Icon/icons/model/baichuan.svg diff --git a/projects/app/public/imgs/model/chatglm.svg b/packages/web/components/common/Icon/icons/model/chatglm.svg similarity index 100% rename from projects/app/public/imgs/model/chatglm.svg rename to packages/web/components/common/Icon/icons/model/chatglm.svg diff --git a/projects/app/public/imgs/model/claude.svg b/packages/web/components/common/Icon/icons/model/claude.svg similarity index 100% rename from projects/app/public/imgs/model/claude.svg rename to packages/web/components/common/Icon/icons/model/claude.svg diff --git a/projects/app/public/imgs/model/deepseek.svg b/packages/web/components/common/Icon/icons/model/deepseek.svg similarity index 100% rename from projects/app/public/imgs/model/deepseek.svg rename to packages/web/components/common/Icon/icons/model/deepseek.svg diff --git a/packages/web/components/common/Icon/icons/model/doubao.svg b/packages/web/components/common/Icon/icons/model/doubao.svg new file mode 100644 index 000000000..f964a1a26 --- /dev/null +++ b/packages/web/components/common/Icon/icons/model/doubao.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/projects/app/public/imgs/model/ernie.svg b/packages/web/components/common/Icon/icons/model/ernie.svg similarity index 100% rename from projects/app/public/imgs/model/ernie.svg rename to packages/web/components/common/Icon/icons/model/ernie.svg diff --git a/packages/web/components/common/Icon/icons/model/fishaudio.svg b/packages/web/components/common/Icon/icons/model/fishaudio.svg new file mode 100644 index 000000000..d40f0febd --- /dev/null +++ b/packages/web/components/common/Icon/icons/model/fishaudio.svg @@ -0,0 +1,5 @@ + + + + + diff --git a/packages/web/components/common/Icon/icons/model/gemini.svg b/packages/web/components/common/Icon/icons/model/gemini.svg new file mode 100644 index 000000000..e96d01953 --- /dev/null +++ b/packages/web/components/common/Icon/icons/model/gemini.svg @@ -0,0 +1,9 @@ + + + + + + + + + diff --git a/packages/web/components/common/Icon/icons/model/groq.svg b/packages/web/components/common/Icon/icons/model/groq.svg new file mode 100644 index 000000000..4b3f1c50d --- /dev/null +++ b/packages/web/components/common/Icon/icons/model/groq.svg @@ -0,0 +1,3 @@ + + + \ No newline at end of file diff --git a/projects/app/public/imgs/model/huggingface.svg b/packages/web/components/common/Icon/icons/model/huggingface.svg similarity index 100% rename from projects/app/public/imgs/model/huggingface.svg rename to packages/web/components/common/Icon/icons/model/huggingface.svg diff --git a/packages/web/components/common/Icon/icons/model/hunyuan.svg b/packages/web/components/common/Icon/icons/model/hunyuan.svg new file mode 100644 index 000000000..ec2157e28 --- /dev/null +++ b/packages/web/components/common/Icon/icons/model/hunyuan.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/model/meta.svg b/packages/web/components/common/Icon/icons/model/meta.svg new file mode 100644 index 000000000..fcc771ee7 --- /dev/null +++ b/packages/web/components/common/Icon/icons/model/meta.svg @@ -0,0 +1,77 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/packages/web/components/common/Icon/icons/model/minimax.svg b/packages/web/components/common/Icon/icons/model/minimax.svg new file mode 100644 index 000000000..c80236e5e --- /dev/null +++ b/packages/web/components/common/Icon/icons/model/minimax.svg @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/model/mistral.svg b/packages/web/components/common/Icon/icons/model/mistral.svg new file mode 100644 index 000000000..735981b91 --- /dev/null +++ b/packages/web/components/common/Icon/icons/model/mistral.svg @@ -0,0 +1,12 @@ + + + + + + + + + + + + \ No newline at end of file diff --git a/projects/app/public/imgs/model/moonshot.svg b/packages/web/components/common/Icon/icons/model/moonshot.svg similarity index 100% rename from projects/app/public/imgs/model/moonshot.svg rename to packages/web/components/common/Icon/icons/model/moonshot.svg diff --git a/packages/web/components/common/Icon/icons/model/ollama.svg b/packages/web/components/common/Icon/icons/model/ollama.svg new file mode 100644 index 000000000..74de384a3 --- /dev/null +++ b/packages/web/components/common/Icon/icons/model/ollama.svg @@ -0,0 +1,5 @@ + + + + + \ No newline at end of file diff --git a/projects/app/public/imgs/model/openai.svg b/packages/web/components/common/Icon/icons/model/openai.svg similarity index 100% rename from projects/app/public/imgs/model/openai.svg rename to packages/web/components/common/Icon/icons/model/openai.svg diff --git a/packages/web/components/common/Icon/icons/model/qwen.svg b/packages/web/components/common/Icon/icons/model/qwen.svg new file mode 100644 index 000000000..c46b8f052 --- /dev/null +++ b/packages/web/components/common/Icon/icons/model/qwen.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/projects/app/public/imgs/model/sparkDesk.svg b/packages/web/components/common/Icon/icons/model/sparkDesk.svg similarity index 100% rename from projects/app/public/imgs/model/sparkDesk.svg rename to packages/web/components/common/Icon/icons/model/sparkDesk.svg diff --git a/packages/web/components/common/Icon/icons/model/stepfun.svg b/packages/web/components/common/Icon/icons/model/stepfun.svg new file mode 100644 index 000000000..a78ebed2f --- /dev/null +++ b/packages/web/components/common/Icon/icons/model/stepfun.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/model/yi.svg b/packages/web/components/common/Icon/icons/model/yi.svg new file mode 100644 index 000000000..38a883dae --- /dev/null +++ b/packages/web/components/common/Icon/icons/model/yi.svg @@ -0,0 +1,14 @@ + + + + + + + + + + + + + + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/phoneTabbar/tool.svg b/packages/web/components/common/Icon/icons/phoneTabbar/tool.svg index b77170d86..440720cb0 100644 --- a/packages/web/components/common/Icon/icons/phoneTabbar/tool.svg +++ b/packages/web/components/common/Icon/icons/phoneTabbar/tool.svg @@ -1,4 +1,3 @@ - - + + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/plugins/dingding.svg b/packages/web/components/common/Icon/icons/plugins/dingding.svg new file mode 100644 index 000000000..c204d8506 --- /dev/null +++ b/packages/web/components/common/Icon/icons/plugins/dingding.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/plugins/qiwei.svg b/packages/web/components/common/Icon/icons/plugins/qiwei.svg new file mode 100644 index 000000000..be307bfa4 --- /dev/null +++ b/packages/web/components/common/Icon/icons/plugins/qiwei.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/support/account/laf.svg b/packages/web/components/common/Icon/icons/support/account/laf.svg new file mode 100644 index 000000000..c908b6a8a --- /dev/null +++ b/packages/web/components/common/Icon/icons/support/account/laf.svg @@ -0,0 +1,9 @@ + + + + + + + + + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/support/outlink/apikeyLight.svg b/packages/web/components/common/Icon/icons/support/outlink/apikeyLight.svg deleted file mode 100644 index dee84de25..000000000 --- a/packages/web/components/common/Icon/icons/support/outlink/apikeyLight.svg +++ /dev/null @@ -1,8 +0,0 @@ - - - - - \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/support/user/individuation.svg b/packages/web/components/common/Icon/icons/support/user/individuation.svg deleted file mode 100644 index 63b6af4f4..000000000 --- a/packages/web/components/common/Icon/icons/support/user/individuation.svg +++ /dev/null @@ -1,4 +0,0 @@ - - - \ No newline at end of file diff --git a/packages/web/components/common/Image/PhotoView.tsx b/packages/web/components/common/Image/PhotoView.tsx index 33cbdd615..e7867e212 100644 --- a/packages/web/components/common/Image/PhotoView.tsx +++ b/packages/web/components/common/Image/PhotoView.tsx @@ -6,8 +6,9 @@ import { useSystem } from '../../../hooks/useSystem'; import Loading from '../MyLoading'; import MyImage from './MyImage'; -const MyPhotoView = ({ ...props }: ImageProps) => { +const MyPhotoView = (props: ImageProps) => { const { isPc } = useSystem(); + return ( { ? register(name, { required: props.isRequired, min: props.min, - max: props.max + max: props.max, + valueAsNumber: true }) : {})} /> @@ -57,4 +58,4 @@ const MyNumberInput = (props: Props) => { ); }; -export default MyNumberInput; +export default React.memo(MyNumberInput); diff --git a/packages/web/components/common/Input/SearchInput/index.tsx b/packages/web/components/common/Input/SearchInput/index.tsx index 313f9b354..99affe0ca 100644 --- a/packages/web/components/common/Input/SearchInput/index.tsx +++ b/packages/web/components/common/Input/SearchInput/index.tsx @@ -13,4 +13,4 @@ const SearchInput = (props: InputProps) => { ); }; -export default SearchInput; +export default React.memo(SearchInput); diff --git a/packages/web/components/common/LightTip/index.tsx b/packages/web/components/common/LightTip/index.tsx index 6aaffb33e..5d6bc6540 100644 --- a/packages/web/components/common/LightTip/index.tsx +++ b/packages/web/components/common/LightTip/index.tsx @@ -8,9 +8,17 @@ const LightTip = ({ text: string; } & StackProps) => { return ( - + - {text} + {text} ); }; diff --git a/packages/web/components/common/MyBox/index.tsx b/packages/web/components/common/MyBox/index.tsx index e5f8d70cd..501ff568a 100644 --- a/packages/web/components/common/MyBox/index.tsx +++ b/packages/web/components/common/MyBox/index.tsx @@ -17,4 +17,4 @@ const MyBox = ({ text, isLoading, children, size, ...props }: Props, ref: any) = ); }; -export default forwardRef(MyBox); +export default React.memo(forwardRef(MyBox)); diff --git a/packages/web/components/common/MyLoading/index.tsx b/packages/web/components/common/MyLoading/index.tsx index 9303b6482..a2664f764 100644 --- a/packages/web/components/common/MyLoading/index.tsx +++ b/packages/web/components/common/MyLoading/index.tsx @@ -44,4 +44,4 @@ const Loading = ({ ); }; -export default Loading; +export default React.memo(Loading); diff --git a/packages/web/components/common/MyModal/index.tsx b/packages/web/components/common/MyModal/index.tsx index dc6fd2138..d49337a5c 100644 --- a/packages/web/components/common/MyModal/index.tsx +++ b/packages/web/components/common/MyModal/index.tsx @@ -50,6 +50,8 @@ const MyModal = ({ autoFocus={false} isCentered={isPc ? isCentered : true} blockScrollOnMount={false} + allowPinchZoom + scrollBehavior={'inside'} closeOnOverlayClick={closeOnOverlayClick} > @@ -73,6 +75,7 @@ const MyModal = ({ py={'10px'} fontSize={'md'} fontWeight={'bold'} + minH={['46px', '53px']} > {iconSrc && ( <> diff --git a/packages/web/components/common/MySelect/CronSelector.tsx b/packages/web/components/common/MySelect/CronSelector.tsx index b7266f7d1..33a0e4eb2 100644 --- a/packages/web/components/common/MySelect/CronSelector.tsx +++ b/packages/web/components/common/MySelect/CronSelector.tsx @@ -42,7 +42,7 @@ export const cronString2Fields = (cronString?: string) => { }; export const cronString2Label = ( - cronString: string, + cronString = '', t: any // i18nT ) => { const cronField = cronString2Fields(cronString); diff --git a/packages/web/components/common/MySelect/MultipleRowSelect.tsx b/packages/web/components/common/MySelect/MultipleRowSelect.tsx index 64ad94678..52cf223b3 100644 --- a/packages/web/components/common/MySelect/MultipleRowSelect.tsx +++ b/packages/web/components/common/MySelect/MultipleRowSelect.tsx @@ -1,5 +1,16 @@ -import React, { useRef, useCallback, useState, useMemo } from 'react'; -import { Button, useDisclosure, Box, Flex, useOutsideClick, Checkbox } from '@chakra-ui/react'; +import React, { useRef, useCallback, useState, useMemo, useEffect } from 'react'; +import { + Button, + useDisclosure, + Box, + Flex, + useOutsideClick, + Checkbox, + css, + Menu, + MenuButton, + MenuList +} from '@chakra-ui/react'; import { ListItemType, MultipleArraySelectProps, MultipleSelectProps } from './type'; import EmptyTip from '../EmptyTip'; import { useTranslation } from 'next-i18next'; @@ -13,19 +24,34 @@ export const MultipleRowSelect = ({ emptyTip, maxH = 300, onSelect, - popDirection = 'bottom', - styles, - changeOnEverySelect = false -}: MultipleSelectProps) => { + ButtonProps, + changeOnEverySelect = false, + rowMinWidth = 'autp' +}: MultipleSelectProps & { + rowMinWidth?: string; +}) => { const { t } = useTranslation(); - const ref = useRef(null); + const ButtonRef = useRef(null); + const { isOpen, onOpen, onClose } = useDisclosure(); const [cloneValue, setCloneValue] = useState(value); - useOutsideClick({ - ref: ref, - handler: onClose - }); + const MenuRef = useRef<(HTMLDivElement | null)[]>([]); + const SelectedItemRef = useRef<(HTMLDivElement | null)[]>([]); + + useEffect(() => { + if (isOpen) { + for (let i = 0; i < MenuRef.current.length; i++) { + const menu = MenuRef.current[i]; + const selectedItem = SelectedItemRef.current[i]; + if (menu && selectedItem) { + menu.scrollTop = selectedItem.offsetTop - menu.offsetTop - 100; + } + } + } + }, [isOpen]); + + const minWidth = `${MenuRef.current?.[0]?.offsetWidth || 0}px`; const RenderList = useCallback( ({ index, list }: { index: number; list: MultipleSelectProps['list'] }) => { @@ -34,14 +60,26 @@ export const MultipleRowSelect = ({ const children = list[selectedIndex]?.children || []; const hasChildren = list.some((item) => item.children && item.children?.length > 0); + // Store current scroll position before update + const currentScrollTop = MenuRef.current[index]?.scrollTop; + // Use useEffect to restore scroll position after render + useEffect(() => { + if (currentScrollTop !== undefined && MenuRef.current[index]) { + MenuRef.current[index]!.scrollTop = currentScrollTop; + } + }, [cloneValue, currentScrollTop]); + return ( <> { + MenuRef.current[index] = ref; + }} className="nowheel" flex={'1 0 auto'} - // width={0} px={2} borderLeft={index !== 0 ? 'base' : 'none'} + minW={index !== 0 ? minWidth : rowMinWidth} maxH={`${maxH}px`} overflowY={'auto'} whiteSpace={'nowrap'} @@ -49,13 +87,18 @@ export const MultipleRowSelect = ({ {list.map((item) => ( { + if (item.value === selectedValue) { + SelectedItemRef.current[index] = ref; + } + }} + py={1.5} + _notLast={{ mb: 1 }} cursor={'pointer'} - px={2} - borderRadius={'md'} + px={1.5} + borderRadius={'sm'} _hover={{ - bg: 'primary.50', - color: 'primary.600' + bg: 'primary.50' }} onClick={() => { const newValue = [...cloneValue]; @@ -81,6 +124,7 @@ export const MultipleRowSelect = ({ }} {...(item.value === selectedValue ? { + bg: 'primary.50', color: 'primary.600' } : {})} @@ -109,60 +153,73 @@ export const MultipleRowSelect = ({ }, [value, onOpen]); return ( - - } - _active={{ - transform: 'none' - }} - {...(isOpen - ? { - borderColor: 'primary.600', - color: 'primary.700', - boxShadow: '0px 0px 0px 2.4px rgba(51, 112, 255, 0.15)' - } - : { - borderColor: 'myGray.200', - boxShadow: 'none' - })} - {...styles} - onClick={() => (isOpen ? onClose() : onOpenSelect())} + + - {label ?? placeholder} - - {isOpen && ( - } + variant={'whitePrimaryOutline'} + size={'lg'} + fontSize={'sm'} + textAlign={'left'} + _active={{ + transform: 'none' + }} + {...(isOpen ? { - transform: 'translateY(-105%)', - top: '0' + boxShadow: '0px 0px 0px 2.4px rgba(51, 112, 255, 0.15)', + borderColor: 'primary.600', + color: 'primary.700' } - : { - transform: 'translateY(105%)', - bottom: '0' - })} - py={2} - bg={'white'} - border={'1px solid #fff'} - boxShadow={'5'} - borderRadius={'md'} - zIndex={1} - minW={'100%'} - w={'max-content'} + : {})} + {...ButtonProps} > - - - - - )} + {label ?? placeholder} + + { + const w = ButtonRef.current?.clientWidth; + if (w) { + return `${w}px !important`; + } + + const width = ButtonProps?.width; + return Array.isArray(width) + ? width.map((item) => `${item} !important`) + : `${width} !important`; + })()} + w={'auto'} + py={'6px'} + border={'1px solid #fff'} + boxShadow={ + '0px 2px 4px rgba(161, 167, 179, 0.25), 0px 0px 1px rgba(121, 141, 159, 0.25);' + } + zIndex={99} + maxH={'40vh'} + overflowY={'auto'} + display={'flex'} + userSelect={'none'} + > + + + ); }; @@ -176,7 +233,7 @@ export const MultipleRowArraySelect = ({ maxH = 300, onSelect, popDirection = 'bottom', - styles + ButtonProps }: MultipleArraySelectProps) => { const { t } = useTranslation(); const ref = useRef(null); @@ -184,8 +241,9 @@ export const MultipleRowArraySelect = ({ const [navigationPath, setNavigationPath] = useState([]); + // Make sure the value is an array of arrays const formatValue = useMemo(() => { - return Array.isArray(value) ? value : []; + return Array.isArray(value) ? value.filter((v) => Array.isArray(v)) : []; }, [value]); // Close when clicking outside @@ -193,6 +251,18 @@ export const MultipleRowArraySelect = ({ ref: ref, handler: onClose }); + const onChange = useCallback( + (val: any[][]) => { + // Filter invalid value + const validList = val.filter((item) => { + const listItem = list.find((v) => v.value === item[0]); + if (!listItem) return false; + return listItem.children?.some((v) => v.value === item[1]); + }); + onSelect(validList); + }, + [onSelect] + ); const RenderList = useCallback( ({ index, list }: { index: number; list: MultipleSelectProps['list'] }) => { @@ -213,9 +283,9 @@ export const MultipleRowArraySelect = ({ const newValue = [parentValue, item.value]; if (newValues.some((v) => v[0] === parentValue && v[1] === item.value)) { - onSelect(newValues.filter((v) => !(v[0] === parentValue && v[1] === item.value))); + onChange(newValues.filter((v) => !(v[0] === parentValue && v[1] === item.value))); } else { - onSelect([...newValues, newValue]); + onChange([...newValues, newValue]); } } }; @@ -305,7 +375,7 @@ export const MultipleRowArraySelect = ({ borderColor: 'myGray.200', boxShadow: 'none' })} - {...styles} + {...ButtonProps} onClick={() => (isOpen ? onClose() : onOpenSelect())} className="nowheel" > @@ -330,7 +400,7 @@ export const MultipleRowArraySelect = ({ border={'1px solid #fff'} boxShadow={'5'} borderRadius={'md'} - zIndex={1} + zIndex={1000} minW={'100%'} w={'max-content'} > diff --git a/packages/web/components/common/MySelect/index.tsx b/packages/web/components/common/MySelect/index.tsx index 36a3eb59f..7d503d21b 100644 --- a/packages/web/components/common/MySelect/index.tsx +++ b/packages/web/components/common/MySelect/index.tsx @@ -110,7 +110,7 @@ const MySelect = ( px={3} rightIcon={} variant={'whitePrimaryOutline'} - size={'lg'} + size={'md'} fontSize={'sm'} textAlign={'left'} _active={{ diff --git a/packages/web/components/common/MySelect/type.d.ts b/packages/web/components/common/MySelect/type.d.ts index 10dc2de2a..783d1e395 100644 --- a/packages/web/components/common/MySelect/type.d.ts +++ b/packages/web/components/common/MySelect/type.d.ts @@ -1,3 +1,5 @@ +import { ButtonProps } from '@chakra-ui/react'; + type ListItemType = { alias?: string; label: string | React.ReactNode; @@ -12,9 +14,9 @@ export type MultipleSelectProps = { emptyTip?: string; maxH?: number; onSelect: (val: any[]) => void; - styles?: ButtonProps; popDirection?: 'top' | 'bottom'; changeOnEverySelect?: boolean; + ButtonProps?: ButtonProps; }; export type MultipleArraySelectProps = Omit & { value?: any[][]; diff --git a/packages/web/components/common/MySlider/InputSlider.tsx b/packages/web/components/common/MySlider/InputSlider.tsx new file mode 100644 index 000000000..a8726e433 --- /dev/null +++ b/packages/web/components/common/MySlider/InputSlider.tsx @@ -0,0 +1,80 @@ +import React, { useMemo } from 'react'; +import { Slider, SliderTrack, SliderThumb, HStack, SliderMark } from '@chakra-ui/react'; +import MyNumberInput from '../Input/NumberInput'; + +const InputSlider = ({ + onChange, + value, + max = 100, + min = 0, + step = 1, + isDisabled +}: { + value?: number; + onChange: (index: number) => void; + max: number; + min: number; + step?: number; + isDisabled?: boolean; +}) => { + const markList = useMemo(() => { + const valLen = max - min; + return [ + valLen * 0.007 + min, + valLen * 0.2 + min, + valLen * 0.4 + min, + valLen * 0.6 + min, + valLen * 0.8 + min, + valLen * 0.985 + min + ]; + }, []); + + return ( + + + + {markList.map((val, i) => ( + + ))} + + + onChange(e ?? min)} + /> + + ); +}; + +export default React.memo(InputSlider); diff --git a/packages/web/components/common/MyTooltip/QuestionTip.tsx b/packages/web/components/common/MyTooltip/QuestionTip.tsx index 506d22c64..f9d409003 100644 --- a/packages/web/components/common/MyTooltip/QuestionTip.tsx +++ b/packages/web/components/common/MyTooltip/QuestionTip.tsx @@ -15,4 +15,4 @@ const QuestionTip = ({ label, maxW, ...props }: Props) => { ); }; -export default QuestionTip; +export default React.memo(QuestionTip); diff --git a/packages/web/components/common/Tabs/FillRowTabs.tsx b/packages/web/components/common/Tabs/FillRowTabs.tsx index 7388af742..f09834955 100644 --- a/packages/web/components/common/Tabs/FillRowTabs.tsx +++ b/packages/web/components/common/Tabs/FillRowTabs.tsx @@ -1,15 +1,15 @@ -import React from 'react'; +import React, { forwardRef } from 'react'; import { Flex, Box, BoxProps } from '@chakra-ui/react'; import MyIcon from '../Icon'; -type Props = Omit & { +type Props = Omit & { list: { icon?: string; label: string | React.ReactNode; - value: string; + value: T; }[]; - value: string; - onChange: (e: string) => void; + value: T; + onChange: (e: T) => void; }; const FillRowTabs = ({ list, value, onChange, py = '7px', px = '12px', ...props }: Props) => { @@ -61,4 +61,6 @@ const FillRowTabs = ({ list, value, onChange, py = '7px', px = '12px', ...props ); }; -export default FillRowTabs; +export default forwardRef(FillRowTabs) as ( + props: Props & { ref?: React.Ref } +) => JSX.Element; diff --git a/packages/web/components/common/Textarea/CustomPromptEditor/index.tsx b/packages/web/components/common/Textarea/CustomPromptEditor/index.tsx new file mode 100644 index 000000000..776dcf4af --- /dev/null +++ b/packages/web/components/common/Textarea/CustomPromptEditor/index.tsx @@ -0,0 +1,158 @@ +import { + Box, + Button, + Flex, + Textarea, + ModalFooter, + HStack, + Icon, + ModalBody +} from '@chakra-ui/react'; +import MyIcon from '../../Icon/index'; + +import React, { useCallback, useEffect, useRef, useState } from 'react'; +import { useTranslation } from 'next-i18next'; +import MyModal from '../../MyModal'; + +const CustomLightTip = () => { + const { t } = useTranslation(); + + return ( + + + + {t('common:core.app.QG.Custom prompt tip1')} + + {t('common:core.app.QG.Custom prompt tip2')} + + {t('common:core.app.QG.Custom prompt tip3')} + + + ); +}; + +const FixBox = ({ children }: { children: React.ReactNode }) => { + return ( + + + {children} + + + ); +}; + +const CustomPromptEditor = ({ + defaultValue = '', + defaultPrompt, + footerPrompt, + onChange, + onClose +}: { + defaultValue?: string; + defaultPrompt: string; + footerPrompt?: string; + onChange: (e: string) => void; + onClose: () => void; +}) => { + const ref = useRef(null); + const { t } = useTranslation(); + const [value, setValue] = useState(defaultValue || defaultPrompt); + + const adjustHeight = useCallback(() => { + const textarea = ref.current; + if (!textarea) return; + + textarea.style.height = '22px'; + textarea.style.height = `${textarea.scrollHeight}px`; + }, []); + + useEffect(() => { + adjustHeight(); + const timer = setTimeout(adjustHeight, 0); + return () => clearTimeout(timer); + }, [value, adjustHeight]); + + return ( + + + + + + + {t('common:core.ai.Prompt')} + + + + + + +