feat: Text check before synchronization (#689)

* fix: icon

* fix: web selector

* fix: web selector

* perf: link sync

* dev doc

* chomd doc

* perf: git intro

* 466 intro

* intro img

* add json editor (#5)

* team limit

* websync limit

* json editor

* text editor

* perf: search test

* change cq value type

* doc

* intro img

---------

Co-authored-by: heheer <71265218+newfish-cmyk@users.noreply.github.com>

.dockerignore

@@ -8,4 +8,5 @@ README.md
 
 .yalc/
 yalc.lock
-testApi/
+testApi/
+*.local.*

.gitignore

@@ -35,4 +35,5 @@ dist/
 **/.hugo_build.lock
 docSite/public/
 docSite/resources/_gen/
-docSite/.vercel
+docSite/.vercel
+*.local.*

.prettierignore

@@ -2,4 +2,5 @@ dist
 .vscode
 **/.DS_Store
 node_modules
-docSite/
+docSite/
+*.md

.vscode/settings.json

@@ -4,7 +4,7 @@
     "typescript.tsdk": "node_modules/typescript/lib",
     "prettier.prettierPath": "./node_modules/prettier",
     "i18n-ally.localesPaths": [
-        "projects/app/public/locales"
+        "projects/app/public/locales",
     ],
     "i18n-ally.enabledParsers": ["json"],
     "i18n-ally.keystyle": "nested",

Dockerfile

@@ -1,59 +1,84 @@
-# Install dependencies only when needed
-FROM node:18.15-alpine AS deps
-# Check https://github.com/nodejs/docker-node/tree/b4117f9333da4138b03a546ec926ef50a31506c3#nodealpine to understand why libc6-compat might be needed.
-RUN apk add libc6-compat && npm install -g pnpm
+# --------- install dependence -----------
+FROM node:18.17-alpine AS mainDeps
 WORKDIR /app
 
 ARG name
+ARG proxy
+
+RUN [ -z "$proxy" ] || sed -i 's/dl-cdn.alpinelinux.org/mirrors.ustc.edu.cn/g' /etc/apk/repositories
+RUN apk add --no-cache libc6-compat && npm install -g pnpm@8.6.0
+# if proxy exists, set proxy
+RUN [ -z "$proxy" ] || pnpm config set registry https://registry.npm.taobao.org
 
 # copy packages and one project
-COPY package.json pnpm-lock.yaml pnpm-workspace.yaml ./
+COPY pnpm-lock.yaml pnpm-workspace.yaml ./
 COPY ./packages ./packages
 COPY ./projects/$name/package.json ./projects/$name/package.json
 
 RUN [ -f pnpm-lock.yaml ] || (echo "Lockfile not found." && exit 1)
 
-RUN pnpm install
+RUN pnpm i
 
-# Rebuild the source code only when needed
-FROM node:18.15-alpine AS builder
+# --------- install dependence -----------
+FROM node:18.17-alpine AS workerDeps
+WORKDIR /app
+
+ARG proxy
+
+RUN [ -z "$proxy" ] || sed -i 's/dl-cdn.alpinelinux.org/mirrors.ustc.edu.cn/g' /etc/apk/repositories
+RUN apk add --no-cache libc6-compat && npm install -g pnpm@8.6.0
+# if proxy exists, set proxy
+RUN [ -z "$proxy" ] || pnpm config set registry https://registry.npm.taobao.org
+
+COPY ./worker /app/worker
+RUN cd /app/worker && pnpm i --production --ignore-workspace
+
+# --------- builder -----------
+FROM node:18.17-alpine AS builder
 WORKDIR /app
 
 ARG name
+ARG proxy
 
 # copy common node_modules and one project node_modules
 COPY package.json pnpm-workspace.yaml ./
-COPY --from=deps /app/node_modules ./node_modules
-COPY --from=deps /app/packages ./packages
+COPY --from=mainDeps /app/node_modules ./node_modules
+COPY --from=mainDeps /app/packages ./packages
 COPY ./projects/$name ./projects/$name
-COPY --from=deps /app/projects/$name/node_modules ./projects/$name/node_modules
+COPY --from=mainDeps /app/projects/$name/node_modules ./projects/$name/node_modules
 
-# Uncomment the following line in case you want to disable telemetry during the build.
-ENV NEXT_TELEMETRY_DISABLED 1
-RUN npm install -g pnpm
-RUN pnpm --filter=$name run build
+RUN [ -z "$proxy" ] || sed -i 's/dl-cdn.alpinelinux.org/mirrors.ustc.edu.cn/g' /etc/apk/repositories
 
-FROM node:18.15-alpine AS runner
+RUN apk add --no-cache libc6-compat && npm install -g pnpm@8.6.0
+RUN pnpm --filter=$name build
+
+# --------- runner -----------
+FROM node:18.17-alpine AS runner
 WORKDIR /app
 
 ARG name
+ARG proxy
 
 # create user and use it
 RUN addgroup --system --gid 1001 nodejs
 RUN adduser --system --uid 1001 nextjs
 
-RUN sed -i 's/https/http/' /etc/apk/repositories
-RUN apk add curl \
-  && apk add ca-certificates \
+RUN [ -z "$proxy" ] || sed -i 's/dl-cdn.alpinelinux.org/mirrors.ustc.edu.cn/g' /etc/apk/repositories
+RUN apk add --no-cache curl ca-certificates \
   && update-ca-certificates
 
 # copy running files
-COPY --from=builder /app/projects/$name/public ./projects/$name/public
-COPY --from=builder /app/projects/$name/next.config.js ./projects/$name/next.config.js
-COPY --from=builder --chown=nextjs:nodejs /app/projects/$name/.next/standalone ./
-COPY --from=builder --chown=nextjs:nodejs /app/projects/$name/.next/static ./projects/$name/.next/static
+COPY --from=builder /app/projects/$name/public /app/projects/$name/public
+COPY --from=builder /app/projects/$name/next.config.js /app/projects/$name/next.config.js
+COPY --from=builder --chown=nextjs:nodejs /app/projects/$name/.next/standalone /app/
+COPY --from=builder --chown=nextjs:nodejs /app/projects/$name/.next/static /app/projects/$name/.next/static
 # copy package.json to version file
 COPY --from=builder /app/projects/$name/package.json ./package.json 
+# copy woker
+COPY --from=workerDeps /app/worker /app/worker
+# copy config
+COPY ./projects/$name/data /app/data
+
 
 ENV NODE_ENV production
 ENV NEXT_TELEMETRY_DISABLED 1

README.md

@@ -1,6 +1,6 @@
 <div align="center">
 
-<a href="https://fastgpt.run/"><img src="/.github/imgs/logo.svg" width="120" height="120" alt="fastgpt logo"></a>
+<a href="https://fastgpt.in/"><img src="/.github/imgs/logo.svg" width="120" height="120" alt="fastgpt logo"></a>
 
 # FastGPT
 
@@ -15,7 +15,7 @@ FastGPT 是一个基于 LLM 大语言模型的知识库问答系统，提供开
 </div>
 
 <p align="center">
-  <a href="https://fastgpt.run/">
+  <a href="https://fastgpt.in/">
     <img height="21" src="https://img.shields.io/badge/在线使用-d4eaf7?style=flat-square&logo=spoj&logoColor=7d09f1" alt="cloud">
   </a>
   <a href="https://doc.fastgpt.in/docs/intro">
@@ -36,7 +36,7 @@ https://github.com/labring/FastGPT/assets/15308462/7d3a38df-eb0e-4388-9250-2409b
 
 ## 🛸 在线使用
 
-- 🌐 国内版：[ai.fastgpt.in](https://ai.fastgpt.in/)
+- 🌐 国内临时可访问：[fastgpt.in](https://fastgpt.in/)
 - 🌍 海外版：[fastgpt.run](https://fastgpt.run/)
 
 |                                    |                                    |
@@ -48,39 +48,47 @@ https://github.com/labring/FastGPT/assets/15308462/7d3a38df-eb0e-4388-9250-2409b
     <img src="https://img.shields.io/badge/-返回顶部-7d09f1.svg" alt="#" align="right">
 </a>
 
-## 💡 功能
+## 💡 RoadMap
 
-1. 强大的可视化编排，轻松构建 AI 应用
+`1` 应用编排能力
    - [x] 提供简易模式，无需操作编排
-   - [x] 用户对话前引导，全局字符串变量
-   - [x] 知识库搜索
-   - [x] 多 LLM 模型对话
-   - [x] 文本内容提取成结构化数据
-   - [x] HTTP 扩展
-   - [ ] 嵌入 Laf，实现在线编写 HTTP 模块
    - [x] 对话下一步指引
-   - [ ] 对话多路线选择
+   - [x] 工作流编排
    - [x] 源文件引用追踪
    - [x] 模块封装，实现多级复用
-2. 丰富的知识库预处理
+   - [x] 混合检索 & 重排
+   - [ ] 自查询规划
+   - [ ] 嵌入 [Laf](https://github.com/labring/laf)，实现在线编写 HTTP 模块
+   - [ ] 插件封装功能
+
+`2` 知识库能力
    - [x] 多库复用，混用
    - [x] chunk 记录修改和删除
-   - [x] 支持手动输入，直接分段，QA 拆分导入
-   - [x] 支持 url 读取、CSV 批量导入
    - [x] 支持知识库单独设置向量模型
    - [x] 源文件存储
-   - [ ] 文件学习 Agent
-3. 多种效果测试渠道
+   - [x] 支持手动输入，直接分段，QA 拆分导入
+   - [x] 支持 pdf、word、txt、md 等常用文件，支持 url 读取、CSV 批量导入
+   - [ ] 支持 HTML、csv、PPT、Excel 导入
+   - [ ] 支持文件阅读器
+   - [ ] 支持差异性文件同步
+   - [ ] 更多的数据预处理方案
+  
+`3` 应用调试能力
    - [x] 知识库单点搜索测试
    - [x] 对话时反馈引用并可修改与删除
    - [x] 完整上下文呈现
    - [x] 完整模块中间值呈现
-4. OpenAPI
+   - [ ] 高级编排 DeBug 模式
+  
+`4` OpenAPI 接口
    - [x] completions 接口 (对齐 GPT 接口)
    - [ ] 知识库 CRUD
-5. 运营功能
+   - [ ] 对话 CRUD
+  
+`5` 运营能力
    - [x] 免登录分享窗口
    - [x] Iframe 一键嵌入
+   - [x] 聊天窗口嵌入支持自定义 Icon，默认打开，拖拽等功能
    - [x] 统一查阅对话记录，并对数据进行标注
 
 <a href="#readme">
@@ -93,17 +101,17 @@ https://github.com/labring/FastGPT/assets/15308462/7d3a38df-eb0e-4388-9250-2409b
 
 - **⚡ 快速部署**
 
-  > Sealos 的服务器在国外，不需要额外处理网络问题，无需服务器、无需魔法、无需域名，支持高并发 & 动态伸缩。点击以下按钮即可一键部署 👇
+  > [Sealos](https://sealos.io) 的服务器在国外，不需要额外处理网络问题，无需服务器、无需魔法、无需域名，支持高并发 & 动态伸缩。点击以下按钮即可一键部署 👇
 
   [![](https://cdn.jsdelivr.us/gh/labring-actions/templates@main/Deploy-on-Sealos.svg)](https://cloud.sealos.io/?openapp=system-fastdeploy%3FtemplateName%3Dfastgpt)
 
   由于需要部署数据库，部署完后需要等待 2~4 分钟才能正常访问。默认用了最低配置，首次访问时会有些慢。
 
 * [快开始本地开发](https://doc.fastgpt.in/docs/development/intro/)
-* [部署 FastGPT](https://doc.fastgpt.in/docs/installation)
+* [部署 FastGPT](https://doc.fastgpt.in/docs/development/sealos)
 * [系统配置文件说明](https://doc.fastgpt.in/docs/development/configuration/)
-* [多模型配置](https://doc.fastgpt.in/docs/installation/one-api/)
-* [版本更新/升级介绍](https://doc.fastgpt.in/docs/installation/upgrading)
+* [多模型配置](https://doc.fastgpt.in/docs/development/one-api/)
+* [版本更新/升级介绍](https://doc.fastgpt.in/docs/development/upgrading)
 * [OpenAPI API 文档](https://doc.fastgpt.in/docs/development/openapi/)
 * [知识库结构详解](https://doc.fastgpt.in/docs/use-cases/datasetengine/)
 
@@ -142,7 +150,7 @@ https://github.com/labring/FastGPT/assets/15308462/7d3a38df-eb0e-4388-9250-2409b
     <img src="https://img.shields.io/badge/-返回顶部-7d09f1.svg" alt="#" align="right">
 </a>
 
-## 🤝 第三方生态
+## 🌿 第三方生态
 
 - [OnWeChat 个人微信/企微机器人](https://doc.fastgpt.in/docs/use-cases/onwechat/)
 
@@ -150,9 +158,51 @@ https://github.com/labring/FastGPT/assets/15308462/7d3a38df-eb0e-4388-9250-2409b
     <img src="https://img.shields.io/badge/-返回顶部-7d09f1.svg" alt="#" align="right">
 </a>
 
+## 🤝 参与贡献
+
+我们非常欢迎各种形式的贡献。如果你对贡献代码感兴趣，可以查看我们的 GitHub [Issues](https://github.com/labring/FastGPT/issues?q=is%3Aissue+is%3Aopen+sort%3Aupdated-desc)，大展身手，向我们展示你的奇思妙想。
+
+<a href="https://github.com/labring/FastGPT/graphs/contributors" target="_blank">
+  <table>
+    <tr>
+      <th colspan="2">
+        <br><img src="https://contrib.rocks/image?repo=labring/FastGPT"><br><br>
+      </th>
+    </tr>
+    <tr>
+      <td>
+        <picture>
+          <source media="(prefers-color-scheme: dark)" srcset="https://next.ossinsight.io/widgets/official/compose-org-active-contributors/thumbnail.png?activity=active&period=past_28_days&owner_id=102226726&repo_ids=605673387&image_size=2x3&color_scheme=dark">
+          <img alt="Active participants of labring - past 28 days" src="https://next.ossinsight.io/widgets/official/compose-org-active-contributors/thumbnail.png?activity=active&period=past_28_days&owner_id=102226726&repo_ids=605673387&image_size=2x3&color_scheme=light">
+        </picture>
+      </td>
+      <td rowspan="2">
+        <picture>
+          <source media="(prefers-color-scheme: dark)" srcset="https://next.ossinsight.io/widgets/official/compose-org-participants-growth/thumbnail.png?activity=new&period=past_28_days&owner_id=102226726&repo_ids=605673387&image_size=4x7&color_scheme=dark">
+          <img alt="New trends of labring" src="https://next.ossinsight.io/widgets/official/compose-org-participants-growth/thumbnail.png?activity=new&period=past_28_days&owner_id=102226726&repo_ids=605673387&image_size=4x7&color_scheme=light">
+        </picture>
+      </td>
+    </tr>
+    <tr>
+      <td>
+        <picture>
+          <source media="(prefers-color-scheme: dark)" srcset="https://next.ossinsight.io/widgets/official/compose-org-active-contributors/thumbnail.png?activity=new&period=past_28_days&owner_id=102226726&repo_ids=605673387&image_size=2x3&color_scheme=dark">
+          <img alt="New participants of labring - past 28 days" src="https://next.ossinsight.io/widgets/official/compose-org-active-contributors/thumbnail.png?activity=new&period=past_28_days&owner_id=102226726&repo_ids=605673387&image_size=2x3&color_scheme=light">
+        </picture>
+      </td>
+    </tr>
+  </table>
+</a>
+
 ## 🌟 Star History
 
-[![Star History Chart](https://api.star-history.com/svg?repos=labring/FastGPT&type=Date)](https://star-history.com/#labring/FastGPT&Date)
+<a href="https://github.com/labring/FastGPT/stargazers" target="_blank" style="display: block" align="center">
+  <picture>
+    <source media="(prefers-color-scheme: dark)" srcset="https://api.star-history.com/svg?repos=labring/FastGPT&type=Date&theme=dark" />
+    <source media="(prefers-color-scheme: light)" srcset="https://api.star-history.com/svg?repos=labring/FastGPT&type=Date" />
+    <img alt="Star History Chart" src="https://api.star-history.com/svg?repos=labring/FastGPT&type=Date" />
+  </picture>
+</a>
 
 <a href="#readme">
     <img src="https://img.shields.io/badge/-返回顶部-7d09f1.svg" alt="#" align="right">
@@ -165,4 +215,4 @@ https://github.com/labring/FastGPT/assets/15308462/7d3a38df-eb0e-4388-9250-2409b
 1. 允许作为后台服务直接商用，但不允许提供 SaaS 服务。
 2. 未经商业授权，任何形式的商用服务均需保留相关版权信息。
 3. 完整请查看 [FastGPT Open Source License](./LICENSE)
-4. 联系方式：yujinlong@sealos.io，[点击查看商业版定价策略](https://doc.fastgpt.in/docs/commercial)
+4. 联系方式：yujinlong@sealos.io，[点击查看商业版定价策略](https://doc.fastgpt.in/docs/commercial)

README_en.md

@@ -1,6 +1,6 @@
 <div align="center">
 
-<a href="https://fastgpt.run/"><img src="/.github/imgs/logo.svg" width="120" height="120" alt="fastgpt logo"></a>
+<a href="https://fastgpt.in/"><img src="/.github/imgs/logo.svg" width="120" height="120" alt="fastgpt logo"></a>
 
 # FastGPT
 
@@ -15,13 +15,13 @@ FastGPT is a knowledge-based Q&A system built on the LLM, offers out-of-the-box
 </div>
 
 <p align="center">
-  <a href="https://fastgpt.run/">
+  <a href="https://fastgpt.in/">
     <img height="21" src="https://img.shields.io/badge/在线使用-d4eaf7?style=flat-square&logo=spoj&logoColor=7d09f1" alt="cloud">
   </a>
-  <a href="https://doc.fastgpt.run/docs/intro">
+  <a href="https://doc.fastgpt.in/docs/intro">
     <img height="21" src="https://img.shields.io/badge/相关文档-7d09f1?style=flat-square" alt="document">
   </a>
-  <a href="https://doc.fastgpt.run/docs/development">
+  <a href="https://doc.fastgpt.in/docs/development">
     <img height="21" src="https://img.shields.io/badge/本地开发-%23d4eaf7?style=flat-square&logo=xcode&logoColor=7d09f1" alt="development">
   </a>
   <a href="/#-%E7%9B%B8%E5%85%B3%E9%A1%B9%E7%9B%AE">
@@ -36,7 +36,8 @@ https://github.com/labring/FastGPT/assets/15308462/7d3a38df-eb0e-4388-9250-2409b
 
 ## 🛸 Use Cloud Services
 
-[fastgpt.run](https://fastgpt.run/)
+Cloud: [fastgpt.in](https://fastgpt.in/)
+
 | | |
 | ---------------------------------- | ---------------------------------- |
 | ![Demo](./.github/imgs/intro1.png) | ![Demo](./.github/imgs/intro2.png) |
@@ -105,11 +106,11 @@ Project tech stack: NextJs + TS + ChakraUI + Mongo + Postgres (Vector plugin)
 
   Give it a 2-4 minute wait after deployment as it sets up the database. Initially, it might be a tad slow since we're using the basic settings.
 
-- [Getting Started with Local Development](https://doc.fastgpt.run/docs/development)
-- [Deploying FastGPT](https://doc.fastgpt.run/docs/installation)
-- [Guide on System Configs](https://doc.fastgpt.run/docs/installation/reference)
-- [Configuring Multiple Models](https://doc.fastgpt.run/docs/installation/reference/models)
-- [Version Updates & Upgrades](https://doc.fastgpt.run/docs/installation/upgrading)
+- [Getting Started with Local Development](https://doc.fastgpt.in/docs/development)
+- [Deploying FastGPT](https://doc.fastgpt.in/docs/installation)
+- [Guide on System Configs](https://doc.fastgpt.in/docs/installation/reference)
+- [Configuring Multiple Models](https://doc.fastgpt.in/docs/installation/reference/models)
+- [Version Updates & Upgrades](https://doc.fastgpt.in/docs/installation/upgrading)
 
 <!-- ## :point_right: RoadMap
 - [FastGPT RoadMap](https://kjqvjse66l.feishu.cn/docx/RVUxdqE2WolDYyxEKATcM0XXnte) -->
@@ -156,4 +157,10 @@ Project tech stack: NextJs + TS + ChakraUI + Mongo + Postgres (Vector plugin)
 
 ## 🌟 Star History
 
-[![Star History Chart](https://api.star-history.com/svg?repos=labring/FastGPT&type=Date)](https://star-history.com/#labring/FastGPT&Date)
+<a href="https://github.com/labring/FastGPT/stargazers" target="_blank" style="display: block" align="center">
+  <picture>
+    <source media="(prefers-color-scheme: dark)" srcset="https://api.star-history.com/svg?repos=labring/FastGPT&type=Date&theme=dark" />
+    <source media="(prefers-color-scheme: light)" srcset="https://api.star-history.com/svg?repos=labring/FastGPT&type=Date" />
+    <img alt="Star History Chart" src="https://api.star-history.com/svg?repos=labring/FastGPT&type=Date" />
+  </picture>
+</a>

README_ja.md

@@ -1,6 +1,6 @@
 <div align="center">
 
-<a href="https://fastgpt.run/"><img src="/.github/imgs/logo.svg" width="120" height="120" alt="fastgpt logo"></a>
+<a href="https://fastgpt.in/"><img src="/.github/imgs/logo.svg" width="120" height="120" alt="fastgpt logo"></a>
 
 # FastGPT
 
@@ -15,13 +15,13 @@ FastGPT は、LLM 上 に 構築 された 知識 ベースの Q&A システム
 </div>
 
 <p align="center">
-  <a href="https://fastgpt.run/">
+  <a href="https://fastgpt.in/">
     <img height="21" src="https://img.shields.io/badge/在线使用-d4eaf7?style=flat-square&logo=spoj&logoColor=7d09f1" alt="cloud">
   </a>
-  <a href="https://doc.fastgpt.run/docs/intro">
+  <a href="https://doc.fastgpt.in/docs/intro">
     <img height="21" src="https://img.shields.io/badge/相关文档-7d09f1?style=flat-square" alt="document">
   </a>
-  <a href="https://doc.fastgpt.run/docs/development">
+  <a href="https://doc.fastgpt.in/docs/development">
     <img height="21" src="https://img.shields.io/badge/本地开发-%23d4eaf7?style=flat-square&logo=xcode&logoColor=7d09f1" alt="development">
   </a>
   <a href="/#-%E7%9B%B8%E5%85%B3%E9%A1%B9%E7%9B%AE">
@@ -36,7 +36,8 @@ https://github.com/labring/FastGPT/assets/15308462/7d3a38df-eb0e-4388-9250-2409b
 
 ## 🛸 クラウドサービスの 利用
 
-[fastgpt.run](https://fastgpt.run/)
+[fastgpt.in](https://fastgpt.in/)
+
 | | |
 | ---------------------------------- | ---------------------------------- |
 | ![Demo](./.github/imgs/intro1.png) | ![Demo](./.github/imgs/intro2.png) |
@@ -97,11 +98,11 @@ https://github.com/labring/FastGPT/assets/15308462/7d3a38df-eb0e-4388-9250-2409b
 
   デプロイ 後、データベースをセットアップするので、2～4分待 ってください。基本設定 を 使 っているので、最初 は 少 し 遅 いかもしれません。
 
-- [ローカル 開発入門](https://doc.fastgpt.run/docs/development)
-- [FastGPT のデプロイ](https://doc.fastgpt.run/docs/installation)
-- [システム 設定 ガイド](https://doc.fastgpt.run/docs/installation/reference)
-- [複数 モデルの 設定](https://doc.fastgpt.run/docs/installation/reference/models)
-- [バージョン 更新 とアップグレード](https://doc.fastgpt.run/docs/installation/upgrading)
+- [ローカル 開発入門](https://doc.fastgpt.in/docs/development)
+- [FastGPT のデプロイ](https://doc.fastgpt.in/docs/installation)
+- [システム 設定 ガイド](https://doc.fastgpt.in/docs/installation/reference)
+- [複数 モデルの 設定](https://doc.fastgpt.in/docs/installation/reference/models)
+- [バージョン 更新 とアップグレード](https://doc.fastgpt.in/docs/installation/upgrading)
 
 <!-- ## :point_right: ロードマップ
 - [FastGPT ロードマップ](https://kjqvjse66l.feishu.cn/docx/RVUxdqE2WolDYyxEKATcM0XXnte) -->