feat: 根据url获取网站文本

src/pages/api/model/data/exportModelData.ts

@@ -4,7 +4,6 @@ import { connectToDatabase } from '@/service/mongo';
 import { authToken } from '@/service/utils/tools';
 import { connectRedis } from '@/service/redis';
 import { VecModelDataIdx } from '@/constants/redis';
-import { BufferToVector } from '@/utils/tools';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
   try {

src/pages/api/model/data/fetchingUrlData.ts

@@ -0,0 +1,36 @@
+import type { NextApiRequest, NextApiResponse } from 'next';
+import { jsonRes } from '@/service/response';
+import { connectToDatabase } from '@/service/mongo';
+import { authToken } from '@/service/utils/tools';
+import axios from 'axios';
+import { httpsAgent } from '@/service/utils/tools';
+
+/**
+ * 读取网站的内容
+ */
+export default async function handler(req: NextApiRequest, res: NextApiResponse) {
+  try {
+    const { url } = req.body as { url: string };
+    if (!url) {
+      throw new Error('缺少 url');
+    }
+    await connectToDatabase();
+
+    const { authorization } = req.headers;
+
+    await authToken(authorization);
+
+    const data = await axios
+      .get(url, {
+        httpsAgent
+      })
+      .then((res) => res.data as string);
+
+    jsonRes(res, { data });
+  } catch (err) {
+    jsonRes(res, {
+      code: 500,
+      error: err
+    });
+  }
+}

src/pages/api/model/data/getSplitData.ts

@@ -24,7 +24,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
     });
 
     jsonRes(res, {
-      data
+      data: data.map((item) => item.textList).flat().length
     });
   } catch (err) {
     jsonRes(res, {