fix: add order:true to all create transactions (#3948)

2025-03-03 11:37:51 +08:00
parent 113e8f711f
commit 4bc7f21182
30 changed files with 425 additions and 352 deletions
--- a/plugins/webcrawler/SPIDER/src/controllers/quickfetchController.ts
+++ b/plugins/webcrawler/SPIDER/src/controllers/quickfetchController.ts
@@ -5,56 +5,56 @@ import dotenv from 'dotenv';
 dotenv.config();

 const userAgents = [
-    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3',
-    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0.3 Safari/605.1.15',
-    'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:89.0) Gecko/20100101 Firefox/89.0'
+  'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3',
+  'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0.3 Safari/605.1.15',
+  'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:89.0) Gecko/20100101 Firefox/89.0'
 ];

 export const quickFetch = async (req: Request, res: Response): Promise<void> => {
-    const { url } = req.query;
+  const { url } = req.query;

-    if (!url) {
-        res.status(400).json({
-            status: 400,
-            error: {
-                code: "MISSING_PARAM",
-                message: "缺少必要参数: url"
-            }
-        });
-        return;
-    }
+  if (!url) {
+    res.status(400).json({
+      status: 400,
+      error: {
+        code: 'MISSING_PARAM',
+        message: '缺少必要参数: url'
+      }
+    });
+    return;
+  }

-    try {
-        const response = await fetch(url as string, {
-            headers: {
-                'User-Agent': userAgents[Math.floor(Math.random() * userAgents.length)],
-                'Referer': 'https://www.google.com/',
-                'Accept-Language': 'en-US,en;q=0.9',
-                'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
-                'Connection': 'keep-alive',
-                'Cache-Control': 'no-cache'
-            }
-        });
-        if (!response.ok) {
-            throw new Error(`HTTP error! status: ${response.status}`);
-        }
-        const data = await response.text();
-        res.status(200).json({
-            status: 200,
-            data: {
-                content: data
-            }
-        });
-    } catch (error) {
-        console.error('Error fetching the page:', error);
-        res.status(500).json({
-            status: 500,
-            error: {
-                code: "INTERNAL_SERVER_ERROR",
-                message: "发生错误"
-            }
-        });
+  try {
+    const response = await fetch(url as string, {
+      headers: {
+        'User-Agent': userAgents[Math.floor(Math.random() * userAgents.length)],
+        Referer: 'https://www.google.com/',
+        'Accept-Language': 'en-US,en;q=0.9',
+        Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
+        Connection: 'keep-alive',
+        'Cache-Control': 'no-cache'
+      }
+    });
+    if (!response.ok) {
+      throw new Error(`HTTP error! status: ${response.status}`);
    }
+    const data = await response.text();
+    res.status(200).json({
+      status: 200,
+      data: {
+        content: data
+      }
+    });
+  } catch (error) {
+    console.error('Error fetching the page:', error);
+    res.status(500).json({
+      status: 500,
+      error: {
+        code: 'INTERNAL_SERVER_ERROR',
+        message: '发生错误'
+      }
+    });
+  }
 };

-export default { quickFetch };
+export default { quickFetch };
--- a/plugins/webcrawler/SPIDER/src/controllers/readController.ts
+++ b/plugins/webcrawler/SPIDER/src/controllers/readController.ts
@@ -16,16 +16,16 @@ const blacklistDomains = process.env.BLACKLIST ? JSON.parse(process.env.BLACKLIS

 export const readPage = async (req: Request, res: Response): Promise<void> => {
  const { queryUrl } = req.query;
-  console.log("-------");
+  console.log('-------');
  console.log(queryUrl);
-  console.log("-------");
+  console.log('-------');

  if (!queryUrl) {
    res.status(400).json({
      status: 400,
      error: {
-        code: "MISSING_PARAM",
-        message: "缺少必要参数: queryUrl"
+        code: 'MISSING_PARAM',
+        message: '缺少必要参数: queryUrl'
      }
    });
    return;
@@ -36,8 +36,8 @@ export const readPage = async (req: Request, res: Response): Promise<void> => {
    res.status(403).json({
      status: 403,
      error: {
-        code: "BLACKLISTED_DOMAIN",
-        message: "该域名受到保护中"
+        code: 'BLACKLISTED_DOMAIN',
+        message: '该域名受到保护中'
      }
    });
    return;
@@ -46,11 +46,14 @@ export const readPage = async (req: Request, res: Response): Promise<void> => {
  try {
    const response = await fetch(queryUrl as string, {
      headers: {
-        'User-Agent': new UserAgent({ deviceCategory: 'desktop', platform: 'Linux x86_64' }).toString(),
-        'Referer': 'https://www.google.com/',
+        'User-Agent': new UserAgent({
+          deviceCategory: 'desktop',
+          platform: 'Linux x86_64'
+        }).toString(),
+        Referer: 'https://www.google.com/',
        'Accept-Language': 'en-US,en;q=0.9',
-        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
-        'Connection': 'keep-alive',
+        Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
+        Connection: 'keep-alive',
        'Cache-Control': 'no-cache'
      }
    });
@@ -69,7 +72,7 @@ export const readPage = async (req: Request, res: Response): Promise<void> => {
      });

      await updateCacheAsync(queryUrl as string, cleanedContent || '');
-      console.log("Page read successfully");
+      console.log('Page read successfully');
      return;
    } else {
      throw new Error(`HTTP error! status: ${response.status}`);
@@ -79,23 +82,26 @@ export const readPage = async (req: Request, res: Response): Promise<void> => {
  }

  try {
-    const browser = await puppeteer.launch({ 
-      ignoreDefaultArgs: ["--enable-automation"],
-      headless: true,  
-      executablePath: "/usr/bin/chromium",  // 明确指定 Chromium 路径
+    const browser = await puppeteer.launch({
+      ignoreDefaultArgs: ['--enable-automation'],
+      headless: true,
+      executablePath: '/usr/bin/chromium', // 明确指定 Chromium 路径
      pipe: true,
      args: [
        '--no-sandbox',
        '--disable-setuid-sandbox',
        '--disable-dev-shm-usage',
-        '--disable-gpu',
-       // '--single-process' 
+        '--disable-gpu'
+        // '--single-process'
      ]
    });
    const page = await browser.newPage();

    // 检测是否需要特殊处理
-    if (typeof queryUrl === 'string' && detectWebsites.some(website => queryUrl.includes(website))) {
+    if (
+      typeof queryUrl === 'string' &&
+      detectWebsites.some((website) => queryUrl.includes(website))
+    ) {
      await setupPage(page);
    } else {
      const userAgent = new UserAgent({ deviceCategory: 'desktop', platform: 'Linux x86_64' });
@@ -128,15 +134,15 @@ export const readPage = async (req: Request, res: Response): Promise<void> => {
    });

    await updateCacheAsync(queryUrl as string, cleanedContent || '');
-    console.log("Page read successfully");
+    console.log('Page read successfully');
  } catch (error) {
    console.error(error);
    res.status(500).json({
      status: 500,
      error: {
-        code: "INTERNAL_SERVER_ERROR",
-        message: "读取页面时发生内部服务器错误"
+        code: 'INTERNAL_SERVER_ERROR',
+        message: '读取页面时发生内部服务器错误'
      }
    });
  }
-};
+};
--- a/plugins/webcrawler/SPIDER/src/controllers/searchController.ts
+++ b/plugins/webcrawler/SPIDER/src/controllers/searchController.ts
@@ -12,15 +12,21 @@ const detectWebsites = process.env.DETECT_WEBSITES?.split(',') || [];
 const maxConcurrency = parseInt(process.env.MAX_CONCURRENCY || '10', 10);

 export const search = async (req: Request, res: Response): Promise<void> => {
-  const { query, pageCount = 10, needDetails = 'false', engine = 'baidu', categories = 'general' } = req.query;
-  const needDetailsBool = (needDetails === 'true');
+  const {
+    query,
+    pageCount = 10,
+    needDetails = 'false',
+    engine = 'baidu',
+    categories = 'general'
+  } = req.query;
+  const needDetailsBool = needDetails === 'true';

  if (!query) {
    res.status(400).json({
      status: 400,
      error: {
-        code: "MISSING_PARAM",
-        message: "缺少必要参数: query"
+        code: 'MISSING_PARAM',
+        message: '缺少必要参数: query'
      }
    });
    return;
@@ -28,24 +34,29 @@ export const search = async (req: Request, res: Response): Promise<void> => {
  let fetchSearchResults;
  let searchUrlBase;
  try {
-      if (engine === 'baidu') {
-        fetchSearchResults = fetchBaiduResults;
-        searchUrlBase = process.env.ENGINE_BAIDUURL;
-      } else if (engine === 'searchxng') {
-        fetchSearchResults = fetchSearchxngResults;
-        searchUrlBase = process.env.ENGINE_SEARCHXNGURL;
-      } else {
-        res.status(400).json({
-          status: 400,
-          error: {
-            code: "INVALID_ENGINE",
-            message: "无效的搜索引擎"
-          }
-        });
-        return;
-      }
+    if (engine === 'baidu') {
+      fetchSearchResults = fetchBaiduResults;
+      searchUrlBase = process.env.ENGINE_BAIDUURL;
+    } else if (engine === 'searchxng') {
+      fetchSearchResults = fetchSearchxngResults;
+      searchUrlBase = process.env.ENGINE_SEARCHXNGURL;
+    } else {
+      res.status(400).json({
+        status: 400,
+        error: {
+          code: 'INVALID_ENGINE',
+          message: '无效的搜索引擎'
+        }
+      });
+      return;
+    }

-    const { resultUrls, results } = await fetchSearchResults(query as string, Number(pageCount), searchUrlBase || '', categories as string);
+    const { resultUrls, results } = await fetchSearchResults(
+      query as string,
+      Number(pageCount),
+      searchUrlBase || '',
+      categories as string
+    );

    //如果返回值为空，返回空数组
    if (results.size === 0) {
@@ -79,20 +90,27 @@ export const search = async (req: Request, res: Response): Promise<void> => {
        concurrency: Cluster.CONCURRENCY_CONTEXT,
        maxConcurrency: maxConcurrency,
        puppeteerOptions: {
-          ignoreDefaultArgs: ["--enable-automation"],
-          headless: "true",
-          executablePath: "/usr/bin/chromium",  // 明确指定 Chromium 路径
+          ignoreDefaultArgs: ['--enable-automation'],
+          headless: 'true',
+          executablePath: '/usr/bin/chromium', // 明确指定 Chromium 路径
          pipe: true,
          args: [
            '--no-sandbox',
            '--disable-setuid-sandbox',
            '--disable-dev-shm-usage',
-            '--disable-gpu',
+            '--disable-gpu'
          ]
        }
      });

-      const sortedResults = await performDeepSearch(clusterInstance, resultUrls, results, strategies, detectWebsites, Number(pageCount));
+      const sortedResults = await performDeepSearch(
+        clusterInstance,
+        resultUrls,
+        results,
+        strategies,
+        detectWebsites,
+        Number(pageCount)
+      );
      res.status(200).json({
        status: 200,
        data: {
@@ -104,11 +122,11 @@ export const search = async (req: Request, res: Response): Promise<void> => {
    res.status(500).json({
      status: 500,
      error: {
-        code: "INTERNAL_SERVER_ERROR",
-        message: "发生错误"
+        code: 'INTERNAL_SERVER_ERROR',
+        message: '发生错误'
      }
    });
  }
 };

-export default { search };
+export default { search };