gitea/FastGPT
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: add order:true to all create transactions (#3948)

Browse Source
This commit is contained in:
Finley Ge
2025-03-03 11:37:51 +08:00
committed by GitHub
parent 113e8f711f
commit 4bc7f21182
30 changed files with 425 additions and 352 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
plugins/webcrawler/SPIDER/src/specialHandlers/index.ts
View File

@@ -3,19 +3,24 @@ import { Page } from 'puppeteer';
export const handleSpecialWebsite = async (page: Page, url: string): Promise<string | null> => {
if (url.includes('blog.csdn.net')) {
await page.waitForSelector('article');
const content = await page.$eval('article', el => el.innerHTML);
const content = await page.$eval('article', (el) => el.innerHTML);
return content;
}
if (url.includes('zhuanlan.zhihu.com')) {
console.log('是知乎，需要点击按掉！');
console.log(await page.content());
if((await page.content()).includes('{"error":{"message":"您当前请求存在异常，暂时限制本次访问。如有疑问，您可以通过手机摇一摇或登录后私信知乎小管家反馈。","code":40362}}')) return null;
if (
(await page.content()).includes(
'{"error":{"message":"您当前请求存在异常，暂时限制本次访问。如有疑问，您可以通过手机摇一摇或登录后私信知乎小管家反馈。","code":40362}}'
)
)
return null;
await page.waitForSelector('button[aria-label="关闭"]');
await page.click('button[aria-label="关闭"]'); // 使用 aria-label 选择按钮
await page.waitForSelector('article');
const content = await page.$eval('article', el => el.innerHTML);
const content = await page.$eval('article', (el) => el.innerHTML);
return content;
}
// 可以添加更多特殊网站的处理逻辑
return null;
};
};