gitea/FastGPT
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: add order:true to all create transactions (#3948)

Browse Source
This commit is contained in:
Finley Ge
2025-03-03 11:37:51 +08:00
committed by GitHub
parent 113e8f711f
commit 4bc7f21182
30 changed files with 425 additions and 352 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
plugins/webcrawler/SPIDER/src/engines/searchxngEngine.ts
View File

@@ -6,9 +6,13 @@ dotenv.config();
const blacklistDomains = process.env.BLACKLIST ? JSON.parse(process.env.BLACKLIST) : [];
export const fetchSearchResults = async (query: string, pageCount: number, searchUrlBase: string, categories: string) => {
const MAX_PAGES = (pageCount / 10 +1) * 2+1; // 最多搜索的页面数
export const fetchSearchResults = async (
query: string,
pageCount: number,
searchUrlBase: string,
categories: string
) => {
const MAX_PAGES = (pageCount / 10 + 1) * 2 + 1; // 最多搜索的页面数
//如果searchUrlBase为空，返回空数组，pagecount是需要搜索结果的数量
if (!searchUrlBase) {
return { resultUrls: [], results: new Map() };
@@ -20,7 +24,9 @@ export const fetchSearchResults = async (query: string, pageCount: number, searc
let pageIndex = 0;
while (fetchedResultsCount < pageCount && pageIndex < MAX_PAGES) {
const searchUrl = new URL(`${searchUrlBase}?q=${encodeURIComponent(query)}&pageno=${pageIndex + 1}&format=json&categories=${categories}`);
const searchUrl = new URL(
`${searchUrlBase}?q=${encodeURIComponent(query)}&pageno=${pageIndex + 1}&format=json&categories=${categories}`
);
console.log(`Fetching page ${pageIndex + 1} from SearchXNG: ${searchUrl.toString()}`);
const response = await axios.get(searchUrl.toString());
const jsonResults = response.data.results;
@@ -28,7 +34,10 @@ export const fetchSearchResults = async (query: string, pageCount: number, searc
for (let index = 0; index < jsonResults.length; index++) {
const result = jsonResults[index];
const resultDomain = new URL(result.url).hostname;
if (blacklistDomains.some((domain: string) => resultDomain.endsWith(domain)) || resultDomain.includes('zhihu')) {
if (
blacklistDomains.some((domain: string) => resultDomain.endsWith(domain)) ||
resultDomain.includes('zhihu')
) {
continue;
}
resultUrls.push(result.url);
@@ -52,4 +61,4 @@ export const fetchSearchResults = async (query: string, pageCount: number, searc
}
return { resultUrls, results };
};
};