External dataset (#1497)

* perf: read rawText and chunk code * perf: read raw text * perf: read rawtext * perf: token count * log
2024-05-16 11:47:53 +08:00
parent d5073f98ab
commit c6d9b15897
36 changed files with 531 additions and 267 deletions
--- a/packages/service/common/string/cheerio.ts
+++ b/packages/service/common/string/cheerio.ts
@@ -77,9 +77,8 @@ export const urlsFetch = async ({
          $,
          selector
        });
-        console.log('html====', html);
+
        const md = await htmlToMarkdown(html);
-        console.log('html====', md);

        return {
          url,