bug fix;

package-lock.json

@@ -10,6 +10,7 @@
       "dependencies": {
         "axios": "^1.7.9",
         "bcryptjs": "^2.4.3",
+        "cheerio": "^1.1.0",
         "crypto": "^1.0.1",
         "express": "^4.18.2",
         "file": "^0.2.2",
@@ -19,12 +20,14 @@
         "multer": "^1.4.5-lts.1",
         "mysql": "^2.18.1",
         "pg": "^8.13.1",
+        "playwright": "^1.53.0",
         "querystring": "^0.2.1",
         "r2r-js": "^0.4.34",
         "r2r-webdev-template": "file:",
         "react": "^18",
         "react-dom": "^18",
-        "request": "^2.88.2"
+        "request": "^2.88.2",
+        "turndown": "^7.2.0"
       },
       "devDependencies": {
         "nodemon": "^3.1.9",
@@ -724,6 +727,12 @@
         "@jridgewell/sourcemap-codec": "^1.4.14"
       }
     },
+    "node_modules/@mixmark-io/domino": {
+      "version": "2.2.0",
+      "resolved": "https://registry.npmjs.org/@mixmark-io/domino/-/domino-2.2.0.tgz",
+      "integrity": "sha512-Y28PR25bHXUg88kCV7nivXrP2Nj2RueZ3/l/jdx6J9f8J4nsEGcgX0Qe6lt7Pa+J79+kPiJU3LguR6O/6zrLOw==",
+      "license": "BSD-2-Clause"
+    },
     "node_modules/@rrweb/types": {
       "version": "2.0.0-alpha.17",
       "resolved": "https://registry.npmmirror.com/@rrweb/types/-/types-2.0.0-alpha.17.tgz",
@@ -1077,6 +1086,12 @@
         "npm": "1.2.8000 || >= 1.4.16"
       }
     },
+    "node_modules/boolbase": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz",
+      "integrity": "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==",
+      "license": "ISC"
+    },
     "node_modules/brace-expansion": {
       "version": "1.1.11",
       "resolved": "https://registry.npmmirror.com/brace-expansion/-/brace-expansion-1.1.11.tgz",
@@ -1258,6 +1273,48 @@
         "node": ">=8"
       }
     },
+    "node_modules/cheerio": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/cheerio/-/cheerio-1.1.0.tgz",
+      "integrity": "sha512-+0hMx9eYhJvWbgpKV9hN7jg0JcwydpopZE4hgi+KvQtByZXPp04NiCWU0LzcAbP63abZckIHkTQaXVF52mX3xQ==",
+      "license": "MIT",
+      "dependencies": {
+        "cheerio-select": "^2.1.0",
+        "dom-serializer": "^2.0.0",
+        "domhandler": "^5.0.3",
+        "domutils": "^3.2.2",
+        "encoding-sniffer": "^0.2.0",
+        "htmlparser2": "^10.0.0",
+        "parse5": "^7.3.0",
+        "parse5-htmlparser2-tree-adapter": "^7.1.0",
+        "parse5-parser-stream": "^7.1.2",
+        "undici": "^7.10.0",
+        "whatwg-mimetype": "^4.0.0"
+      },
+      "engines": {
+        "node": ">=18.17"
+      },
+      "funding": {
+        "url": "https://github.com/cheeriojs/cheerio?sponsor=1"
+      }
+    },
+    "node_modules/cheerio-select": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/cheerio-select/-/cheerio-select-2.1.0.tgz",
+      "integrity": "sha512-9v9kG0LvzrlcungtnJtpGNxY+fzECQKhK4EGJX2vByejiMX84MFNQw4UxPJl3bFbTMw+Dfs37XaIkCwTZfLh4g==",
+      "license": "BSD-2-Clause",
+      "dependencies": {
+        "boolbase": "^1.0.0",
+        "css-select": "^5.1.0",
+        "css-what": "^6.1.0",
+        "domelementtype": "^2.3.0",
+        "domhandler": "^5.0.3",
+        "domutils": "^3.0.1"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/fb55"
+      }
+    },
     "node_modules/chokidar": {
       "version": "3.6.0",
       "resolved": "https://registry.npmmirror.com/chokidar/-/chokidar-3.6.0.tgz",
@@ -1438,6 +1495,34 @@
       "integrity": "sha512-VxBKmeNcqQdiUQUW2Tzq0t377b54N2bMtXO/qiLa+6eRRmmC4qT3D4OnTGoT/U6O9aklQ/jTwbOtRMTTY8G0Ig==",
       "license": "ISC"
     },
+    "node_modules/css-select": {
+      "version": "5.1.0",
+      "resolved": "https://registry.npmjs.org/css-select/-/css-select-5.1.0.tgz",
+      "integrity": "sha512-nwoRF1rvRRnnCqqY7updORDsuqKzqYJ28+oSMaJMMgOauh3fvwHqMS7EZpIPqK8GL+g9mKxF1vP/ZjSeNjEVHg==",
+      "license": "BSD-2-Clause",
+      "dependencies": {
+        "boolbase": "^1.0.0",
+        "css-what": "^6.1.0",
+        "domhandler": "^5.0.2",
+        "domutils": "^3.0.1",
+        "nth-check": "^2.0.1"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/fb55"
+      }
+    },
+    "node_modules/css-what": {
+      "version": "6.1.0",
+      "resolved": "https://registry.npmjs.org/css-what/-/css-what-6.1.0.tgz",
+      "integrity": "sha512-HTUrgRJ7r4dsZKU6GjmpfRK1O76h97Z8MfS1G0FozR+oF2kG6Vfe8JE6zwrkbxigziPHinCJ+gCPjA9EaBDtRw==",
+      "license": "BSD-2-Clause",
+      "engines": {
+        "node": ">= 6"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/fb55"
+      }
+    },
     "node_modules/dashdash": {
       "version": "1.14.1",
       "resolved": "https://registry.npmmirror.com/dashdash/-/dashdash-1.14.1.tgz",
@@ -1502,6 +1587,61 @@
         "node": "^14.15.0 || ^16.10.0 || >=18.0.0"
       }
     },
+    "node_modules/dom-serializer": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-2.0.0.tgz",
+      "integrity": "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==",
+      "license": "MIT",
+      "dependencies": {
+        "domelementtype": "^2.3.0",
+        "domhandler": "^5.0.2",
+        "entities": "^4.2.0"
+      },
+      "funding": {
+        "url": "https://github.com/cheeriojs/dom-serializer?sponsor=1"
+      }
+    },
+    "node_modules/domelementtype": {
+      "version": "2.3.0",
+      "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-2.3.0.tgz",
+      "integrity": "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==",
+      "funding": [
+        {
+          "type": "github",
+          "url": "https://github.com/sponsors/fb55"
+        }
+      ],
+      "license": "BSD-2-Clause"
+    },
+    "node_modules/domhandler": {
+      "version": "5.0.3",
+      "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-5.0.3.tgz",
+      "integrity": "sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==",
+      "license": "BSD-2-Clause",
+      "dependencies": {
+        "domelementtype": "^2.3.0"
+      },
+      "engines": {
+        "node": ">= 4"
+      },
+      "funding": {
+        "url": "https://github.com/fb55/domhandler?sponsor=1"
+      }
+    },
+    "node_modules/domutils": {
+      "version": "3.2.2",
+      "resolved": "https://registry.npmjs.org/domutils/-/domutils-3.2.2.tgz",
+      "integrity": "sha512-6kZKyUajlDuqlHKVX1w7gyslj9MPIXzIFiz/rGu35uC1wMi+kMhQwGhl4lt9unC9Vb9INnY9Z3/ZA3+FhASLaw==",
+      "license": "BSD-2-Clause",
+      "dependencies": {
+        "dom-serializer": "^2.0.0",
+        "domelementtype": "^2.3.0",
+        "domhandler": "^5.0.3"
+      },
+      "funding": {
+        "url": "https://github.com/fb55/domutils?sponsor=1"
+      }
+    },
     "node_modules/dunder-proto": {
       "version": "1.0.1",
       "resolved": "https://registry.npmmirror.com/dunder-proto/-/dunder-proto-1.0.1.tgz",
@@ -1547,6 +1687,43 @@
         "node": ">= 0.8"
       }
     },
+    "node_modules/encoding-sniffer": {
+      "version": "0.2.1",
+      "resolved": "https://registry.npmjs.org/encoding-sniffer/-/encoding-sniffer-0.2.1.tgz",
+      "integrity": "sha512-5gvq20T6vfpekVtqrYQsSCFZ1wEg5+wW0/QaZMWkFr6BqD3NfKs0rLCx4rrVlSWJeZb5NBJgVLswK/w2MWU+Gw==",
+      "license": "MIT",
+      "dependencies": {
+        "iconv-lite": "^0.6.3",
+        "whatwg-encoding": "^3.1.1"
+      },
+      "funding": {
+        "url": "https://github.com/fb55/encoding-sniffer?sponsor=1"
+      }
+    },
+    "node_modules/encoding-sniffer/node_modules/iconv-lite": {
+      "version": "0.6.3",
+      "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz",
+      "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==",
+      "license": "MIT",
+      "dependencies": {
+        "safer-buffer": ">= 2.1.2 < 3.0.0"
+      },
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
+    "node_modules/entities": {
+      "version": "4.5.0",
+      "resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz",
+      "integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==",
+      "license": "BSD-2-Clause",
+      "engines": {
+        "node": ">=0.12"
+      },
+      "funding": {
+        "url": "https://github.com/fb55/entities?sponsor=1"
+      }
+    },
     "node_modules/es-define-property": {
       "version": "1.0.1",
       "resolved": "https://registry.npmmirror.com/es-define-property/-/es-define-property-1.0.1.tgz",
@@ -1859,6 +2036,20 @@
       "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==",
       "license": "ISC"
     },
+    "node_modules/fsevents": {
+      "version": "2.3.2",
+      "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz",
+      "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==",
+      "hasInstallScript": true,
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "darwin"
+      ],
+      "engines": {
+        "node": "^8.16.0 || ^10.6.0 || >=11.0.0"
+      }
+    },
     "node_modules/function-bind": {
       "version": "1.1.2",
       "resolved": "https://registry.npmmirror.com/function-bind/-/function-bind-1.1.2.tgz",
@@ -2047,6 +2238,37 @@
         "node": ">= 0.4"
       }
     },
+    "node_modules/htmlparser2": {
+      "version": "10.0.0",
+      "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-10.0.0.tgz",
+      "integrity": "sha512-TwAZM+zE5Tq3lrEHvOlvwgj1XLWQCtaaibSN11Q+gGBAS7Y1uZSWwXXRe4iF6OXnaq1riyQAPFOBtYc77Mxq0g==",
+      "funding": [
+        "https://github.com/fb55/htmlparser2?sponsor=1",
+        {
+          "type": "github",
+          "url": "https://github.com/sponsors/fb55"
+        }
+      ],
+      "license": "MIT",
+      "dependencies": {
+        "domelementtype": "^2.3.0",
+        "domhandler": "^5.0.3",
+        "domutils": "^3.2.1",
+        "entities": "^6.0.0"
+      }
+    },
+    "node_modules/htmlparser2/node_modules/entities": {
+      "version": "6.0.1",
+      "resolved": "https://registry.npmjs.org/entities/-/entities-6.0.1.tgz",
+      "integrity": "sha512-aN97NXWF6AWBTahfVOIrB/NShkzi5H7F9r1s9mD3cDj4Ko5f2qhhVoYMibXF7GlLveb/D2ioWay8lxI97Ven3g==",
+      "license": "BSD-2-Clause",
+      "engines": {
+        "node": ">=0.12"
+      },
+      "funding": {
+        "url": "https://github.com/fb55/entities?sponsor=1"
+      }
+    },
     "node_modules/http-errors": {
       "version": "2.0.0",
       "resolved": "https://registry.npmmirror.com/http-errors/-/http-errors-2.0.0.tgz",
@@ -2850,6 +3072,18 @@
         "node": ">=0.10.0"
       }
     },
+    "node_modules/nth-check": {
+      "version": "2.1.1",
+      "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-2.1.1.tgz",
+      "integrity": "sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w==",
+      "license": "BSD-2-Clause",
+      "dependencies": {
+        "boolbase": "^1.0.0"
+      },
+      "funding": {
+        "url": "https://github.com/fb55/nth-check?sponsor=1"
+      }
+    },
     "node_modules/oauth-sign": {
       "version": "0.9.0",
       "resolved": "https://registry.npmmirror.com/oauth-sign/-/oauth-sign-0.9.0.tgz",
@@ -2946,6 +3180,55 @@
         "node": ">=6"
       }
     },
+    "node_modules/parse5": {
+      "version": "7.3.0",
+      "resolved": "https://registry.npmjs.org/parse5/-/parse5-7.3.0.tgz",
+      "integrity": "sha512-IInvU7fabl34qmi9gY8XOVxhYyMyuH2xUNpb2q8/Y+7552KlejkRvqvD19nMoUW/uQGGbqNpA6Tufu5FL5BZgw==",
+      "license": "MIT",
+      "dependencies": {
+        "entities": "^6.0.0"
+      },
+      "funding": {
+        "url": "https://github.com/inikulin/parse5?sponsor=1"
+      }
+    },
+    "node_modules/parse5-htmlparser2-tree-adapter": {
+      "version": "7.1.0",
+      "resolved": "https://registry.npmjs.org/parse5-htmlparser2-tree-adapter/-/parse5-htmlparser2-tree-adapter-7.1.0.tgz",
+      "integrity": "sha512-ruw5xyKs6lrpo9x9rCZqZZnIUntICjQAd0Wsmp396Ul9lN/h+ifgVV1x1gZHi8euej6wTfpqX8j+BFQxF0NS/g==",
+      "license": "MIT",
+      "dependencies": {
+        "domhandler": "^5.0.3",
+        "parse5": "^7.0.0"
+      },
+      "funding": {
+        "url": "https://github.com/inikulin/parse5?sponsor=1"
+      }
+    },
+    "node_modules/parse5-parser-stream": {
+      "version": "7.1.2",
+      "resolved": "https://registry.npmjs.org/parse5-parser-stream/-/parse5-parser-stream-7.1.2.tgz",
+      "integrity": "sha512-JyeQc9iwFLn5TbvvqACIF/VXG6abODeB3Fwmv/TGdLk2LfbWkaySGY72at4+Ty7EkPZj854u4CrICqNk2qIbow==",
+      "license": "MIT",
+      "dependencies": {
+        "parse5": "^7.0.0"
+      },
+      "funding": {
+        "url": "https://github.com/inikulin/parse5?sponsor=1"
+      }
+    },
+    "node_modules/parse5/node_modules/entities": {
+      "version": "6.0.1",
+      "resolved": "https://registry.npmjs.org/entities/-/entities-6.0.1.tgz",
+      "integrity": "sha512-aN97NXWF6AWBTahfVOIrB/NShkzi5H7F9r1s9mD3cDj4Ko5f2qhhVoYMibXF7GlLveb/D2ioWay8lxI97Ven3g==",
+      "license": "BSD-2-Clause",
+      "engines": {
+        "node": ">=0.12"
+      },
+      "funding": {
+        "url": "https://github.com/fb55/entities?sponsor=1"
+      }
+    },
     "node_modules/parseurl": {
       "version": "1.3.3",
       "resolved": "https://registry.npmmirror.com/parseurl/-/parseurl-1.3.3.tgz",
@@ -3101,6 +3384,36 @@
         "node": ">= 6"
       }
     },
+    "node_modules/playwright": {
+      "version": "1.53.0",
+      "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.53.0.tgz",
+      "integrity": "sha512-ghGNnIEYZC4E+YtclRn4/p6oYbdPiASELBIYkBXfaTVKreQUYbMUYQDwS12a8F0/HtIjr/CkGjtwABeFPGcS4Q==",
+      "license": "Apache-2.0",
+      "dependencies": {
+        "playwright-core": "1.53.0"
+      },
+      "bin": {
+        "playwright": "cli.js"
+      },
+      "engines": {
+        "node": ">=18"
+      },
+      "optionalDependencies": {
+        "fsevents": "2.3.2"
+      }
+    },
+    "node_modules/playwright-core": {
+      "version": "1.53.0",
+      "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.53.0.tgz",
+      "integrity": "sha512-mGLg8m0pm4+mmtB7M89Xw/GSqoNC+twivl8ITteqvAndachozYe2ZA7srU6uleV1vEdAHYqjq+SV8SNxRRFYBw==",
+      "license": "Apache-2.0",
+      "bin": {
+        "playwright-core": "cli.js"
+      },
+      "engines": {
+        "node": ">=18"
+      }
+    },
     "node_modules/postcss": {
       "version": "8.5.1",
       "resolved": "https://registry.npmmirror.com/postcss/-/postcss-8.5.1.tgz",
@@ -3909,6 +4222,15 @@
       ],
       "license": "MIT"
     },
+    "node_modules/turndown": {
+      "version": "7.2.0",
+      "resolved": "https://registry.npmjs.org/turndown/-/turndown-7.2.0.tgz",
+      "integrity": "sha512-eCZGBN4nNNqM9Owkv9HAtWRYfLA4h909E/WGAWWBpmB275ehNhZyk87/Tpvjbp0jjNl9XwCsbe6bm6CqFsgD+A==",
+      "license": "MIT",
+      "dependencies": {
+        "@mixmark-io/domino": "^2.2.0"
+      }
+    },
     "node_modules/tweetnacl": {
       "version": "0.14.5",
       "resolved": "https://registry.npmmirror.com/tweetnacl/-/tweetnacl-0.14.5.tgz",
@@ -3950,6 +4272,15 @@
       "dev": true,
       "license": "MIT"
     },
+    "node_modules/undici": {
+      "version": "7.10.0",
+      "resolved": "https://registry.npmjs.org/undici/-/undici-7.10.0.tgz",
+      "integrity": "sha512-u5otvFBOBZvmdjWLVW+5DAc9Nkq8f24g0O9oY7qw2JVIF1VocIFoyz9JFkuVOS2j41AufeO0xnlweJ2RLT8nGw==",
+      "license": "MIT",
+      "engines": {
+        "node": ">=20.18.1"
+      }
+    },
     "node_modules/undici-types": {
       "version": "6.20.0",
       "resolved": "https://registry.npmmirror.com/undici-types/-/undici-types-6.20.0.tgz",
@@ -4079,6 +4410,39 @@
         "makeerror": "1.0.12"
       }
     },
+    "node_modules/whatwg-encoding": {
+      "version": "3.1.1",
+      "resolved": "https://registry.npmjs.org/whatwg-encoding/-/whatwg-encoding-3.1.1.tgz",
+      "integrity": "sha512-6qN4hJdMwfYBtE3YBTTHhoeuUrDBPZmbQaxWAqSALV/MeEnR5z1xd8UKud2RAkFoPkmB+hli1TZSnyi84xz1vQ==",
+      "license": "MIT",
+      "dependencies": {
+        "iconv-lite": "0.6.3"
+      },
+      "engines": {
+        "node": ">=18"
+      }
+    },
+    "node_modules/whatwg-encoding/node_modules/iconv-lite": {
+      "version": "0.6.3",
+      "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz",
+      "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==",
+      "license": "MIT",
+      "dependencies": {
+        "safer-buffer": ">= 2.1.2 < 3.0.0"
+      },
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
+    "node_modules/whatwg-mimetype": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/whatwg-mimetype/-/whatwg-mimetype-4.0.0.tgz",
+      "integrity": "sha512-QaKxh0eNIi2mE9p2vEdzfagOKHCcj1pJ56EEHGQOVxp8r9/iszLUUV7v89x9O1p/T+NlTM5W7jW6+cz4Fq1YVg==",
+      "license": "MIT",
+      "engines": {
+        "node": ">=18"
+      }
+    },
     "node_modules/wrappy": {
       "version": "1.0.2",
       "resolved": "https://registry.npmmirror.com/wrappy/-/wrappy-1.0.2.tgz",
@@ -4858,6 +5222,11 @@
         "unpipe": "1.0.0"
       }
     },
+    "boolbase": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz",
+      "integrity": "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww=="
+    },
     "brace-expansion": {
       "version": "1.1.11",
       "resolved": "https://registry.npmmirror.com/brace-expansion/-/brace-expansion-1.1.11.tgz",
@@ -4964,6 +5333,37 @@
         }
       }
     },
+    "cheerio": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/cheerio/-/cheerio-1.1.0.tgz",
+      "integrity": "sha512-+0hMx9eYhJvWbgpKV9hN7jg0JcwydpopZE4hgi+KvQtByZXPp04NiCWU0LzcAbP63abZckIHkTQaXVF52mX3xQ==",
+      "requires": {
+        "cheerio-select": "^2.1.0",
+        "dom-serializer": "^2.0.0",
+        "domhandler": "^5.0.3",
+        "domutils": "^3.2.2",
+        "encoding-sniffer": "^0.2.0",
+        "htmlparser2": "^10.0.0",
+        "parse5": "^7.3.0",
+        "parse5-htmlparser2-tree-adapter": "^7.1.0",
+        "parse5-parser-stream": "^7.1.2",
+        "undici": "^7.10.0",
+        "whatwg-mimetype": "^4.0.0"
+      }
+    },
+    "cheerio-select": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/cheerio-select/-/cheerio-select-2.1.0.tgz",
+      "integrity": "sha512-9v9kG0LvzrlcungtnJtpGNxY+fzECQKhK4EGJX2vByejiMX84MFNQw4UxPJl3bFbTMw+Dfs37XaIkCwTZfLh4g==",
+      "requires": {
+        "boolbase": "^1.0.0",
+        "css-select": "^5.1.0",
+        "css-what": "^6.1.0",
+        "domelementtype": "^2.3.0",
+        "domhandler": "^5.0.3",
+        "domutils": "^3.0.1"
+      }
+    },
     "chokidar": {
       "version": "3.6.0",
       "resolved": "https://registry.npmmirror.com/chokidar/-/chokidar-3.6.0.tgz",
@@ -5083,6 +5483,23 @@
       "resolved": "https://registry.npmmirror.com/crypto/-/crypto-1.0.1.tgz",
       "integrity": "sha512-VxBKmeNcqQdiUQUW2Tzq0t377b54N2bMtXO/qiLa+6eRRmmC4qT3D4OnTGoT/U6O9aklQ/jTwbOtRMTTY8G0Ig=="
     },
+    "css-select": {
+      "version": "5.1.0",
+      "resolved": "https://registry.npmjs.org/css-select/-/css-select-5.1.0.tgz",
+      "integrity": "sha512-nwoRF1rvRRnnCqqY7updORDsuqKzqYJ28+oSMaJMMgOauh3fvwHqMS7EZpIPqK8GL+g9mKxF1vP/ZjSeNjEVHg==",
+      "requires": {
+        "boolbase": "^1.0.0",
+        "css-what": "^6.1.0",
+        "domhandler": "^5.0.2",
+        "domutils": "^3.0.1",
+        "nth-check": "^2.0.1"
+      }
+    },
+    "css-what": {
+      "version": "6.1.0",
+      "resolved": "https://registry.npmjs.org/css-what/-/css-what-6.1.0.tgz",
+      "integrity": "sha512-HTUrgRJ7r4dsZKU6GjmpfRK1O76h97Z8MfS1G0FozR+oF2kG6Vfe8JE6zwrkbxigziPHinCJ+gCPjA9EaBDtRw=="
+    },
     "dashdash": {
       "version": "1.14.1",
       "resolved": "https://registry.npmmirror.com/dashdash/-/dashdash-1.14.1.tgz",
@@ -5126,6 +5543,39 @@
       "resolved": "https://registry.npmmirror.com/diff-sequences/-/diff-sequences-29.6.3.tgz",
       "integrity": "sha512-EjePK1srD3P08o2j4f0ExnylqRs5B9tJjcp9t1krH2qRi8CCdsYfwe9JgSLurFBWwq4uOlipzfk5fHNvwFKr8Q=="
     },
+    "dom-serializer": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-2.0.0.tgz",
+      "integrity": "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==",
+      "requires": {
+        "domelementtype": "^2.3.0",
+        "domhandler": "^5.0.2",
+        "entities": "^4.2.0"
+      }
+    },
+    "domelementtype": {
+      "version": "2.3.0",
+      "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-2.3.0.tgz",
+      "integrity": "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw=="
+    },
+    "domhandler": {
+      "version": "5.0.3",
+      "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-5.0.3.tgz",
+      "integrity": "sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==",
+      "requires": {
+        "domelementtype": "^2.3.0"
+      }
+    },
+    "domutils": {
+      "version": "3.2.2",
+      "resolved": "https://registry.npmjs.org/domutils/-/domutils-3.2.2.tgz",
+      "integrity": "sha512-6kZKyUajlDuqlHKVX1w7gyslj9MPIXzIFiz/rGu35uC1wMi+kMhQwGhl4lt9unC9Vb9INnY9Z3/ZA3+FhASLaw==",
+      "requires": {
+        "dom-serializer": "^2.0.0",
+        "domelementtype": "^2.3.0",
+        "domhandler": "^5.0.3"
+      }
+    },
     "dunder-proto": {
       "version": "1.0.1",
       "resolved": "https://registry.npmmirror.com/dunder-proto/-/dunder-proto-1.0.1.tgz",
@@ -5160,6 +5610,30 @@
       "resolved": "https://registry.npmmirror.com/encodeurl/-/encodeurl-2.0.0.tgz",
       "integrity": "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg=="
     },
+    "encoding-sniffer": {
+      "version": "0.2.1",
+      "resolved": "https://registry.npmjs.org/encoding-sniffer/-/encoding-sniffer-0.2.1.tgz",
+      "integrity": "sha512-5gvq20T6vfpekVtqrYQsSCFZ1wEg5+wW0/QaZMWkFr6BqD3NfKs0rLCx4rrVlSWJeZb5NBJgVLswK/w2MWU+Gw==",
+      "requires": {
+        "iconv-lite": "^0.6.3",
+        "whatwg-encoding": "^3.1.1"
+      },
+      "dependencies": {
+        "iconv-lite": {
+          "version": "0.6.3",
+          "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz",
+          "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==",
+          "requires": {
+            "safer-buffer": ">= 2.1.2 < 3.0.0"
+          }
+        }
+      }
+    },
+    "entities": {
+      "version": "4.5.0",
+      "resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz",
+      "integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw=="
+    },
     "es-define-property": {
       "version": "1.0.1",
       "resolved": "https://registry.npmmirror.com/es-define-property/-/es-define-property-1.0.1.tgz",
@@ -5486,6 +5960,24 @@
         "function-bind": "^1.1.2"
       }
     },
+    "htmlparser2": {
+      "version": "10.0.0",
+      "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-10.0.0.tgz",
+      "integrity": "sha512-TwAZM+zE5Tq3lrEHvOlvwgj1XLWQCtaaibSN11Q+gGBAS7Y1uZSWwXXRe4iF6OXnaq1riyQAPFOBtYc77Mxq0g==",
+      "requires": {
+        "domelementtype": "^2.3.0",
+        "domhandler": "^5.0.3",
+        "domutils": "^3.2.1",
+        "entities": "^6.0.0"
+      },
+      "dependencies": {
+        "entities": {
+          "version": "6.0.1",
+          "resolved": "https://registry.npmjs.org/entities/-/entities-6.0.1.tgz",
+          "integrity": "sha512-aN97NXWF6AWBTahfVOIrB/NShkzi5H7F9r1s9mD3cDj4Ko5f2qhhVoYMibXF7GlLveb/D2ioWay8lxI97Ven3g=="
+        }
+      }
+    },
     "http-errors": {
       "version": "2.0.0",
       "resolved": "https://registry.npmmirror.com/http-errors/-/http-errors-2.0.0.tgz",
@@ -6034,6 +6526,14 @@
       "resolved": "https://registry.npmmirror.com/normalize-path/-/normalize-path-3.0.0.tgz",
       "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA=="
     },
+    "nth-check": {
+      "version": "2.1.1",
+      "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-2.1.1.tgz",
+      "integrity": "sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w==",
+      "requires": {
+        "boolbase": "^1.0.0"
+      }
+    },
     "oauth-sign": {
       "version": "0.9.0",
       "resolved": "https://registry.npmmirror.com/oauth-sign/-/oauth-sign-0.9.0.tgz",
@@ -6091,6 +6591,38 @@
       "resolved": "https://registry.npmmirror.com/p-try/-/p-try-2.2.0.tgz",
       "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ=="
     },
+    "parse5": {
+      "version": "7.3.0",
+      "resolved": "https://registry.npmjs.org/parse5/-/parse5-7.3.0.tgz",
+      "integrity": "sha512-IInvU7fabl34qmi9gY8XOVxhYyMyuH2xUNpb2q8/Y+7552KlejkRvqvD19nMoUW/uQGGbqNpA6Tufu5FL5BZgw==",
+      "requires": {
+        "entities": "^6.0.0"
+      },
+      "dependencies": {
+        "entities": {
+          "version": "6.0.1",
+          "resolved": "https://registry.npmjs.org/entities/-/entities-6.0.1.tgz",
+          "integrity": "sha512-aN97NXWF6AWBTahfVOIrB/NShkzi5H7F9r1s9mD3cDj4Ko5f2qhhVoYMibXF7GlLveb/D2ioWay8lxI97Ven3g=="
+        }
+      }
+    },
+    "parse5-htmlparser2-tree-adapter": {
+      "version": "7.1.0",
+      "resolved": "https://registry.npmjs.org/parse5-htmlparser2-tree-adapter/-/parse5-htmlparser2-tree-adapter-7.1.0.tgz",
+      "integrity": "sha512-ruw5xyKs6lrpo9x9rCZqZZnIUntICjQAd0Wsmp396Ul9lN/h+ifgVV1x1gZHi8euej6wTfpqX8j+BFQxF0NS/g==",
+      "requires": {
+        "domhandler": "^5.0.3",
+        "parse5": "^7.0.0"
+      }
+    },
+    "parse5-parser-stream": {
+      "version": "7.1.2",
+      "resolved": "https://registry.npmjs.org/parse5-parser-stream/-/parse5-parser-stream-7.1.2.tgz",
+      "integrity": "sha512-JyeQc9iwFLn5TbvvqACIF/VXG6abODeB3Fwmv/TGdLk2LfbWkaySGY72at4+Ty7EkPZj854u4CrICqNk2qIbow==",
+      "requires": {
+        "parse5": "^7.0.0"
+      }
+    },
     "parseurl": {
       "version": "1.3.3",
       "resolved": "https://registry.npmmirror.com/parseurl/-/parseurl-1.3.3.tgz",
@@ -6330,6 +6862,7 @@
       "requires": {
         "axios": "^1.7.9",
         "bcryptjs": "^2.4.3",
+        "cheerio": "^1.1.0",
         "crypto": "^1.0.1",
         "express": "^4.18.2",
         "File": "^0.10.2",
@@ -7093,6 +7626,11 @@
             "unpipe": "1.0.0"
           }
         },
+        "boolbase": {
+          "version": "1.0.0",
+          "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz",
+          "integrity": "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww=="
+        },
         "brace-expansion": {
           "version": "1.1.11",
           "resolved": "https://registry.npmmirror.com/brace-expansion/-/brace-expansion-1.1.11.tgz",
@@ -7199,6 +7737,37 @@
             }
           }
         },
+        "cheerio": {
+          "version": "1.1.0",
+          "resolved": "https://registry.npmjs.org/cheerio/-/cheerio-1.1.0.tgz",
+          "integrity": "sha512-+0hMx9eYhJvWbgpKV9hN7jg0JcwydpopZE4hgi+KvQtByZXPp04NiCWU0LzcAbP63abZckIHkTQaXVF52mX3xQ==",
+          "requires": {
+            "cheerio-select": "^2.1.0",
+            "dom-serializer": "^2.0.0",
+            "domhandler": "^5.0.3",
+            "domutils": "^3.2.2",
+            "encoding-sniffer": "^0.2.0",
+            "htmlparser2": "^10.0.0",
+            "parse5": "^7.3.0",
+            "parse5-htmlparser2-tree-adapter": "^7.1.0",
+            "parse5-parser-stream": "^7.1.2",
+            "undici": "^7.10.0",
+            "whatwg-mimetype": "^4.0.0"
+          }
+        },
+        "cheerio-select": {
+          "version": "2.1.0",
+          "resolved": "https://registry.npmjs.org/cheerio-select/-/cheerio-select-2.1.0.tgz",
+          "integrity": "sha512-9v9kG0LvzrlcungtnJtpGNxY+fzECQKhK4EGJX2vByejiMX84MFNQw4UxPJl3bFbTMw+Dfs37XaIkCwTZfLh4g==",
+          "requires": {
+            "boolbase": "^1.0.0",
+            "css-select": "^5.1.0",
+            "css-what": "^6.1.0",
+            "domelementtype": "^2.3.0",
+            "domhandler": "^5.0.3",
+            "domutils": "^3.0.1"
+          }
+        },
         "chokidar": {
           "version": "3.6.0",
           "resolved": "https://registry.npmmirror.com/chokidar/-/chokidar-3.6.0.tgz",
@@ -7318,6 +7887,23 @@
           "resolved": "https://registry.npmmirror.com/crypto/-/crypto-1.0.1.tgz",
           "integrity": "sha512-VxBKmeNcqQdiUQUW2Tzq0t377b54N2bMtXO/qiLa+6eRRmmC4qT3D4OnTGoT/U6O9aklQ/jTwbOtRMTTY8G0Ig=="
         },
+        "css-select": {
+          "version": "5.1.0",
+          "resolved": "https://registry.npmjs.org/css-select/-/css-select-5.1.0.tgz",
+          "integrity": "sha512-nwoRF1rvRRnnCqqY7updORDsuqKzqYJ28+oSMaJMMgOauh3fvwHqMS7EZpIPqK8GL+g9mKxF1vP/ZjSeNjEVHg==",
+          "requires": {
+            "boolbase": "^1.0.0",
+            "css-what": "^6.1.0",
+            "domhandler": "^5.0.2",
+            "domutils": "^3.0.1",
+            "nth-check": "^2.0.1"
+          }
+        },
+        "css-what": {
+          "version": "6.1.0",
+          "resolved": "https://registry.npmjs.org/css-what/-/css-what-6.1.0.tgz",
+          "integrity": "sha512-HTUrgRJ7r4dsZKU6GjmpfRK1O76h97Z8MfS1G0FozR+oF2kG6Vfe8JE6zwrkbxigziPHinCJ+gCPjA9EaBDtRw=="
+        },
         "dashdash": {
           "version": "1.14.1",
           "resolved": "https://registry.npmmirror.com/dashdash/-/dashdash-1.14.1.tgz",
@@ -7361,6 +7947,39 @@
           "resolved": "https://registry.npmmirror.com/diff-sequences/-/diff-sequences-29.6.3.tgz",
           "integrity": "sha512-EjePK1srD3P08o2j4f0ExnylqRs5B9tJjcp9t1krH2qRi8CCdsYfwe9JgSLurFBWwq4uOlipzfk5fHNvwFKr8Q=="
         },
+        "dom-serializer": {
+          "version": "2.0.0",
+          "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-2.0.0.tgz",
+          "integrity": "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==",
+          "requires": {
+            "domelementtype": "^2.3.0",
+            "domhandler": "^5.0.2",
+            "entities": "^4.2.0"
+          }
+        },
+        "domelementtype": {
+          "version": "2.3.0",
+          "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-2.3.0.tgz",
+          "integrity": "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw=="
+        },
+        "domhandler": {
+          "version": "5.0.3",
+          "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-5.0.3.tgz",
+          "integrity": "sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==",
+          "requires": {
+            "domelementtype": "^2.3.0"
+          }
+        },
+        "domutils": {
+          "version": "3.2.2",
+          "resolved": "https://registry.npmjs.org/domutils/-/domutils-3.2.2.tgz",
+          "integrity": "sha512-6kZKyUajlDuqlHKVX1w7gyslj9MPIXzIFiz/rGu35uC1wMi+kMhQwGhl4lt9unC9Vb9INnY9Z3/ZA3+FhASLaw==",
+          "requires": {
+            "dom-serializer": "^2.0.0",
+            "domelementtype": "^2.3.0",
+            "domhandler": "^5.0.3"
+          }
+        },
         "dunder-proto": {
           "version": "1.0.1",
           "resolved": "https://registry.npmmirror.com/dunder-proto/-/dunder-proto-1.0.1.tgz",
@@ -7395,6 +8014,30 @@
           "resolved": "https://registry.npmmirror.com/encodeurl/-/encodeurl-2.0.0.tgz",
           "integrity": "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg=="
         },
+        "encoding-sniffer": {
+          "version": "0.2.1",
+          "resolved": "https://registry.npmjs.org/encoding-sniffer/-/encoding-sniffer-0.2.1.tgz",
+          "integrity": "sha512-5gvq20T6vfpekVtqrYQsSCFZ1wEg5+wW0/QaZMWkFr6BqD3NfKs0rLCx4rrVlSWJeZb5NBJgVLswK/w2MWU+Gw==",
+          "requires": {
+            "iconv-lite": "^0.6.3",
+            "whatwg-encoding": "^3.1.1"
+          },
+          "dependencies": {
+            "iconv-lite": {
+              "version": "0.6.3",
+              "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz",
+              "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==",
+              "requires": {
+                "safer-buffer": ">= 2.1.2 < 3.0.0"
+              }
+            }
+          }
+        },
+        "entities": {
+          "version": "4.5.0",
+          "resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz",
+          "integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw=="
+        },
         "es-define-property": {
           "version": "1.0.1",
           "resolved": "https://registry.npmmirror.com/es-define-property/-/es-define-property-1.0.1.tgz",
@@ -7721,6 +8364,24 @@
             "function-bind": "^1.1.2"
           }
         },
+        "htmlparser2": {
+          "version": "10.0.0",
+          "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-10.0.0.tgz",
+          "integrity": "sha512-TwAZM+zE5Tq3lrEHvOlvwgj1XLWQCtaaibSN11Q+gGBAS7Y1uZSWwXXRe4iF6OXnaq1riyQAPFOBtYc77Mxq0g==",
+          "requires": {
+            "domelementtype": "^2.3.0",
+            "domhandler": "^5.0.3",
+            "domutils": "^3.2.1",
+            "entities": "^6.0.0"
+          },
+          "dependencies": {
+            "entities": {
+              "version": "6.0.1",
+              "resolved": "https://registry.npmjs.org/entities/-/entities-6.0.1.tgz",
+              "integrity": "sha512-aN97NXWF6AWBTahfVOIrB/NShkzi5H7F9r1s9mD3cDj4Ko5f2qhhVoYMibXF7GlLveb/D2ioWay8lxI97Ven3g=="
+            }
+          }
+        },
         "http-errors": {
           "version": "2.0.0",
           "resolved": "https://registry.npmmirror.com/http-errors/-/http-errors-2.0.0.tgz",
@@ -8269,6 +8930,14 @@
           "resolved": "https://registry.npmmirror.com/normalize-path/-/normalize-path-3.0.0.tgz",
           "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA=="
         },
+        "nth-check": {
+          "version": "2.1.1",
+          "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-2.1.1.tgz",
+          "integrity": "sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w==",
+          "requires": {
+            "boolbase": "^1.0.0"
+          }
+        },
         "oauth-sign": {
           "version": "0.9.0",
           "resolved": "https://registry.npmmirror.com/oauth-sign/-/oauth-sign-0.9.0.tgz",
@@ -8326,6 +8995,38 @@
           "resolved": "https://registry.npmmirror.com/p-try/-/p-try-2.2.0.tgz",
           "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ=="
         },
+        "parse5": {
+          "version": "7.3.0",
+          "resolved": "https://registry.npmjs.org/parse5/-/parse5-7.3.0.tgz",
+          "integrity": "sha512-IInvU7fabl34qmi9gY8XOVxhYyMyuH2xUNpb2q8/Y+7552KlejkRvqvD19nMoUW/uQGGbqNpA6Tufu5FL5BZgw==",
+          "requires": {
+            "entities": "^6.0.0"
+          },
+          "dependencies": {
+            "entities": {
+              "version": "6.0.1",
+              "resolved": "https://registry.npmjs.org/entities/-/entities-6.0.1.tgz",
+              "integrity": "sha512-aN97NXWF6AWBTahfVOIrB/NShkzi5H7F9r1s9mD3cDj4Ko5f2qhhVoYMibXF7GlLveb/D2ioWay8lxI97Ven3g=="
+            }
+          }
+        },
+        "parse5-htmlparser2-tree-adapter": {
+          "version": "7.1.0",
+          "resolved": "https://registry.npmjs.org/parse5-htmlparser2-tree-adapter/-/parse5-htmlparser2-tree-adapter-7.1.0.tgz",
+          "integrity": "sha512-ruw5xyKs6lrpo9x9rCZqZZnIUntICjQAd0Wsmp396Ul9lN/h+ifgVV1x1gZHi8euej6wTfpqX8j+BFQxF0NS/g==",
+          "requires": {
+            "domhandler": "^5.0.3",
+            "parse5": "^7.0.0"
+          }
+        },
+        "parse5-parser-stream": {
+          "version": "7.1.2",
+          "resolved": "https://registry.npmjs.org/parse5-parser-stream/-/parse5-parser-stream-7.1.2.tgz",
+          "integrity": "sha512-JyeQc9iwFLn5TbvvqACIF/VXG6abODeB3Fwmv/TGdLk2LfbWkaySGY72at4+Ty7EkPZj854u4CrICqNk2qIbow==",
+          "requires": {
+            "parse5": "^7.0.0"
+          }
+        },
         "parseurl": {
           "version": "1.3.3",
           "resolved": "https://registry.npmmirror.com/parseurl/-/parseurl-1.3.3.tgz",
@@ -9002,6 +9703,11 @@
           "integrity": "sha512-WxONCrssBM8TSPRqN5EmsjVrsv4A8X12J4ArBiiayv3DyyG3ZlIg6yysuuSYdZsVz3TKcTg2fd//Ujd4CHV1iA==",
           "dev": true
         },
+        "undici": {
+          "version": "7.10.0",
+          "resolved": "https://registry.npmjs.org/undici/-/undici-7.10.0.tgz",
+          "integrity": "sha512-u5otvFBOBZvmdjWLVW+5DAc9Nkq8f24g0O9oY7qw2JVIF1VocIFoyz9JFkuVOS2j41AufeO0xnlweJ2RLT8nGw=="
+        },
         "undici-types": {
           "version": "6.20.0",
           "resolved": "https://registry.npmmirror.com/undici-types/-/undici-types-6.20.0.tgz",
@@ -9079,6 +9785,29 @@
             "makeerror": "1.0.12"
           }
         },
+        "whatwg-encoding": {
+          "version": "3.1.1",
+          "resolved": "https://registry.npmjs.org/whatwg-encoding/-/whatwg-encoding-3.1.1.tgz",
+          "integrity": "sha512-6qN4hJdMwfYBtE3YBTTHhoeuUrDBPZmbQaxWAqSALV/MeEnR5z1xd8UKud2RAkFoPkmB+hli1TZSnyi84xz1vQ==",
+          "requires": {
+            "iconv-lite": "0.6.3"
+          },
+          "dependencies": {
+            "iconv-lite": {
+              "version": "0.6.3",
+              "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz",
+              "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==",
+              "requires": {
+                "safer-buffer": ">= 2.1.2 < 3.0.0"
+              }
+            }
+          }
+        },
+        "whatwg-mimetype": {
+          "version": "4.0.0",
+          "resolved": "https://registry.npmjs.org/whatwg-mimetype/-/whatwg-mimetype-4.0.0.tgz",
+          "integrity": "sha512-QaKxh0eNIi2mE9p2vEdzfagOKHCcj1pJ56EEHGQOVxp8r9/iszLUUV7v89x9O1p/T+NlTM5W7jW6+cz4Fq1YVg=="
+        },
         "wrappy": {
           "version": "1.0.2",
           "resolved": "https://registry.npmmirror.com/wrappy/-/wrappy-1.0.2.tgz",
@@ -9547,6 +10276,11 @@
       "integrity": "sha512-WxONCrssBM8TSPRqN5EmsjVrsv4A8X12J4ArBiiayv3DyyG3ZlIg6yysuuSYdZsVz3TKcTg2fd//Ujd4CHV1iA==",
       "dev": true
     },
+    "undici": {
+      "version": "7.10.0",
+      "resolved": "https://registry.npmjs.org/undici/-/undici-7.10.0.tgz",
+      "integrity": "sha512-u5otvFBOBZvmdjWLVW+5DAc9Nkq8f24g0O9oY7qw2JVIF1VocIFoyz9JFkuVOS2j41AufeO0xnlweJ2RLT8nGw=="
+    },
     "undici-types": {
       "version": "6.20.0",
       "resolved": "https://registry.npmmirror.com/undici-types/-/undici-types-6.20.0.tgz",
@@ -9624,6 +10358,29 @@
         "makeerror": "1.0.12"
       }
     },
+    "whatwg-encoding": {
+      "version": "3.1.1",
+      "resolved": "https://registry.npmjs.org/whatwg-encoding/-/whatwg-encoding-3.1.1.tgz",
+      "integrity": "sha512-6qN4hJdMwfYBtE3YBTTHhoeuUrDBPZmbQaxWAqSALV/MeEnR5z1xd8UKud2RAkFoPkmB+hli1TZSnyi84xz1vQ==",
+      "requires": {
+        "iconv-lite": "0.6.3"
+      },
+      "dependencies": {
+        "iconv-lite": {
+          "version": "0.6.3",
+          "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz",
+          "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==",
+          "requires": {
+            "safer-buffer": ">= 2.1.2 < 3.0.0"
+          }
+        }
+      }
+    },
+    "whatwg-mimetype": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/whatwg-mimetype/-/whatwg-mimetype-4.0.0.tgz",
+      "integrity": "sha512-QaKxh0eNIi2mE9p2vEdzfagOKHCcj1pJ56EEHGQOVxp8r9/iszLUUV7v89x9O1p/T+NlTM5W7jW6+cz4Fq1YVg=="
+    },
     "wrappy": {
       "version": "1.0.2",
       "resolved": "https://registry.npmmirror.com/wrappy/-/wrappy-1.0.2.tgz",

package.json

@@ -12,6 +12,7 @@
   "dependencies": {
     "axios": "^1.7.9",
     "bcryptjs": "^2.4.3",
+    "cheerio": "^1.1.0",
     "crypto": "^1.0.1",
     "express": "^4.18.2",
     "file": "^0.2.2",

pbl.js

@@ -25,6 +25,8 @@ const axios = require('axios');
 const multer = require("multer");
 const {chromium} = require('playwright');
 const TurndownService = require('turndown');
+const cheerio = require('cheerio');
+const { v4: uuidv4 } = require('uuid');
 
 const loginHandler = require('./login'); // 确保路径正确
 const { r2rClient } = require("r2r-js");
@@ -1306,26 +1308,38 @@ router.post('/knowledge-base/html-to-markdown', upload.none(), async (req, res)
     // console.log('收到的 req.body:', req.body);
     // console.log('收到的 req.files:', req.files);
     // console.log('收到的 headers:', req.headers);
-    const url = req.body.url; // 前端传入 url
-    if (!url) {
+    const requestUrl = req.body.url; // 前端传入 url
+    if (!requestUrl) {
         return res.status(400).json({error: 'Missing url'});
     }
     try {
         const browser = await chromium.launch({headless: true});
         const page = await browser.newPage();
-        await page.goto(url, {waitUntil: 'networkidle'});
+        await page.goto(requestUrl, {waitUntil: 'networkidle'});
         const html = await page.content();
         await browser.close();
         // 新增：HTML 转 Markdown
         const turndownService = new TurndownService();
-        const markdownContent = turndownService.turndown(html);
+        // 用 cheerio 只提取正文部分
+        const cheerioDoc = cheerio.load(html);
+        let mainHtml = cheerioDoc('main').html() || cheerioDoc('article').html() || cheerioDoc('#content').html() || cheerioDoc('.content').html() || cheerioDoc('.post').html() || cheerioDoc('body').html();
+        // 用 cheerio 过滤无关标签
+        const cheerioMain = cheerio.load(mainHtml || '');
+        cheerioMain('style, script, noscript, iframe, link, header, footer, nav, form, .ads, .advertisement').remove();
+        mainHtml = cheerioMain.html();
+        const markdownContent = turndownService.turndown(mainHtml);
         // 新增：解析 <title> 标签内容
         let title = '';
         const titleMatch = html.match(/<title>([\s\S]*?)<\/title>/i);
         if (titleMatch && titleMatch[1]) {
             title = titleMatch[1].trim();
         }
-
+        let finalTitle;
+        if (title && title.trim()) {
+            finalTitle = title + '.md';
+        } else {
+            finalTitle = 'untitled-' + uuidv4() + '.md';
+        }
         console.log("抓取到的内容长度:", html.length, "markdown的内容长度:", markdownContent.length);
 
         // 解析用户传入的参数
@@ -1346,15 +1360,15 @@ router.post('/knowledge-base/html-to-markdown', upload.none(), async (req, res)
                 return res.status(400).json({error: 'metadata 不是合法的 JSON 字符串'});
             }
         }
-        if (!finalMetadata) {
-            finalMetadata = {
-                title: title || "抓取的网页内容",
-                source_url: url,
-                content_type: "markdown",
-                scraped_at: new Date().toISOString(),
-                upload_id: Date.now() + '-' + Math.random()
-            };
-        }
+        // if (!finalMetadata) {
+        //     finalMetadata = {
+        //         title: title || uuidv4(),
+        //         // source_url: url,
+        //         content_type: "markdown",
+        //         // scraped_at: new Date().toISOString(),
+        //         // upload_id: Date.now() + '-' + Math.random()
+        //     };
+        // }
         // 处理 collection_ids 字段
         let finalCollectionIds = collection_ids;
         if (typeof finalCollectionIds === 'string') {
@@ -1369,68 +1383,44 @@ router.post('/knowledge-base/html-to-markdown', upload.none(), async (req, res)
             finalMetadata.collection_ids = finalCollectionIds;
         }
 
-        // 组装 create 参数
-        const createParams = {
-            raw_text: markdownContent,
-            metadata: finalMetadata
-        };
+        // 补全 finalMetadata.title 和 content_type
+        if (!finalMetadata.title) {
+            finalMetadata.title = finalTitle;
+        }
+        finalMetadata.content_type = 'markdown';
+
+        // 用 axios 发送 multipart/form-data 方式上传到 r2r
+        const FormData = require('form-data');
+        const fs = require('fs');
+        const path = require('path');
+        const tmpFilePath = path.join(__dirname, `${title || 'untitled'}-${uuidv4()}.md`);
+        fs.writeFileSync(tmpFilePath, markdownContent, 'utf8');
+        const form = new FormData();
+        form.append('file', fs.createReadStream(tmpFilePath), finalTitle);
         if (finalCollectionIds && Array.isArray(finalCollectionIds)) {
-            createParams.collection_ids = finalCollectionIds;
+            form.append('collection_ids', JSON.stringify(finalCollectionIds));
         }
         if (customId) {
-            createParams.id = customId;
-        }
-        if (ingestion_mode) {
-            createParams.ingestion_mode = ingestion_mode;
-        } else {
-            createParams.ingestionMode = "fast";
-        }
-        if (typeof run_with_orchestration !== 'undefined') {
-            createParams.run_with_orchestration = run_with_orchestration;
+            form.append('id', customId);
         }
-
-        // 4. 将 markdown 内容上传到 R2R
-        console.log("正在上传文档...");
-        try {
-            const documentResponse = await client.documents.create(createParams);
-            console.log("文档上传成功!");
-            console.log("文档ID:", documentResponse.results.documentId);
-            result = {
-                results: documentResponse.results,
-                title: title
+        form.append('metadata', JSON.stringify(finalMetadata));
+        form.append('ingestion_mode', ingestion_mode || 'fast');
+        form.append('run_with_orchestration', 'false');
+        // 打印上传参数，方便排查问题
+        console.log('Request Headers:', form.getHeaders());
+        const url = 'https://r2rserver.cocorobo.cn/v3/documents';
+        const response = await axios.post(url, form, {
+            headers: {
+                ...form.getHeaders(),
+                // 'User-Agent': 'NodeClient/1.0'
             }
-            res.json(result);
-        } catch (error) {
-            console.error('Error occurred:', error);
-            res.status(500).json({error: error.message});
-            // 如果 409 冲突，尝试删除后重试
-            // if (error.message && error.message.includes('409')) {
-            //     // 从错误信息中提取 documentId
-            //     const match = error.message.match(/Document ([\w-]+) is currently ingesting/);
-            //     const conflictDocId = match && match[1];
-            //     if (conflictDocId) {
-            //         try {
-            //             await client.documents.delete({id: conflictDocId});
-            //             console.log('冲突时已删除重复文档:', conflictDocId);
-            //             // 再次尝试上传
-            //             const documentResponse = await client.documents.create(createParams);
-            //             result = {
-            //                 results: documentResponse.results,
-            //                 title: title
-            //             }
-            //             res.json(result);
-            //             return;
-            //         } catch (e2) {
-            //             console.error('冲突时删除重试也失败:', e2);
-            //             throw e2;
-            //         }
-            //     } else {
-            //         throw error;
-            //     }
-            // } else {
-            //     throw error;
-            // }
+        });
+        fs.unlinkSync(tmpFilePath);
+        result = {
+            results: response.data.results,
+            title: finalTitle
         }
+        res.json(result);
     } catch (error) {
         console.error('Error occurred:', error);
         res.status(500).json({error: error.message});