pbl.js 61 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675
  1. // ROUTES FOR OUR APIselectClassStudent
  2. // =============================================================================
  3. const express = require("express");
  4. const request = require("request");
  5. const bcrypt = require("bcryptjs");
  6. const router = express.Router(); // get an instance of the express Router
  7. const querystring = require("querystring");
  8. var mysql = require("./mysql");
  9. // const _mysqlLabor = ["183.36.25.93", "pbl"]; //edu數據庫信息
  10. // const _mysqluser = ["183.36.25.93", "cocorobouser"]; //用户数据库信息
  11. // const _getmysqlLabor = ["123.58.32.151", "pbl"]; //edu數據庫信息f
  12. // const _getmysqluser = ["123.58.32.151", "cocorobouser"]; //edu數據庫信息
  13. const _mysqlLabor = ["172.16.12.5", "pbl"]; //edu數據庫信息
  14. const _mysqluser = ["172.16.12.5", "cocorobouser"]; //edu數據庫信息
  15. // const _getmysqlLabor = ["172.16.12.5", "pbl"]; //edu數據庫信息
  16. // const _getmysqluser = ["172.16.12.5", "cocorobouser"]; //edu數據庫信息
  17. const _getmysqlLabor2 = ["172.16.12.7", "pbl"]; //edu數據庫信息
  18. const _getmysqlLabor = ["172.16.12.7", "pbl"]; //edu數據庫信息
  19. const _getmysqluser = ["172.16.12.7", "cocorobouser"]; //edu數據庫信息
  20. const _localhost = ["127.0.0.1", "pbl"]; //edu數據庫信息
  21. const crypto = require("crypto");
  22. const https = require("https");
  23. const { Pool } = require('pg');
  24. const axios = require('axios');
  25. const multer = require("multer");
  26. const {chromium} = require('playwright');
  27. const TurndownService = require('turndown');
  28. const cheerio = require('cheerio');
  29. const { v4: uuidv4 } = require('uuid');
  30. const loginHandler = require('./login'); // 确保路径正确
  31. const { r2rClient } = require("r2r-js");
  32. const client = new r2rClient("https://r2rserver.cocorobo.cn/");
  33. // 配置 multer 存储选项
  34. const storage = multer.diskStorage({
  35. destination: function (req, file, cb) {
  36. req.setEncoding('utf-8');
  37. cb(null, 'uploads/'); // 文件上传目录
  38. },
  39. filename: function (req, file, cb) {
  40. req.setEncoding('utf-8');
  41. // 使用 Buffer 处理文件名,确保编码正确
  42. const fileExtension = file.originalname.split('.').pop(); // 获取文件后缀
  43. const fileNameWithoutExt = Buffer.from(file.originalname.slice(0, -(fileExtension.length + 1)), 'latin1').toString('utf8'); // 转换编码
  44. cb(null, fileNameWithoutExt + '-' + Date.now() + '.' + fileExtension); // 使用时间戳和原始文件名
  45. }
  46. });
  47. const upload = multer({
  48. storage: storage, limits: {
  49. fileSize: 100 * 1024 * 1024 // 100MB
  50. }
  51. });
  52. // 全局浏览器实例
  53. let globalBrowser = null;
  54. async function initBrowser() {
  55. if (!globalBrowser) {
  56. globalBrowser = await chromium.launch({headless: true});
  57. console.log('Browser initialized');
  58. }
  59. return globalBrowser;
  60. }
  61. router.route("/login").all(async (req, res, next) => {
  62. loginHandler(req, res)
  63. });
  64. router.route("/logout").all(async (req, res, next) => {
  65. try {
  66. const response = await client.users.logout();
  67. res.status(200).json({ result: response.results });
  68. } catch (error) {
  69. res.status(500).json({ error: error instanceof Error ? error.message : "An error occurred" });
  70. }
  71. });
  72. //获取文档列表
  73. router.route("/getDocument").all(async (req, res, next) => {
  74. try {
  75. const response = await client.documents.list({
  76. limit: 10,
  77. offset: 0,
  78. });
  79. console.log(response);
  80. res.status(200).json({ result: response.results });
  81. } catch (error) {
  82. res.status(500).json({ error: error instanceof Error ? error.message : "An error occurred" });
  83. }
  84. });
  85. //列出文档块
  86. router.route("/listChunks").all(async (req, res, next) => {
  87. try {
  88. const response = await client.documents.listChunks({
  89. id: "5111165f-965c-5d61-89e3-1f74b0178032",
  90. });
  91. console.log(response);
  92. res.status(200).json({ result: response.results });
  93. } catch (error) {
  94. res.status(500).json({ error: error instanceof Error ? error.message : "An error occurred" });
  95. }
  96. });
  97. //提取实体和关系
  98. router.route("/extract").all(async (req, res, next) => {
  99. try {
  100. const response = await client.documents.extract({ id: "5111165f-965c-5d61-89e3-1f74b0178032" })
  101. console.log(response);
  102. res.status(200).json({ result: response.results });
  103. } catch (error) {
  104. res.status(500).json({ error: error instanceof Error ? error.message : "An error occurred" });
  105. }
  106. });
  107. router.route("/selectUser").all((req, res, next) => {
  108. var json = queryString(req.url);
  109. getmysql(req, res, "selectUser", json["userid"]);
  110. });
  111. // 文件上传接口
  112. router.post('/upload', upload.single('file'), async (req, res) => {
  113. req.setEncoding('utf-8')
  114. try {
  115. console.log(req.file); // 打印上传的文件信息
  116. console.log(`File path: uploads/${req.file.filename}`);
  117. const path = require('path');
  118. const filePath = path.resolve(__dirname, 'uploads', req.file.filename);
  119. console.log(filePath);
  120. let filename = Buffer.from(req.file.originalname, 'latin1').toString('utf8');
  121. const fs = require('fs');
  122. const file = new File([fs.readFileSync('./uploads/q-1739170522767.doc')], filename, { type: req.file.mimetype });
  123. const response = await client.documents.create({
  124. file: file,
  125. metadata: { title: filename }
  126. });
  127. console.log(response);
  128. res.status(200).json({ message: 'File uploaded successfully', file: req.file });
  129. } catch (error) {
  130. res.status(500).json({ error: error instanceof Error ? error.message : "An error occurred" });
  131. }
  132. });
  133. //获取文档列表
  134. router.route("/getFile").all(async (req, res, next) => {
  135. //mode的处理
  136. if (req.body[0]) {
  137. p = Object.values(req.body[0]);
  138. p.unshift(_mysqlLabor[0], _mysqlLabor[1], "getFile");
  139. mysql.usselect(p, async function (ret) {
  140. let data = ret[0];
  141. // const array = data.map(item => item.documentid);
  142. const array = data.map(item => item.documentid);
  143. try {
  144. const response = await client.documents.list({ ids: array });
  145. // const response = await client.collections.listDocuments({ id: req.body[0].folderid });
  146. // const response = await client.collections.listDocuments({ collection_ids: req.body[0].folderid.split(",") });
  147. // console.log(response);
  148. let data2 = response.results.map(doc => {
  149. doc.documentid = doc.id;
  150. delete doc.id;
  151. return doc;
  152. });
  153. // console.log(data2);
  154. // 合并data和data2,根据documentid
  155. const mergedData = data.map(item => {
  156. const correspondingDoc = data2.find(doc => doc.documentid === item.documentid);
  157. return { ...item, ...correspondingDoc };
  158. });
  159. res.status(200).json({ result: mergedData });
  160. } catch (error) {
  161. res.status(500).json({ error: error instanceof Error ? error.message : "An error occurred" });
  162. }
  163. });
  164. }
  165. });
  166. //获取文档列表
  167. router.route("/getFileR2r").all(async (req, res, next) => {
  168. //mode的处理
  169. if (req.body[0]) {
  170. let fileid = decodeURIComponent(req.body[0].fileids).split(",");
  171. try {
  172. const response = await client.documents.list({ ids: fileid });
  173. res.status(200).json({ result: response.results });
  174. } catch (error) {
  175. res.status(500).json({ error: error instanceof Error ? error.message : "An error occurred" });
  176. }
  177. }
  178. });
  179. //获取文档列表
  180. router.route("/getFileD").all(async (req, res, next) => {
  181. //mode的处理
  182. if (req.body[0]) {
  183. p = Object.values(req.body[0]);
  184. p.unshift(_mysqlLabor[0], _mysqlLabor[1], "getFileD");
  185. mysql.usselect(p, async function (ret) {
  186. let data = ret[0];
  187. // const array = data.map(item => item.documentid);
  188. const array = data.map(item => item.documentid);
  189. try {
  190. const response = await client.documents.list({ ids: array });
  191. // const response = await client.collections.listDocuments({ id: req.body[0].folderid });
  192. // const response = await client.collections.listDocuments({ collection_ids: req.body[0].folderid.split(",") });
  193. // console.log(response);
  194. let data2 = response.results.map(doc => {
  195. doc.documentid = doc.id;
  196. delete doc.id;
  197. return doc;
  198. });
  199. // console.log(data2);
  200. // 合并data和data2,根据documentid
  201. const mergedData = data.map(item => {
  202. const correspondingDoc = data2.find(doc => doc.documentid === item.documentid);
  203. return { ...item, ...correspondingDoc };
  204. });
  205. res.status(200).json({ result: mergedData });
  206. } catch (error) {
  207. res.status(500).json({ error: error instanceof Error ? error.message : "An error occurred" });
  208. }
  209. });
  210. }
  211. });
  212. //获取文档列表
  213. router.route("/getFileArray").all(async (req, res, next) => {
  214. //mode的处理
  215. if (req.body[0]) {
  216. // const array = data.map(item => item.documentid);
  217. try {
  218. let data = decodeURIComponent(req.body[0].array).split(",");
  219. const response = await client.documents.list({ ids: data });
  220. res.status(200).json({ result: response.results });
  221. } catch (error) {
  222. res.status(500).json({ error: error instanceof Error ? error.message : "An error occurred" });
  223. }
  224. }
  225. });
  226. //获取文档列表
  227. router.route("/getFileD2").all(async (req, res, next) => {
  228. //mode的处理
  229. if (req.body) {
  230. p = [req.body.uid, req.body.folderid, req.body.n, req.body.page, req.body.num];
  231. p.unshift(_mysqlLabor[0], _mysqlLabor[1], "getFileD");
  232. mysql.usselect(p, async function (ret) {
  233. let data = ret[0];
  234. // const array = data.map(item => item.documentid);
  235. const array = data.map(item => item.documentid);
  236. try {
  237. const response = await client.documents.list({ ids: array });
  238. // const response = await client.collections.listDocuments({ id: req.body[0].folderid });
  239. // const response = await client.collections.listDocuments({ collection_ids: req.body[0].folderid.split(",") });
  240. // console.log(response);
  241. let data2 = response.results.map(doc => {
  242. doc.documentid = doc.id;
  243. delete doc.id;
  244. return doc;
  245. });
  246. // console.log(data2);
  247. // 合并data和data2,根据documentid
  248. const mergedData = data.map(item => {
  249. const correspondingDoc = data2.find(doc => doc.documentid === item.documentid);
  250. return { ...item, ...correspondingDoc };
  251. });
  252. res.status(200).json({ result: mergedData });
  253. } catch (error) {
  254. res.status(500).json({ error: error instanceof Error ? error.message : "An error occurred" });
  255. }
  256. });
  257. }
  258. });
  259. //获取文档列表
  260. router.route("/getFileD2").all(async (req, res, next) => {
  261. //mode的处理
  262. if (req.body) {
  263. p = [req.body.uid, req.body.folderid, req.body.n, req.body.page, req.body.num];
  264. p.unshift(_mysqlLabor[0], _mysqlLabor[1], "getFileD");
  265. mysql.usselect(p, async function (ret) {
  266. let data = ret[0];
  267. // const array = data.map(item => item.documentid);
  268. const array = data.map(item => item.documentid);
  269. try {
  270. const response = await client.documents.list({ ids: array });
  271. // const response = await client.collections.listDocuments({ id: req.body[0].folderid });
  272. // const response = await client.collections.listDocuments({ collection_ids: req.body[0].folderid.split(",") });
  273. // console.log(response);
  274. let data2 = response.results.map(doc => {
  275. doc.documentid = doc.id;
  276. delete doc.id;
  277. return doc;
  278. });
  279. // console.log(data2);
  280. // 合并data和data2,根据documentid
  281. const mergedData = data.map(item => {
  282. const correspondingDoc = data2.find(doc => doc.documentid === item.documentid);
  283. return { ...item, ...correspondingDoc };
  284. });
  285. res.status(200).json({ result: mergedData });
  286. } catch (error) {
  287. res.status(500).json({ error: error instanceof Error ? error.message : "An error occurred" });
  288. }
  289. });
  290. }
  291. });
  292. //获取文档列表
  293. router.route("/getFileOpen").all(async (req, res, next) => {
  294. //mode的处理
  295. if (req.body[0]) {
  296. p = Object.values(req.body[0]);
  297. p.unshift(_mysqlLabor[0], _mysqlLabor[1], "getFileOpen");
  298. mysql.usselect(p, async function (ret) {
  299. let data = ret[0];
  300. let user = ret[1];
  301. // const array = data.map(item => item.documentid);
  302. const array = data.map(item => item.documentid);
  303. try {
  304. const response = await client.documents.list({ ids: array });
  305. let data2 = response.results.map(doc => {
  306. doc.documentid = doc.id;
  307. delete doc.id;
  308. return doc;
  309. });
  310. // 合并data和data2,根据documentid
  311. const mergedData = data.map(item => {
  312. const correspondingDoc = data2.find(doc => doc.documentid === item.documentid);
  313. return { ...item, ...correspondingDoc };
  314. });
  315. res.status(200).json({ result: mergedData, user: user });
  316. } catch (error) {
  317. res.status(500).json({ error: error instanceof Error ? error.message : "An error occurred" });
  318. }
  319. });
  320. }
  321. });
  322. //获取文档列表2
  323. router.route("/getFileP").all(async (req, res, next) => {
  324. //mode的处理
  325. if (req.body) {
  326. p = [req.body.uid, req.body.folderid, req.body.n, req.body.page, req.body.num];
  327. p.unshift(_mysqlLabor[0], _mysqlLabor[1], "getFile");
  328. mysql.usselect(p, async function (ret) {
  329. let data = ret[0];
  330. // const array = data.map(item => item.documentid);
  331. try {
  332. // const response = await client.documents.list({ collection_ids: [req.body.folderid] });
  333. const response = await client.collections.listDocuments({ id: req.body[0].folderid });
  334. // console.log(response);
  335. let data2 = response.results.map(doc => {
  336. doc.documentid = doc.id;
  337. delete doc.id;
  338. return doc;
  339. });
  340. // 合并data和data2,根据documentid
  341. const mergedData = data.map(item => {
  342. const correspondingDoc = data2.find(doc => doc.documentid === item.documentid);
  343. return { ...item, ...correspondingDoc };
  344. });
  345. res.status(200).json({ result: mergedData });
  346. } catch (error) {
  347. res.status(500).json({ error: error instanceof Error ? error.message : "An error occurred" });
  348. }
  349. });
  350. }
  351. });
  352. //获取能关联的文件
  353. router.route("/getFile2").all(async (req, res, next) => {
  354. //mode的处理
  355. if (req.body[0]) {
  356. p = Object.values(req.body[0]);
  357. p.unshift(_mysqlLabor[0], _mysqlLabor[1], "getFile2");
  358. mysql.usselect(p, async function (ret) {
  359. let data = ret[0];
  360. const array = data.map(item => item.documentid);
  361. try {
  362. const response = await client.documents.list({ ids: array });
  363. // const response = await client.documents.list({ collection_ids: [req.body[0].folderid] });
  364. // console.log(response);
  365. let data2 = response.results.map(doc => {
  366. doc.documentid = doc.id;
  367. delete doc.id;
  368. return doc;
  369. });
  370. // 合并data和data2,根据documentid
  371. const mergedData = data.map(item => {
  372. const correspondingDoc = data2.find(doc => doc.documentid === item.documentid);
  373. return { ...item, ...correspondingDoc };
  374. });
  375. res.status(200).json({ result: mergedData });
  376. } catch (error) {
  377. res.status(500).json({ error: error instanceof Error ? error.message : "An error occurred" });
  378. }
  379. });
  380. }
  381. });
  382. //添加文件夹
  383. router.route("/addFolder").all(async (req, res, next) => {
  384. //mode的处理
  385. if (req.body[0]) {
  386. try {
  387. const response = await client.collections.create({
  388. name: decodeURIComponent(req.body[0].n),
  389. description: decodeURIComponent(req.body[0].fd)
  390. });
  391. p = [...Object.values(req.body[0]), response.results.id];
  392. p.unshift(_mysqlLabor[0], _mysqlLabor[1], "addFolder");
  393. mysql.usselect(p, async function (ret) {
  394. res.end(JSON.stringify(ret));
  395. });
  396. } catch (error) {
  397. res.status(500).json({ error: error instanceof Error ? error.message : "An error occurred" });
  398. }
  399. }
  400. });
  401. //添加文件夹
  402. router.route("/addFolderP").all(async (req, res, next) => {
  403. //mode的处理
  404. if (req.body) {
  405. try {
  406. const response = await client.collections.create({
  407. name: decodeURIComponent(req.body.n),
  408. description: decodeURIComponent(req.body.fd)
  409. });
  410. p = [req.body.n, req.body.fd, req.body.uid, req.body.im , response.results.id];
  411. p.unshift(_mysqlLabor[0], _mysqlLabor[1], "addFolder");
  412. mysql.usselect(p, async function (ret) {
  413. res.end(JSON.stringify(ret));
  414. });
  415. } catch (error) {
  416. res.status(500).json({ error: error instanceof Error ? error.message : "An error occurred" });
  417. }
  418. }
  419. });
  420. //删除文件夹
  421. router.route("/deleteFolder").all(async (req, res, next) => {
  422. //mode的处理
  423. if (req.body[0]) {
  424. try {
  425. const folderids = decodeURIComponent(req.body[0].folderids).split(",");
  426. const ids = req.body[0].ids;
  427. // 并行处理文件夹下的文件删除和文件夹删除
  428. await Promise.all(folderids.map(async folderid => {
  429. const files = await client.collections.listDocuments({ id: folderid, offset: 0, limit: 999 });
  430. await Promise.all(files.results.map(file =>
  431. client.collections.removeDocument({ id: folderid, documentId: file.id })
  432. ));
  433. }));
  434. await Promise.all(folderids.map(folderid => client.collections.delete({ id: folderid })));
  435. const params = [_mysqlLabor[0], _mysqlLabor[1], "deleteFolder", ids];
  436. mysql.usselect(params, async function (ret) {
  437. res.end(JSON.stringify(ret));
  438. });
  439. } catch (error) {
  440. res.status(500).json({ error: error instanceof Error ? error.message : "An error occurred" });
  441. }
  442. }
  443. });
  444. //修改文件夹
  445. router.route("/updateFolder").all(async (req, res, next) => {
  446. //mode的处理
  447. if (req.body[0]) {
  448. try {
  449. const { folderid, ...folderData } = req.body[0]; // 解构赋值,排除folderid
  450. const response = await client.collections.update({
  451. id: folderid,
  452. name: decodeURIComponent(req.body[0].n),
  453. description: decodeURIComponent(req.body[0].d)
  454. });
  455. p = Object.values(folderData); // 获取剩余属性的键
  456. p.unshift(_mysqlLabor[0], _mysqlLabor[1], "updateFolder2");
  457. mysql.usselect(p, async function (ret) {
  458. res.end(JSON.stringify(ret));
  459. });
  460. } catch (error) {
  461. res.status(500).json({ error: error instanceof Error ? error.message : "An error occurred" });
  462. }
  463. }
  464. });
  465. //修改文件夹
  466. router.route("/updateFolder2").all(async (req, res, next) => {
  467. //mode的处理
  468. if (req.body[0]) {
  469. try {
  470. const { folderid, ...folderData } = req.body[0]; // 解构赋值,排除folderid
  471. const response = await client.collections.update({
  472. id: folderid,
  473. name: decodeURIComponent(req.body[0].n),
  474. description: decodeURIComponent(req.body[0].d)
  475. });
  476. p = Object.values(folderData); // 获取剩余属性的键
  477. p.unshift(_mysqlLabor[0], _mysqlLabor[1], "updateFolder3");
  478. mysql.usselect(p, async function (ret) {
  479. res.end(JSON.stringify(ret));
  480. });
  481. } catch (error) {
  482. res.status(500).json({ error: error instanceof Error ? error.message : "An error occurred" });
  483. }
  484. }
  485. });
  486. //移除文件
  487. router.route("/removeFile").all(async (req, res, next) => {
  488. if (req.body[0]) {
  489. let documentids = decodeURIComponent(req.body[0].documentids).split(",");
  490. let folderid = req.body[0].folderid;
  491. let ids = req.body[0].ids;
  492. // 使用 Promise.all 确保所有删除操作都被执行
  493. const removePromises = documentids.map(documentid =>
  494. client.collections.removeDocument({ id: folderid, documentId: documentid })
  495. );
  496. // 不管client.documents报错没有也要执行removeFile
  497. Promise.all(removePromises).catch(error => {
  498. console.error("移除文件时发生错误:", error);
  499. });
  500. p = [ids, folderid];
  501. p.unshift(_mysqlLabor[0], _mysqlLabor[1], "removeFile");
  502. mysql.usselect(p, async function (ret) {
  503. res.end(JSON.stringify(ret));
  504. });
  505. }
  506. });
  507. //移入文件
  508. router.route("/moveFile").all(async (req, res, next) => {
  509. if (req.body[0]) {
  510. try {
  511. let documentids = decodeURIComponent(req.body[0].documentids).split(",")
  512. let folderid = req.body[0].folderid
  513. let ids = decodeURIComponent(req.body[0].ids).split(",")
  514. // 使用 Promise.all 确保所有添加操作都被执行
  515. await Promise.all(documentids.map(documentid =>
  516. client.collections.addDocument({
  517. id: folderid,
  518. documentId: documentid
  519. }).catch(error => {
  520. console.error("移动文件时发生错误:", error);
  521. })
  522. ));
  523. for (const id of ids) {
  524. p = [id, folderid];
  525. p.unshift(_mysqlLabor[0], _mysqlLabor[1], "moveFile");
  526. await mysql.usselect(p, async function (ret) { });
  527. }
  528. res.status(200).json({ messages: "移动成功" });
  529. } catch (error) {
  530. // 这里不返回错误,确保deleteFile执行完
  531. console.error("处理过程中发生错误:", error);
  532. }
  533. }
  534. });
  535. //删除文件
  536. router.route("/deleteFile").all(async (req, res, next) => {
  537. if (req.body[0]) {
  538. let documentids = decodeURIComponent(req.body[0].documentids).split(",");
  539. let ids = req.body[0].ids;
  540. // 使用 Promise.all 确保所有删除操作都被执行
  541. Promise.all(documentids.map(documentid =>
  542. client.documents.delete({ id: documentid }).catch(error => {
  543. console.error("删除文件时发生错误:", error);
  544. })
  545. ));
  546. p = [ids];
  547. p.unshift(_mysqlLabor[0], _mysqlLabor[1], "deleteFile");
  548. mysql.usselect(p, async function (ret) {
  549. res.end(JSON.stringify(ret));
  550. });
  551. }
  552. });
  553. //删除文件
  554. router.route("/deleteFileByFilter").all(async (req, res, next) => {
  555. if (req.body[0]) {
  556. let documentids = decodeURIComponent(req.body[0].documentids).split(",");
  557. let ids = req.body[0].ids;
  558. client.documents.deleteByFilter({ filters: {"document_id": { "$in": documentids }} }).then(() => {
  559. console.log("文件删除成功");
  560. }).catch(error => {
  561. console.error("删除文件时发生错误:", error);
  562. })
  563. p = [ids];
  564. p.unshift(_mysqlLabor[0], _mysqlLabor[1], "deleteFile");
  565. mysql.usselect(p, async function (ret) {
  566. res.end(JSON.stringify(ret));
  567. });
  568. }
  569. });
  570. router.route("/getFolder").all((req, res, next) => {
  571. getmysql2(req, res, "getFolder", req.body.uid);
  572. });
  573. router.route("/getFolder2").all((req, res, next) => {
  574. var json = queryString(req.url);
  575. getmysql2(req, res, "getFolder", json['uid']);
  576. });
  577. router.route("/getFolderPage").all((req, res, next) => {
  578. getmysql2(req, res, "getFolderPage2", req.body[0].uid, req.body[0].n, req.body[0].sub, req.body[0].class, req.body[0].page, req.body[0].num);
  579. });
  580. router.route("/getFolderPage3").all((req, res, next) => {
  581. getmysql2(req, res, "getFolderPage3", req.body[0].uid, req.body[0].n, req.body[0].sub, req.body[0].class, req.body[0].tag, req.body[0].page, req.body[0].num);
  582. });
  583. router.route("/getFolderPage32").all((req, res, next) => {
  584. var json = queryString(req.url);
  585. getmysql2(req, res, "getFolderPage3", json['uid'], json['n'], json['sub'], json['class'], json['tag'], json['page'], json["num"]);
  586. });
  587. router.route("/getFolderPageOpen").all((req, res, next) => {
  588. getmysql2(req, res, "getFolderPageOpen", req.body[0].uid, req.body[0].n, req.body[0].sub, req.body[0].class, req.body[0].tag, req.body[0].page, req.body[0].num);
  589. });
  590. router.route("/haveFile").all((req, res, next) => {
  591. postmysql(req, res, "haveFile");
  592. });
  593. router.route("/haveFolder").all((req, res, next) => {
  594. postmysql(req, res, "haveFolder");
  595. });
  596. //search文件
  597. router.route("/searchFile").all(async (req, res, next) => {
  598. if (req.body) {
  599. try {
  600. let filters = {}
  601. if (req.body.type == 1) {
  602. filters = { "document_id": { "$in": req.body.ids } }
  603. } else if (req.body.type == 2) {
  604. filters = { "collection_ids": { "$in": req.body.ids } }
  605. }
  606. const response = await client.retrieval.search({
  607. query: req.body.query,
  608. search_settings: {
  609. filters: filters,
  610. useSemanticSearch: true,
  611. chunkSettings: {
  612. limit: req.body.limit ? req.body.limit : 10,
  613. enabled: true
  614. },
  615. }
  616. });
  617. res.status(200).json({ result: response.results });
  618. } catch (error) {
  619. res.status(500).json({ error: error instanceof Error ? error.message : "An error occurred" });
  620. }
  621. }
  622. });
  623. //提取文件实体
  624. router.route("/extractFile").all(async (req, res, next) => {
  625. if (req.body) {
  626. try {
  627. const response = client.documents.extract({
  628. id: req.body.id
  629. })
  630. res.status(200).json({ result: response.results });
  631. } catch (error) {
  632. res.status(500).json({ error: error instanceof Error ? error.message : "An error occurred" });
  633. }
  634. }
  635. });
  636. //添加上传好的知识库文件
  637. router.route("/addFile").all((req, res, next) => {
  638. getmysql2(req, res, "add_file", req.body.n, req.body.did, req.body.uid, req.body.fid, req.body.mofid);
  639. })
  640. //获取文件详情
  641. router.route("/getFileDetail").all(async (req, res, next) => {
  642. if (req.body[0]) {
  643. p = Object.values(req.body[0]);
  644. p.unshift(_mysqlLabor[0], _mysqlLabor[1], "getFileDetail");
  645. mysql.usselect(p, async function (ret) {
  646. let data = ret[0][0];
  647. // const array = data.map(item => item.documentid);
  648. try {
  649. const response = await client.documents.list({ ids: [req.body[0].documentId] });
  650. // console.log(response);
  651. let data2 = response.results[0]
  652. // 合并data和data2,根据documentid
  653. const mergedData = { ...data, ...data2 };
  654. res.status(200).json({ result: mergedData });
  655. } catch (error) {
  656. res.status(500).json({ error: error instanceof Error ? error.message : "An error occurred" });
  657. }
  658. });
  659. }
  660. });
  661. //获取切片
  662. router.route("/getChunks").all(async (req, res, next) => {
  663. if (req.body[0]) {
  664. try {
  665. const { page = 1, limit = 10, documentId } = req.body[0];
  666. const offset = (page - 1) * limit;
  667. const response = await client.documents.listChunks({
  668. id: documentId,
  669. includeVectors: false,
  670. offset,
  671. limit,
  672. });
  673. res.status(200).json({ result: response });
  674. } catch (error) {
  675. res.status(500).json({ error: error instanceof Error ? error.message : "An error occurred" });
  676. }
  677. }
  678. });
  679. //获取实体
  680. router.route("/getEntities").all(async (req, res, next) => {
  681. if (req.body[0]) {
  682. try {
  683. const { page = 1, limit = 10, documentId } = req.body[0];
  684. const offset = (page - 1) * limit;
  685. const response = await client.documents.listEntities({
  686. id: documentId,
  687. includeVectors: false,
  688. offset,
  689. limit,
  690. });
  691. res.status(200).json({ result: response });
  692. } catch (error) {
  693. res.status(500).json({ error: error instanceof Error ? error.message : "An error occurred" });
  694. }
  695. }
  696. });
  697. //获取关系
  698. router.route("/getRelationships").all(async (req, res, next) => {
  699. if (req.body[0]) {
  700. try {
  701. const { page = 1, limit = 10, documentId } = req.body[0];
  702. const offset = (page - 1) * limit;
  703. const response = await client.documents.listRelationships({
  704. id: documentId,
  705. includeVectors: false,
  706. offset,
  707. limit,
  708. });
  709. res.status(200).json({ result: response });
  710. } catch (error) {
  711. res.status(500).json({ error: error instanceof Error ? error.message : "An error occurred" });
  712. }
  713. }
  714. });
  715. //提取整个文件夹的关系
  716. router.route("/resetGraphs").all(async (req, res, next) => {
  717. if (req.body[0]) {
  718. try {
  719. // const response = await client.graphs.pull({
  720. // collectionId: req.body[0].collectionId,
  721. // });
  722. const response = await client.graphs.reset({
  723. collectionId: req.body[0].collectionId,
  724. });
  725. res.status(200).json({ result: response });
  726. } catch (error) {
  727. res.status(500).json({ error: error instanceof Error ? error.message : "An error occurred" });
  728. }
  729. }
  730. });
  731. //获取整个文件夹的关系
  732. router.route("/getRelationshipsGraphs").all(async (req, res, next) => {
  733. if (req.body[0]) {
  734. try {
  735. await client.graphs.pull({
  736. collectionId: req.body[0].collectionId,
  737. });
  738. const response = await client.graphs.listRelationships({
  739. collectionId: req.body[0].collectionId,
  740. offset: req.body[0].page ? req.body[0].page : 0,
  741. limit: 1000
  742. });
  743. res.status(200).json({ result: response });
  744. } catch (error) {
  745. res.status(500).json({ error: error instanceof Error ? error.message : "An error occurred" });
  746. }
  747. }
  748. });
  749. //提取知识图谱
  750. router.route("/getGraphs").all(async (req, res, next) => {
  751. console.log(req.body);
  752. //mode的处理
  753. if (req.body[0]) {
  754. try {
  755. //创建临时文件夹
  756. const folder = await client.collections.create({
  757. name: "临时文件夹",
  758. description: "临时文件夹"
  759. });
  760. let folderid = folder.results.id
  761. console.log(folder.results.id);
  762. //将要提取的文件添加到临时文件夹
  763. let documentids = decodeURIComponent(req.body[0].fileids).split(",");
  764. // 使用 Promise.all 确保所有添加操作都被执行
  765. await Promise.all(documentids.map(documentid =>
  766. client.collections.addDocument({
  767. id: folderid,
  768. documentId: documentid
  769. }).catch(error => {
  770. console.error("移动文件时发生错误:", error);
  771. })
  772. ));
  773. //将临时文件夹提取知识图谱
  774. await client.graphs.pull({
  775. collectionId: folderid
  776. });
  777. let offset = 0;
  778. let results = [];
  779. let totalEntries = 0;
  780. do {
  781. const response = await client.graphs.listRelationships({
  782. collectionId: folderid,
  783. offset: offset,
  784. limit: 1000
  785. });
  786. results = results.concat(response.results);
  787. totalEntries = response.totalEntries;
  788. offset += 1000;
  789. } while (offset < totalEntries);
  790. //移除临时文件夹的文件
  791. const files = await client.collections.listDocuments({ id: folderid, offset: 0, limit: 999 });
  792. await Promise.all(files.results.map(file =>
  793. client.collections.removeDocument({ id: folderid, documentId: file.id })
  794. ));
  795. //删除临时文件夹
  796. await client.collections.delete({ id: folderid });
  797. res.status(200).json({ result: results });
  798. } catch (error) {
  799. res.status(500).json({ error: error instanceof Error ? error.message : "An error occurred" });
  800. }
  801. }
  802. });
  803. //提取知识图谱
  804. router.route("/getGraphs2").all(async (req, res, next) => {
  805. console.log(req.body);
  806. //mode的处理
  807. if (req.body[0]) {
  808. try {
  809. let documentIds = decodeURIComponent(req.body[0].fileids).split(",");
  810. // 使用 Promise.all 并通过 map 返回新的 results 数组
  811. let results = await Promise.all(documentIds.map(async documentId => {
  812. const response = await client.documents.listRelationships({
  813. id: documentId,
  814. includeVectors: false,
  815. offset: 0,
  816. limit: 1000, // 修正这里的参数
  817. });
  818. return response.results;
  819. }));
  820. // 使用 flat() 合并所有数组
  821. results = results.flat();
  822. res.status(200).json({ result: results });
  823. } catch (error) {
  824. res.status(500).json({ error: error instanceof Error ? error.message : "An error occurred" });
  825. }
  826. }
  827. });
  828. //获取实体和关系
  829. router.route("/getEntitiesAndRelationships").all(async (req, res, next) => {
  830. if (req.body[0]) {
  831. try {
  832. const { documentId } = req.body[0];
  833. const offset = (page - 1) * limit;
  834. const response = await client.documents.listEntities({
  835. id: documentId,
  836. includeVectors: false,
  837. });
  838. const response2 = await client.documents.listRelationships({
  839. id: documentId,
  840. includeVectors: false,
  841. });
  842. res.status(200).json({ Entities: response, Relationships: response2 });
  843. } catch (error) {
  844. res.status(500).json({ error: error instanceof Error ? error.message : "An error occurred" });
  845. }
  846. }
  847. });
  848. //提取功能
  849. router.route("/extractFile2").all(async (req, res, next) => {
  850. try {
  851. let setting = {}
  852. if(req.body[0].prompt){
  853. setting.graph_extraction_prompt = decodeURIComponent(req.body[0].prompt)
  854. }
  855. console.log(setting);
  856. const response = await client.documents.extract({ id: req.body[0].documentid, setting });
  857. console.log(response);
  858. res.status(200).json({ result: response.results });
  859. } catch (error) {
  860. res.status(500).json({ error: error instanceof Error ? error.message : "An error occurred" });
  861. }
  862. });
  863. //复制文件夹
  864. router.route("/copyFolder").all(async (req, res, next) => {
  865. if (req.body[0]) {
  866. const response = await client.collections.create({
  867. name: decodeURIComponent(req.body[0].n),
  868. description: decodeURIComponent(req.body[0].fd)
  869. });
  870. let folderid = response.results.id;
  871. p2 = [req.body[0].folderid]
  872. p2.unshift(_mysqlLabor[0], _mysqlLabor[1], "selectFolderFile");
  873. mysql.usselect(p2, async function (ret2) {
  874. let file = ret2[0]
  875. if (file.length) {
  876. let documentids = file.map(item => item.documentid);
  877. // 使用 Promise.all 确保所有添加操作都被执行
  878. await Promise.all(documentids.map(documentid =>
  879. client.collections.addDocument({
  880. id: folderid,
  881. documentId: documentid
  882. }).catch(error => {
  883. console.error("移动文件时发生错误:", error);
  884. })
  885. ));
  886. }
  887. let ids = file.map(item => item.fileid);
  888. console.log(ids);
  889. p = [req.body[0].n, req.body[0].fd, req.body[0].uid, 2, folderid, req.body[0].folderid, ids.join(",")];
  890. p.unshift(_mysqlLabor[0], _mysqlLabor[1], "addFolder2");
  891. mysql.usselect(p, async function (ret) {
  892. res.end(JSON.stringify(ret));
  893. });
  894. });
  895. }
  896. });
  897. // 下载文件功能
  898. router.route("/downloadFile").all(async (req, res, next) => {
  899. var json = queryString(req.url);
  900. let params = {
  901. documentid: (json && json['documentid'] || req.body[0].documentid),
  902. filename: (json && json['filename'] || req.body[0].filename),
  903. };
  904. try {
  905. const response = await client.documents.download({ id: params.documentid });
  906. console.log(response);
  907. // 根据response的类型设置Content-Type
  908. const contentType = response.type || 'application/octet-stream';
  909. res.setHeader('Content-Type', contentType);
  910. res.setHeader('Content-Disposition', `attachment; filename="${encodeURIComponent(params.filename)}"`);
  911. // 直接发送blob数据
  912. res.send(Buffer.from(await response.arrayBuffer()));
  913. } catch (error) {
  914. console.error("下载文件时发生错误:", error);
  915. res.status(500).json({ error: error instanceof Error ? error.message : "发生错误" });
  916. }
  917. });
  918. // 下载文件功能
  919. router.route("/downloadFile2").all(async (req, res, next) => {
  920. var json = queryString(req.url);
  921. let params = {
  922. documentid: (json && json['documentid'] || req.body[0].documentid),
  923. filename: (json && json['filename'] || req.body[0].filename),
  924. };
  925. try {
  926. const response = await client.documents.download({ id: params.documentid });
  927. console.log(response);
  928. // 直接返回blob数据
  929. res.setHeader('Content-Type', response.type || 'application/octet-stream');
  930. res.setHeader('Content-Disposition', `attachment; filename="${encodeURIComponent(params.filename)}"`);
  931. const buffer = Buffer.from(await response.arrayBuffer());
  932. res.send(buffer); // 发送blob数据
  933. } catch (error) {
  934. console.error("下载文件时发生错误:", error);
  935. res.status(500).json({ error: error instanceof Error ? error.message : "发生错误" });
  936. }
  937. });
  938. //召回测试
  939. router.route("/ragFolder").all(async (req, res, next) => {
  940. let reqJson = req.body[0]
  941. if (reqJson) {
  942. reqJson.query = decodeURIComponent(reqJson.query)
  943. try {
  944. let filters = {}
  945. if (reqJson.type == 1) {
  946. filters = { "document_id": { "$in": reqJson.ids.split(",") } }
  947. } else if (reqJson.type == 2) {
  948. filters = { "collection_ids": { "$in": reqJson.ids.split(",") } }
  949. } else if (reqJson.type == 3) {
  950. filters = { "collection_ids": { "$eq": reqJson.ids } }
  951. }
  952. console.log(111);
  953. const response = await client.retrieval.search({
  954. query: reqJson.query,
  955. // searchMode: reqJson.useType == 1 ? 'basic' : reqJson.useType == 2 ? 'custom' : 'advanced',
  956. // searchMode: 'basic',
  957. searchSettings: {
  958. useHybridSearch: reqJson.useType == 3,
  959. useSemanticSearch: reqJson.useType == 1 || reqJson.useType == 2 || reqJson.useType == 3,
  960. useFullTextSearch: reqJson.useType == 2 || reqJson.useType == 3,
  961. hybrid_settings: {
  962. fullTextWeight: reqJson.fullTextWeight ? reqJson.fullTextWeight : 1,
  963. semanticWeight: reqJson.semanticWeight ? reqJson.semanticWeight : 5,
  964. },
  965. filters: filters,
  966. limit: reqJson.limit ? reqJson.limit : 100,
  967. },
  968. ragGenerationConfig: {
  969. stream: false,
  970. temperature: reqJson.temperature ? reqJson.temperature : 0,
  971. maxTokens: reqJson.tokens ? reqJson.tokens : 5000
  972. }
  973. });
  974. // res.status(200).json({ result: response.results });
  975. p = [reqJson.userid, reqJson.ids, encodeURIComponent(JSON.stringify(reqJson)), encodeURIComponent(JSON.stringify(response.results))];
  976. p.unshift(_mysqlLabor[0], _mysqlLabor[1], "addRagData");
  977. mysql.usselect(p, async function (ret) {
  978. res.status(200).json({ result: response.results });
  979. });
  980. } catch (error) {
  981. console.log(error);
  982. res.status(500).json({ error: error instanceof Error ? error.message : "An error occurred" });
  983. }
  984. }
  985. });
  986. //召回测试
  987. router.route("/searchFolder").all(async (req, res, next) => {
  988. let reqJson = req.body[0]
  989. if (reqJson) {
  990. try {
  991. let filters = {}
  992. let documentids = reqJson.documentids ? decodeURIComponent(reqJson.documentids).split(",").filter(id => id) : [];
  993. let folderids = reqJson.folderids ? decodeURIComponent(reqJson.folderids).split(",").filter(id => id) : [];
  994. filters = {
  995. "$or": [
  996. { "collection_ids": { "$in": folderids } },
  997. { "document_id": { "$in": documentids } }
  998. ]
  999. };
  1000. let params = {
  1001. query: reqJson.query,
  1002. searchSettings: {
  1003. useHybridSearch: reqJson.useType == 3,
  1004. useSemanticSearch: reqJson.useType == 1 || reqJson.useType == 2 || reqJson.useType == 3,
  1005. useFullTextSearch: reqJson.useType == 1 || reqJson.useType == 2 || reqJson.useType == 3,
  1006. hybrid_settings: {
  1007. fullTextWeight: reqJson.fullTextWeight ? reqJson.fullTextWeight : 1,
  1008. semanticWeight: reqJson.semanticWeight ? reqJson.semanticWeight : 5,
  1009. },
  1010. filters: filters,
  1011. limit: reqJson.limit ? reqJson.limit : 100,
  1012. },
  1013. ragGenerationConfig: {
  1014. stream: false,
  1015. temperature: reqJson.temperature ? reqJson.temperature : 0,
  1016. maxTokens: reqJson.tokens ? reqJson.tokens : 5000
  1017. }
  1018. }
  1019. // console.log(params);
  1020. const response = await client.retrieval.search(params);
  1021. res.status(200).json({ result: response.results });
  1022. } catch (error) {
  1023. console.log(error);
  1024. res.status(500).json({ error: error instanceof Error ? error.message : "An error occurred" });
  1025. }
  1026. }
  1027. });
  1028. //召回测试
  1029. router.route("/searchFolder2").all(async (req, res, next) => {
  1030. let reqJson = req.body
  1031. if (Object.keys(reqJson).length) {
  1032. try {
  1033. let filters = {}
  1034. let documentids = reqJson.documentids ? reqJson.documentids.split(",").filter(id => id) : [];
  1035. let folderids = reqJson.folderids ? reqJson.folderids.split(",").filter(id => id) : [];
  1036. filters = {
  1037. "$or": [
  1038. { "collection_ids": { "$in": folderids } },
  1039. { "document_id": { "$in": documentids } }
  1040. ]
  1041. };
  1042. if(!folderids.length && !documentids.length){
  1043. res.status(200).json({ result: [] });
  1044. return
  1045. }
  1046. let params = {
  1047. query: reqJson.query,
  1048. searchSettings: {
  1049. useHybridSearch: reqJson.useType == 3,
  1050. useSemanticSearch: reqJson.useType == 1 || reqJson.useType == 2 || reqJson.useType == 3,
  1051. useFullTextSearch: reqJson.useType == 1 || reqJson.useType == 2 || reqJson.useType == 3,
  1052. hybrid_settings: {
  1053. fullTextWeight: reqJson.fullTextWeight ? reqJson.fullTextWeight : 1,
  1054. semanticWeight: reqJson.semanticWeight ? reqJson.semanticWeight : 5,
  1055. },
  1056. filters: filters,
  1057. limit: reqJson.limit ? reqJson.limit : 100,
  1058. },
  1059. ragGenerationConfig: {
  1060. stream: false,
  1061. temperature: reqJson.temperature ? reqJson.temperature : 0,
  1062. maxTokens: reqJson.tokens ? reqJson.tokens : 5000
  1063. }
  1064. }
  1065. // console.log(params);
  1066. const response = await client.retrieval.search(params);
  1067. res.status(200).json({ result: response.results });
  1068. } catch (error) {
  1069. console.log(error);
  1070. res.status(500).json({ error: error instanceof Error ? error.message : "An error occurred" });
  1071. }
  1072. }else{
  1073. res.status(200).json({ result: "请传参" });
  1074. }
  1075. });
  1076. //召回测试
  1077. router.route("/searchFolder3").all(async (req, res, next) => {
  1078. let reqJson = req.body[0]
  1079. if (reqJson) {
  1080. reqJson.query = decodeURIComponent(reqJson.query)
  1081. try {
  1082. let filters = {}
  1083. let documentids = reqJson.documentids ? decodeURIComponent(reqJson.documentids).split(",").filter(id => id) : [];
  1084. let folderids = reqJson.folderids ? decodeURIComponent(reqJson.folderids).split(",").filter(id => id) : [];
  1085. filters = {
  1086. "$or": [
  1087. { "collection_ids": { "$in": folderids } },
  1088. { "document_id": { "$in": documentids } }
  1089. ]
  1090. };
  1091. if(!folderids.length && !documentids.length){
  1092. res.status(200).json({ result: [] });
  1093. return
  1094. }
  1095. let params = {
  1096. query: reqJson.query,
  1097. searchSettings: {
  1098. useHybridSearch: reqJson.useType == 3,
  1099. useSemanticSearch: reqJson.useType == 1 || reqJson.useType == 2 || reqJson.useType == 3,
  1100. useFullTextSearch: reqJson.useType == 1 || reqJson.useType == 2 || reqJson.useType == 3,
  1101. hybrid_settings: {
  1102. fullTextWeight: reqJson.fullTextWeight,
  1103. semanticWeight: reqJson.semanticWeight,
  1104. },
  1105. filters: filters,
  1106. limit: reqJson.limit ? reqJson.limit : 100,
  1107. },
  1108. ragGenerationConfig: {
  1109. stream: false,
  1110. temperature: reqJson.temperature ? reqJson.temperature : 0,
  1111. maxTokens: reqJson.tokens ? reqJson.tokens : 5000
  1112. }
  1113. }
  1114. // console.log(params);
  1115. const response = await client.retrieval.search(params);
  1116. p = [reqJson.userid, reqJson.folderids, encodeURIComponent(JSON.stringify(reqJson)), encodeURIComponent(JSON.stringify(response.results))];
  1117. p.unshift(_mysqlLabor[0], _mysqlLabor[1], "addRagData");
  1118. mysql.usselect(p, async function (ret) {
  1119. res.status(200).json({ result: response.results });
  1120. });
  1121. } catch (error) {
  1122. console.log(error);
  1123. res.status(500).json({ error: error instanceof Error ? error.message : "An error occurred" });
  1124. }
  1125. }
  1126. });
  1127. //召回测试
  1128. router.route("/searchFolder32").all(async (req, res, next) => {
  1129. let reqJson = req.body[0]
  1130. if (reqJson) {
  1131. reqJson.query = decodeURIComponent(reqJson.query)
  1132. try {
  1133. let filters = {}
  1134. let documentids = reqJson.documentids ? decodeURIComponent(reqJson.documentids).split(",").filter(id => id) : [];
  1135. let folderids = reqJson.folderids ? decodeURIComponent(reqJson.folderids).split(",").filter(id => id) : [];
  1136. filters = {
  1137. "$or": [
  1138. { "collection_ids": { "$in": folderids } },
  1139. { "document_id": { "$in": documentids } }
  1140. ]
  1141. };
  1142. if(!folderids.length && !documentids.length){
  1143. res.status(200).json({ result: [] });
  1144. return
  1145. }
  1146. let params = {
  1147. query: reqJson.query,
  1148. searchSettings: {
  1149. useHybridSearch: reqJson.useType == 3,
  1150. useSemanticSearch: reqJson.useType == 1 || reqJson.useType == 2 || reqJson.useType == 3,
  1151. useFullTextSearch: reqJson.useType == 1 || reqJson.useType == 2 || reqJson.useType == 3,
  1152. hybrid_settings: {
  1153. fullTextWeight: reqJson.fullTextWeight,
  1154. semanticWeight: reqJson.semanticWeight,
  1155. },
  1156. filters: filters,
  1157. limit: reqJson.limit ? reqJson.limit : 100,
  1158. },
  1159. ragGenerationConfig: {
  1160. stream: false,
  1161. temperature: reqJson.temperature ? reqJson.temperature : 0,
  1162. maxTokens: reqJson.tokens ? reqJson.tokens : 5000
  1163. }
  1164. }
  1165. console.log(params);
  1166. const response = await client.retrieval.search(params);
  1167. res.status(200).json({ result: response.results });
  1168. // p = [reqJson.userid, reqJson.folderids, encodeURIComponent(JSON.stringify(reqJson)), encodeURIComponent(JSON.stringify(response.results))];
  1169. // p.unshift(_mysqlLabor[0], _mysqlLabor[1], "addRagData");
  1170. // mysql.usselect(p, async function (ret) {
  1171. // });
  1172. } catch (error) {
  1173. console.log(error);
  1174. res.status(500).json({ error: error instanceof Error ? error.message : "An error occurred" });
  1175. }
  1176. }
  1177. });
  1178. //获取召回测试数据
  1179. router.route("/getRagData").all(async (req, res, next) => {
  1180. //mode的处理
  1181. if (req.body[0]) {
  1182. p = Object.values(req.body[0]);
  1183. p.unshift(_mysqlLabor[0], _mysqlLabor[1], "getRagData");
  1184. mysql.usselect(p, async function (ret) {
  1185. res.status(200).json(ret);
  1186. });
  1187. }
  1188. });
  1189. //删除召回测试数据
  1190. router.route("/delRagData").all(async (req, res, next) => {
  1191. //mode的处理
  1192. if (req.body[0]) {
  1193. p = Object.values(req.body[0]);
  1194. p.unshift(_mysqlLabor[0], _mysqlLabor[1], "delRagData");
  1195. mysql.usselect(p, async function (ret) {
  1196. res.status(200).json(ret);
  1197. });
  1198. }
  1199. });
  1200. function generateAPIKey(username) {
  1201. let timeStamp = new Date().getTime();
  1202. let hashStr = timeStamp + username;
  1203. let digest = crypto
  1204. .createHash("sha256")
  1205. .update(hashStr, "utf8")
  1206. .digest("hex");
  1207. return digest;
  1208. }
  1209. queryString = function (url) {
  1210. var a = url.split("?");
  1211. var json = querystring.parse(a[1]);
  1212. return json;
  1213. };
  1214. //post存儲過程處理
  1215. postmysql = function (req, res, functionname) {
  1216. //mode的處理
  1217. if (req.body[0]) {
  1218. p = Object.values(req.body[0]);
  1219. p.unshift(_mysqlLabor[0], _mysqlLabor[1], functionname);
  1220. //執行存儲過程
  1221. mysql.usselect(p, function (ret) {
  1222. res.end(JSON.stringify(ret));
  1223. });
  1224. }
  1225. };
  1226. //post存儲過程處理
  1227. postmysql2 = function (req, res, functionname) {
  1228. //mode的處理
  1229. if (req.body) {
  1230. p = Object.values(req.body)[0].split(",");
  1231. p.unshift(_mysqlLabor[0], _mysqlLabor[1], functionname);
  1232. //執行存儲過程
  1233. mysql.usselect(p, function (ret) {
  1234. res.end(JSON.stringify(ret));
  1235. });
  1236. }
  1237. };
  1238. //post存儲過程處理
  1239. postmysql3 = function (req, res, functionname) {
  1240. //mode的處理
  1241. if (req.body) {
  1242. console.log(req);
  1243. p = Object.values(req.body);
  1244. p.unshift(_mysqlLabor[0], _mysqlLabor[1], functionname);
  1245. //執行存儲過程
  1246. mysql.usselect(p, function (ret) {
  1247. res.end(JSON.stringify(ret));
  1248. });
  1249. }
  1250. };
  1251. //get獲取數據庫信息
  1252. getmysql = function (req, res, functionname, ...p) {
  1253. p = p || [];
  1254. p.unshift(_getmysqlLabor[0], _getmysqlLabor[1], functionname);
  1255. //執行存儲過程
  1256. mysql.usselect(p, function (ret) {
  1257. res.end(JSON.stringify(ret));
  1258. }); //
  1259. };
  1260. //get獲取數據庫信息
  1261. getmysql2 = function (req, res, functionname, ...p) {
  1262. p = p || [];
  1263. p.unshift(_mysqlLabor[0], _mysqlLabor[1], functionname);
  1264. //執行存儲過程
  1265. mysql.usselect(p, function (ret) {
  1266. res.end(JSON.stringify(ret));
  1267. }); //
  1268. };
  1269. //get獲取數據庫信息
  1270. getmysql3 = function (req, res, functionname, ...p) {
  1271. p = p || [];
  1272. p.unshift(_getmysqlLabor2[0], _getmysqlLabor2[1], functionname);
  1273. //執行存儲過程
  1274. mysql.usselect(p, function (ret) {
  1275. res.end(JSON.stringify(ret));
  1276. }); //
  1277. };
  1278. //post存储过程处理
  1279. userpostmysql = function (req, res, functionname) {
  1280. //mode的处理
  1281. if (req.body.mode) {
  1282. p = req.body.mode.split(",");
  1283. p.unshift(_mysqluser[0], _mysqluser[1], functionname);
  1284. //执行存储过程
  1285. mysql.usselect(p, function (ret) {
  1286. res.end(JSON.stringify(ret));
  1287. });
  1288. }
  1289. };
  1290. //get获取数据库信息
  1291. usergetmysql = function (req, res, functionname, ...p) {
  1292. p = p || [];
  1293. p.unshift(_mysqluser[0], _mysqluser[1], functionname);
  1294. //执行存储过程
  1295. mysql.usselect(p, function (ret) {
  1296. res.end(JSON.stringify(ret));
  1297. }); //
  1298. };
  1299. //post存儲過程處理
  1300. postmysqlLocal = function (req, res) {
  1301. //mode的處理
  1302. if (req.body[0]) {
  1303. p = Object.values(req.body[0]);
  1304. p.unshift(_localhost[0], _localhost[1]);
  1305. //執行存儲過程
  1306. mysql.usselect(p, function (ret) {
  1307. res.end(JSON.stringify(ret));
  1308. });
  1309. }
  1310. };
  1311. postmysqlOnline = function (req, res) {
  1312. //mode的處理
  1313. if (req.body[0]) {
  1314. p = Object.values(req.body[0]);
  1315. p.unshift(_mysqlLabor[0], _mysqlLabor[1]);
  1316. //執行存儲過程
  1317. mysql.usselect(p, function (ret) {
  1318. res.end(JSON.stringify(ret));
  1319. });
  1320. }
  1321. };
  1322. // 新增测试接口,直接返回成功
  1323. router.post('/knowledge-base/html-to-markdown', upload.none(), async (req, res) => {
  1324. // console.log('收到的 req.body:', req.body);
  1325. // console.log('收到的 req.files:', req.files);
  1326. // console.log('收到的 headers:', req.headers);
  1327. const requestUrl = req.body.url; // 前端传入 url
  1328. if (!requestUrl) {
  1329. return res.status(400).json({error: 'Missing url'});
  1330. }
  1331. const browser = await initBrowser();
  1332. const page = await browser.newPage();
  1333. try {
  1334. console.log("开始抓取页面:", requestUrl);
  1335. // await page.goto(requestUrl, {waitUntil: 'networkidle'});
  1336. await page.goto(requestUrl);
  1337. const html = await page.content();
  1338. // 新增:HTML 转 Markdown
  1339. const turndownService = new TurndownService();
  1340. // 用 cheerio 只提取正文部分
  1341. const cheerioDoc = cheerio.load(html);
  1342. let mainHtml = cheerioDoc('main').html() || cheerioDoc('article').html() || cheerioDoc('#content').html() || cheerioDoc('.content').html() || cheerioDoc('.post').html() || cheerioDoc('body').html();
  1343. // 用 cheerio 过滤无关标签
  1344. const cheerioMain = cheerio.load(mainHtml || '');
  1345. cheerioMain('style, script, noscript, iframe, link, header, footer, nav, form, .ads, .advertisement').remove();
  1346. mainHtml = cheerioMain.html();
  1347. const markdownContent = turndownService.turndown(mainHtml);
  1348. // 新增:解析 <title> 标签内容
  1349. let title = '';
  1350. const titleMatch = html.match(/<title>([\s\S]*?)<\/title>/i);
  1351. if (titleMatch && titleMatch[1]) {
  1352. title = titleMatch[1].trim();
  1353. }
  1354. let finalTitle;
  1355. if (title && title.trim()) {
  1356. finalTitle = title + '.md';
  1357. } else {
  1358. finalTitle = 'untitled-' + uuidv4() + '.md';
  1359. }
  1360. console.log("抓取到的内容长度:", html.length, "markdown的内容长度:", markdownContent.length);
  1361. // 解析用户传入的参数
  1362. const {
  1363. collection_ids,
  1364. id: customId,
  1365. metadata: customMetadata,
  1366. ingestion_mode,
  1367. run_with_orchestration
  1368. } = req.body;
  1369. // 处理 metadata 字段
  1370. let finalMetadata = customMetadata;
  1371. if (typeof finalMetadata === 'string') {
  1372. try {
  1373. finalMetadata = JSON.parse(finalMetadata);
  1374. } catch (e) {
  1375. return res.status(400).json({ error: 'metadata 不是合法的 JSON 字符串' });
  1376. }
  1377. }
  1378. // if (!finalMetadata) {
  1379. // finalMetadata = {
  1380. // title: title || uuidv4(),
  1381. // // source_url: url,
  1382. // content_type: "markdown",
  1383. // // scraped_at: new Date().toISOString(),
  1384. // // upload_id: Date.now() + '-' + Math.random()
  1385. // };
  1386. // }
  1387. // 处理 collection_ids 字段
  1388. let finalCollectionIds = collection_ids;
  1389. if (typeof finalCollectionIds === 'string') {
  1390. try {
  1391. finalCollectionIds = JSON.parse(finalCollectionIds);
  1392. } catch (e) {
  1393. // 不是合法 JSON 字符串就当作普通字符串
  1394. }
  1395. }
  1396. // 如果 metadata 里没有 collection_ids,但用户传了 collection_ids,则加进去
  1397. if (finalCollectionIds && Array.isArray(finalCollectionIds)) {
  1398. finalMetadata.collection_ids = finalCollectionIds;
  1399. }
  1400. // 补全 finalMetadata.title 和 content_type
  1401. if (!finalMetadata.title) {
  1402. finalMetadata.title = finalTitle;
  1403. }
  1404. finalMetadata.content_type = 'markdown';
  1405. // 用 axios 发送 multipart/form-data 方式上传到 r2r
  1406. const FormData = require('form-data');
  1407. const fs = require('fs');
  1408. const path = require('path');
  1409. const tmpFilePath = path.join(__dirname, `${title || 'untitled'}-${uuidv4()}.md`);
  1410. fs.writeFileSync(tmpFilePath, markdownContent, 'utf8');
  1411. const form = new FormData();
  1412. form.append('file', fs.createReadStream(tmpFilePath), finalTitle);
  1413. if (finalCollectionIds && Array.isArray(finalCollectionIds)) {
  1414. form.append('collection_ids', JSON.stringify(finalCollectionIds));
  1415. }
  1416. if (customId) {
  1417. form.append('id', customId);
  1418. }
  1419. form.append('metadata', JSON.stringify(finalMetadata));
  1420. form.append('ingestion_mode', ingestion_mode || 'fast');
  1421. form.append('run_with_orchestration', 'false');
  1422. // 打印上传参数,方便排查问题
  1423. console.log('Request Headers:', form.getHeaders());
  1424. const url = 'https://r2rserver.cocorobo.cn/v3/documents';
  1425. const response = await axios.post(url, form, {
  1426. headers: {
  1427. ...form.getHeaders(),
  1428. // 'User-Agent': 'NodeClient/1.0'
  1429. }
  1430. });
  1431. fs.unlinkSync(tmpFilePath);
  1432. result = {
  1433. results: response.data.results,
  1434. title: finalTitle
  1435. }
  1436. res.json(result);
  1437. } catch (error) {
  1438. console.error('Message:', error.message);
  1439. console.error('Stack:', error.stack);
  1440. res.status(500).json({error: error.message});
  1441. } finally {
  1442. await page.close(); // 确保页面总是被关闭
  1443. }
  1444. });
  1445. module.exports = router;