CachedInputFileSystem.js 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477
  1. /*
  2. MIT License http://www.opensource.org/licenses/mit-license.php
  3. Author Tobias Koppers @sokra
  4. */
  5. "use strict";
  6. const nextTick = require("process").nextTick;
  7. /** @typedef {import("./Resolver").FileSystem} FileSystem */
  8. /** @typedef {import("./Resolver").SyncFileSystem} SyncFileSystem */
  9. const dirname = path => {
  10. let idx = path.length - 1;
  11. while (idx >= 0) {
  12. const c = path.charCodeAt(idx);
  13. // slash or backslash
  14. if (c === 47 || c === 92) break;
  15. idx--;
  16. }
  17. if (idx < 0) return "";
  18. return path.slice(0, idx);
  19. };
  20. const runCallbacks = (callbacks, err, result) => {
  21. if (callbacks.length === 1) {
  22. callbacks[0](err, result);
  23. callbacks.length = 0;
  24. return;
  25. }
  26. let error;
  27. for (const callback of callbacks) {
  28. try {
  29. callback(err, result);
  30. } catch (e) {
  31. if (!error) error = e;
  32. }
  33. }
  34. callbacks.length = 0;
  35. if (error) throw error;
  36. };
  37. class OperationMergerBackend {
  38. /**
  39. * @param {any} provider async method
  40. * @param {any} syncProvider sync method
  41. * @param {any} providerContext call context for the provider methods
  42. */
  43. constructor(provider, syncProvider, providerContext) {
  44. this._provider = provider;
  45. this._syncProvider = syncProvider;
  46. this._providerContext = providerContext;
  47. this._activeAsyncOperations = new Map();
  48. this.provide = this._provider
  49. ? (path, options, callback) => {
  50. if (typeof options === "function") {
  51. callback = options;
  52. options = undefined;
  53. }
  54. if (options) {
  55. return this._provider.call(
  56. this._providerContext,
  57. path,
  58. options,
  59. callback
  60. );
  61. }
  62. if (typeof path !== "string") {
  63. callback(new TypeError("path must be a string"));
  64. return;
  65. }
  66. let callbacks = this._activeAsyncOperations.get(path);
  67. if (callbacks) {
  68. callbacks.push(callback);
  69. return;
  70. }
  71. this._activeAsyncOperations.set(path, (callbacks = [callback]));
  72. provider(path, (err, result) => {
  73. this._activeAsyncOperations.delete(path);
  74. runCallbacks(callbacks, err, result);
  75. });
  76. }
  77. : null;
  78. this.provideSync = this._syncProvider
  79. ? (path, options) => {
  80. return this._syncProvider.call(this._providerContext, path, options);
  81. }
  82. : null;
  83. }
  84. purge() {}
  85. purgeParent() {}
  86. }
  87. /*
  88. IDLE:
  89. insert data: goto SYNC
  90. SYNC:
  91. before provide: run ticks
  92. event loop tick: goto ASYNC_ACTIVE
  93. ASYNC:
  94. timeout: run tick, goto ASYNC_PASSIVE
  95. ASYNC_PASSIVE:
  96. before provide: run ticks
  97. IDLE --[insert data]--> SYNC --[event loop tick]--> ASYNC_ACTIVE --[interval tick]-> ASYNC_PASSIVE
  98. ^ |
  99. +---------[insert data]-------+
  100. */
  101. const STORAGE_MODE_IDLE = 0;
  102. const STORAGE_MODE_SYNC = 1;
  103. const STORAGE_MODE_ASYNC = 2;
  104. class CacheBackend {
  105. /**
  106. * @param {number} duration max cache duration of items
  107. * @param {any} provider async method
  108. * @param {any} syncProvider sync method
  109. * @param {any} providerContext call context for the provider methods
  110. */
  111. constructor(duration, provider, syncProvider, providerContext) {
  112. this._duration = duration;
  113. this._provider = provider;
  114. this._syncProvider = syncProvider;
  115. this._providerContext = providerContext;
  116. /** @type {Map<string, (function(Error, any): void)[]>} */
  117. this._activeAsyncOperations = new Map();
  118. /** @type {Map<string, { err: Error, result: any, level: Set<string> }>} */
  119. this._data = new Map();
  120. /** @type {Set<string>[]} */
  121. this._levels = [];
  122. for (let i = 0; i < 10; i++) this._levels.push(new Set());
  123. for (let i = 5000; i < duration; i += 500) this._levels.push(new Set());
  124. this._currentLevel = 0;
  125. this._tickInterval = Math.floor(duration / this._levels.length);
  126. /** @type {STORAGE_MODE_IDLE | STORAGE_MODE_SYNC | STORAGE_MODE_ASYNC} */
  127. this._mode = STORAGE_MODE_IDLE;
  128. /** @type {NodeJS.Timeout | undefined} */
  129. this._timeout = undefined;
  130. /** @type {number | undefined} */
  131. this._nextDecay = undefined;
  132. this.provide = provider ? this.provide.bind(this) : null;
  133. this.provideSync = syncProvider ? this.provideSync.bind(this) : null;
  134. }
  135. provide(path, options, callback) {
  136. if (typeof options === "function") {
  137. callback = options;
  138. options = undefined;
  139. }
  140. if (typeof path !== "string") {
  141. callback(new TypeError("path must be a string"));
  142. return;
  143. }
  144. if (options) {
  145. return this._provider.call(
  146. this._providerContext,
  147. path,
  148. options,
  149. callback
  150. );
  151. }
  152. // When in sync mode we can move to async mode
  153. if (this._mode === STORAGE_MODE_SYNC) {
  154. this._enterAsyncMode();
  155. }
  156. // Check in cache
  157. let cacheEntry = this._data.get(path);
  158. if (cacheEntry !== undefined) {
  159. if (cacheEntry.err) return nextTick(callback, cacheEntry.err);
  160. return nextTick(callback, null, cacheEntry.result);
  161. }
  162. // Check if there is already the same operation running
  163. let callbacks = this._activeAsyncOperations.get(path);
  164. if (callbacks !== undefined) {
  165. callbacks.push(callback);
  166. return;
  167. }
  168. this._activeAsyncOperations.set(path, (callbacks = [callback]));
  169. // Run the operation
  170. this._provider.call(this._providerContext, path, (err, result) => {
  171. this._activeAsyncOperations.delete(path);
  172. this._storeResult(path, err, result);
  173. // Enter async mode if not yet done
  174. this._enterAsyncMode();
  175. runCallbacks(callbacks, err, result);
  176. });
  177. }
  178. provideSync(path, options) {
  179. if (typeof path !== "string") {
  180. throw new TypeError("path must be a string");
  181. }
  182. if (options) {
  183. return this._syncProvider.call(this._providerContext, path, options);
  184. }
  185. // In sync mode we may have to decay some cache items
  186. if (this._mode === STORAGE_MODE_SYNC) {
  187. this._runDecays();
  188. }
  189. // Check in cache
  190. let cacheEntry = this._data.get(path);
  191. if (cacheEntry !== undefined) {
  192. if (cacheEntry.err) throw cacheEntry.err;
  193. return cacheEntry.result;
  194. }
  195. // Get all active async operations
  196. // This sync operation will also complete them
  197. const callbacks = this._activeAsyncOperations.get(path);
  198. this._activeAsyncOperations.delete(path);
  199. // Run the operation
  200. // When in idle mode, we will enter sync mode
  201. let result;
  202. try {
  203. result = this._syncProvider.call(this._providerContext, path);
  204. } catch (err) {
  205. this._storeResult(path, err, undefined);
  206. this._enterSyncModeWhenIdle();
  207. if (callbacks) runCallbacks(callbacks, err, undefined);
  208. throw err;
  209. }
  210. this._storeResult(path, undefined, result);
  211. this._enterSyncModeWhenIdle();
  212. if (callbacks) runCallbacks(callbacks, undefined, result);
  213. return result;
  214. }
  215. purge(what) {
  216. if (!what) {
  217. if (this._mode !== STORAGE_MODE_IDLE) {
  218. this._data.clear();
  219. for (const level of this._levels) {
  220. level.clear();
  221. }
  222. this._enterIdleMode();
  223. }
  224. } else if (typeof what === "string") {
  225. for (let [key, data] of this._data) {
  226. if (key.startsWith(what)) {
  227. this._data.delete(key);
  228. data.level.delete(key);
  229. }
  230. }
  231. if (this._data.size === 0) {
  232. this._enterIdleMode();
  233. }
  234. } else {
  235. for (let [key, data] of this._data) {
  236. for (const item of what) {
  237. if (key.startsWith(item)) {
  238. this._data.delete(key);
  239. data.level.delete(key);
  240. break;
  241. }
  242. }
  243. }
  244. if (this._data.size === 0) {
  245. this._enterIdleMode();
  246. }
  247. }
  248. }
  249. purgeParent(what) {
  250. if (!what) {
  251. this.purge();
  252. } else if (typeof what === "string") {
  253. this.purge(dirname(what));
  254. } else {
  255. const set = new Set();
  256. for (const item of what) {
  257. set.add(dirname(item));
  258. }
  259. this.purge(set);
  260. }
  261. }
  262. _storeResult(path, err, result) {
  263. if (this._data.has(path)) return;
  264. const level = this._levels[this._currentLevel];
  265. this._data.set(path, { err, result, level });
  266. level.add(path);
  267. }
  268. _decayLevel() {
  269. const nextLevel = (this._currentLevel + 1) % this._levels.length;
  270. const decay = this._levels[nextLevel];
  271. this._currentLevel = nextLevel;
  272. for (let item of decay) {
  273. this._data.delete(item);
  274. }
  275. decay.clear();
  276. if (this._data.size === 0) {
  277. this._enterIdleMode();
  278. } else {
  279. // @ts-ignore _nextDecay is always a number in sync mode
  280. this._nextDecay += this._tickInterval;
  281. }
  282. }
  283. _runDecays() {
  284. while (
  285. /** @type {number} */ (this._nextDecay) <= Date.now() &&
  286. this._mode !== STORAGE_MODE_IDLE
  287. ) {
  288. this._decayLevel();
  289. }
  290. }
  291. _enterAsyncMode() {
  292. let timeout = 0;
  293. switch (this._mode) {
  294. case STORAGE_MODE_ASYNC:
  295. return;
  296. case STORAGE_MODE_IDLE:
  297. this._nextDecay = Date.now() + this._tickInterval;
  298. timeout = this._tickInterval;
  299. break;
  300. case STORAGE_MODE_SYNC:
  301. this._runDecays();
  302. // @ts-ignore _runDecays may change the mode
  303. if (this._mode === STORAGE_MODE_IDLE) return;
  304. timeout = Math.max(
  305. 0,
  306. /** @type {number} */ (this._nextDecay) - Date.now()
  307. );
  308. break;
  309. }
  310. this._mode = STORAGE_MODE_ASYNC;
  311. const ref = setTimeout(() => {
  312. this._mode = STORAGE_MODE_SYNC;
  313. this._runDecays();
  314. }, timeout);
  315. if (ref.unref) ref.unref();
  316. this._timeout = ref;
  317. }
  318. _enterSyncModeWhenIdle() {
  319. if (this._mode === STORAGE_MODE_IDLE) {
  320. this._mode = STORAGE_MODE_SYNC;
  321. this._nextDecay = Date.now() + this._tickInterval;
  322. }
  323. }
  324. _enterIdleMode() {
  325. this._mode = STORAGE_MODE_IDLE;
  326. this._nextDecay = undefined;
  327. if (this._timeout) clearTimeout(this._timeout);
  328. }
  329. }
  330. const createBackend = (duration, provider, syncProvider, providerContext) => {
  331. if (duration > 0) {
  332. return new CacheBackend(duration, provider, syncProvider, providerContext);
  333. }
  334. return new OperationMergerBackend(provider, syncProvider, providerContext);
  335. };
  336. module.exports = class CachedInputFileSystem {
  337. constructor(fileSystem, duration) {
  338. this.fileSystem = fileSystem;
  339. this._lstatBackend = createBackend(
  340. duration,
  341. this.fileSystem.lstat,
  342. this.fileSystem.lstatSync,
  343. this.fileSystem
  344. );
  345. const lstat = this._lstatBackend.provide;
  346. this.lstat = /** @type {FileSystem["lstat"]} */ (lstat);
  347. const lstatSync = this._lstatBackend.provideSync;
  348. this.lstatSync = /** @type {SyncFileSystem["lstatSync"]} */ (lstatSync);
  349. this._statBackend = createBackend(
  350. duration,
  351. this.fileSystem.stat,
  352. this.fileSystem.statSync,
  353. this.fileSystem
  354. );
  355. const stat = this._statBackend.provide;
  356. this.stat = /** @type {FileSystem["stat"]} */ (stat);
  357. const statSync = this._statBackend.provideSync;
  358. this.statSync = /** @type {SyncFileSystem["statSync"]} */ (statSync);
  359. this._readdirBackend = createBackend(
  360. duration,
  361. this.fileSystem.readdir,
  362. this.fileSystem.readdirSync,
  363. this.fileSystem
  364. );
  365. const readdir = this._readdirBackend.provide;
  366. this.readdir = /** @type {FileSystem["readdir"]} */ (readdir);
  367. const readdirSync = this._readdirBackend.provideSync;
  368. this.readdirSync = /** @type {SyncFileSystem["readdirSync"]} */ (readdirSync);
  369. this._readFileBackend = createBackend(
  370. duration,
  371. this.fileSystem.readFile,
  372. this.fileSystem.readFileSync,
  373. this.fileSystem
  374. );
  375. const readFile = this._readFileBackend.provide;
  376. this.readFile = /** @type {FileSystem["readFile"]} */ (readFile);
  377. const readFileSync = this._readFileBackend.provideSync;
  378. this.readFileSync = /** @type {SyncFileSystem["readFileSync"]} */ (readFileSync);
  379. this._readJsonBackend = createBackend(
  380. duration,
  381. this.fileSystem.readJson ||
  382. (this.readFile &&
  383. ((path, callback) => {
  384. // @ts-ignore
  385. this.readFile(path, (err, buffer) => {
  386. if (err) return callback(err);
  387. if (!buffer || buffer.length === 0)
  388. return callback(new Error("No file content"));
  389. let data;
  390. try {
  391. data = JSON.parse(buffer.toString("utf-8"));
  392. } catch (e) {
  393. return callback(e);
  394. }
  395. callback(null, data);
  396. });
  397. })),
  398. this.fileSystem.readJsonSync ||
  399. (this.readFileSync &&
  400. (path => {
  401. const buffer = this.readFileSync(path);
  402. const data = JSON.parse(buffer.toString("utf-8"));
  403. return data;
  404. })),
  405. this.fileSystem
  406. );
  407. const readJson = this._readJsonBackend.provide;
  408. this.readJson = /** @type {FileSystem["readJson"]} */ (readJson);
  409. const readJsonSync = this._readJsonBackend.provideSync;
  410. this.readJsonSync = /** @type {SyncFileSystem["readJsonSync"]} */ (readJsonSync);
  411. this._readlinkBackend = createBackend(
  412. duration,
  413. this.fileSystem.readlink,
  414. this.fileSystem.readlinkSync,
  415. this.fileSystem
  416. );
  417. const readlink = this._readlinkBackend.provide;
  418. this.readlink = /** @type {FileSystem["readlink"]} */ (readlink);
  419. const readlinkSync = this._readlinkBackend.provideSync;
  420. this.readlinkSync = /** @type {SyncFileSystem["readlinkSync"]} */ (readlinkSync);
  421. }
  422. purge(what) {
  423. this._statBackend.purge(what);
  424. this._lstatBackend.purge(what);
  425. this._readdirBackend.purgeParent(what);
  426. this._readFileBackend.purge(what);
  427. this._readlinkBackend.purge(what);
  428. this._readJsonBackend.purge(what);
  429. }
  430. };