RealContentHashPlugin.js 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408
  1. /*
  2. MIT License http://www.opensource.org/licenses/mit-license.php
  3. Author Tobias Koppers @sokra
  4. */
  5. "use strict";
  6. const { SyncBailHook } = require("tapable");
  7. const { RawSource, CachedSource, CompatSource } = require("webpack-sources");
  8. const Compilation = require("../Compilation");
  9. const WebpackError = require("../WebpackError");
  10. const { compareSelect, compareStrings } = require("../util/comparators");
  11. const createHash = require("../util/createHash");
  12. /** @typedef {import("webpack-sources").Source} Source */
  13. /** @typedef {import("../Compilation").AssetInfo} AssetInfo */
  14. /** @typedef {import("../Compiler")} Compiler */
  15. const EMPTY_SET = new Set();
  16. const addToList = (itemOrItems, list) => {
  17. if (Array.isArray(itemOrItems)) {
  18. for (const item of itemOrItems) {
  19. list.add(item);
  20. }
  21. } else if (itemOrItems) {
  22. list.add(itemOrItems);
  23. }
  24. };
  25. /**
  26. * @template T
  27. * @param {T[]} input list
  28. * @param {function(T): Buffer} fn map function
  29. * @returns {Buffer[]} buffers without duplicates
  30. */
  31. const mapAndDeduplicateBuffers = (input, fn) => {
  32. // Buffer.equals compares size first so this should be efficient enough
  33. // If it becomes a performance problem we can use a map and group by size
  34. // instead of looping over all assets.
  35. const result = [];
  36. outer: for (const value of input) {
  37. const buf = fn(value);
  38. for (const other of result) {
  39. if (buf.equals(other)) continue outer;
  40. }
  41. result.push(buf);
  42. }
  43. return result;
  44. };
  45. /**
  46. * Escapes regular expression metacharacters
  47. * @param {string} str String to quote
  48. * @returns {string} Escaped string
  49. */
  50. const quoteMeta = str => {
  51. return str.replace(/[-[\]\\/{}()*+?.^$|]/g, "\\$&");
  52. };
  53. const cachedSourceMap = new WeakMap();
  54. const toCachedSource = source => {
  55. if (source instanceof CachedSource) {
  56. return source;
  57. }
  58. const entry = cachedSourceMap.get(source);
  59. if (entry !== undefined) return entry;
  60. const newSource = new CachedSource(CompatSource.from(source));
  61. cachedSourceMap.set(source, newSource);
  62. return newSource;
  63. };
  64. /**
  65. * @typedef {Object} AssetInfoForRealContentHash
  66. * @property {string} name
  67. * @property {AssetInfo} info
  68. * @property {Source} source
  69. * @property {RawSource | undefined} newSource
  70. * @property {RawSource | undefined} newSourceWithoutOwn
  71. * @property {string} content
  72. * @property {Set<string>} ownHashes
  73. * @property {Promise} contentComputePromise
  74. * @property {Promise} contentComputeWithoutOwnPromise
  75. * @property {Set<string>} referencedHashes
  76. * @property {Set<string>} hashes
  77. */
  78. /**
  79. * @typedef {Object} CompilationHooks
  80. * @property {SyncBailHook<[Buffer[], string], string>} updateHash
  81. */
  82. /** @type {WeakMap<Compilation, CompilationHooks>} */
  83. const compilationHooksMap = new WeakMap();
  84. class RealContentHashPlugin {
  85. /**
  86. * @param {Compilation} compilation the compilation
  87. * @returns {CompilationHooks} the attached hooks
  88. */
  89. static getCompilationHooks(compilation) {
  90. if (!(compilation instanceof Compilation)) {
  91. throw new TypeError(
  92. "The 'compilation' argument must be an instance of Compilation"
  93. );
  94. }
  95. let hooks = compilationHooksMap.get(compilation);
  96. if (hooks === undefined) {
  97. hooks = {
  98. updateHash: new SyncBailHook(["content", "oldHash"])
  99. };
  100. compilationHooksMap.set(compilation, hooks);
  101. }
  102. return hooks;
  103. }
  104. constructor({ hashFunction, hashDigest }) {
  105. this._hashFunction = hashFunction;
  106. this._hashDigest = hashDigest;
  107. }
  108. /**
  109. * Apply the plugin
  110. * @param {Compiler} compiler the compiler instance
  111. * @returns {void}
  112. */
  113. apply(compiler) {
  114. compiler.hooks.compilation.tap("RealContentHashPlugin", compilation => {
  115. const cacheAnalyse = compilation.getCache(
  116. "RealContentHashPlugin|analyse"
  117. );
  118. const cacheGenerate = compilation.getCache(
  119. "RealContentHashPlugin|generate"
  120. );
  121. const hooks = RealContentHashPlugin.getCompilationHooks(compilation);
  122. compilation.hooks.processAssets.tapPromise(
  123. {
  124. name: "RealContentHashPlugin",
  125. stage: Compilation.PROCESS_ASSETS_STAGE_OPTIMIZE_HASH
  126. },
  127. async () => {
  128. const assets = compilation.getAssets();
  129. /** @type {AssetInfoForRealContentHash[]} */
  130. const assetsWithInfo = [];
  131. const hashToAssets = new Map();
  132. for (const { source, info, name } of assets) {
  133. const cachedSource = toCachedSource(source);
  134. const content = cachedSource.source();
  135. /** @type {Set<string>} */
  136. const hashes = new Set();
  137. addToList(info.contenthash, hashes);
  138. const data = {
  139. name,
  140. info,
  141. source: cachedSource,
  142. /** @type {RawSource | undefined} */
  143. newSource: undefined,
  144. /** @type {RawSource | undefined} */
  145. newSourceWithoutOwn: undefined,
  146. content,
  147. /** @type {Set<string>} */
  148. ownHashes: undefined,
  149. contentComputePromise: undefined,
  150. contentComputeWithoutOwnPromise: undefined,
  151. /** @type {Set<string>} */
  152. referencedHashes: undefined,
  153. hashes
  154. };
  155. assetsWithInfo.push(data);
  156. for (const hash of hashes) {
  157. const list = hashToAssets.get(hash);
  158. if (list === undefined) {
  159. hashToAssets.set(hash, [data]);
  160. } else {
  161. list.push(data);
  162. }
  163. }
  164. }
  165. if (hashToAssets.size === 0) return;
  166. const hashRegExp = new RegExp(
  167. Array.from(hashToAssets.keys(), quoteMeta).join("|"),
  168. "g"
  169. );
  170. await Promise.all(
  171. assetsWithInfo.map(async asset => {
  172. const { name, source, content, hashes } = asset;
  173. if (Buffer.isBuffer(content)) {
  174. asset.referencedHashes = EMPTY_SET;
  175. asset.ownHashes = EMPTY_SET;
  176. return;
  177. }
  178. const etag = cacheAnalyse.mergeEtags(
  179. cacheAnalyse.getLazyHashedEtag(source),
  180. Array.from(hashes).join("|")
  181. );
  182. [asset.referencedHashes, asset.ownHashes] =
  183. await cacheAnalyse.providePromise(name, etag, () => {
  184. const referencedHashes = new Set();
  185. let ownHashes = new Set();
  186. const inContent = content.match(hashRegExp);
  187. if (inContent) {
  188. for (const hash of inContent) {
  189. if (hashes.has(hash)) {
  190. ownHashes.add(hash);
  191. continue;
  192. }
  193. referencedHashes.add(hash);
  194. }
  195. }
  196. return [referencedHashes, ownHashes];
  197. });
  198. })
  199. );
  200. const getDependencies = hash => {
  201. const assets = hashToAssets.get(hash);
  202. if (!assets) {
  203. const referencingAssets = assetsWithInfo.filter(asset =>
  204. asset.referencedHashes.has(hash)
  205. );
  206. const err = new WebpackError(`RealContentHashPlugin
  207. Some kind of unexpected caching problem occurred.
  208. An asset was cached with a reference to another asset (${hash}) that's not in the compilation anymore.
  209. Either the asset was incorrectly cached, or the referenced asset should also be restored from cache.
  210. Referenced by:
  211. ${referencingAssets
  212. .map(a => {
  213. const match = new RegExp(`.{0,20}${quoteMeta(hash)}.{0,20}`).exec(
  214. a.content
  215. );
  216. return ` - ${a.name}: ...${match ? match[0] : "???"}...`;
  217. })
  218. .join("\n")}`);
  219. compilation.errors.push(err);
  220. return undefined;
  221. }
  222. const hashes = new Set();
  223. for (const { referencedHashes, ownHashes } of assets) {
  224. if (!ownHashes.has(hash)) {
  225. for (const hash of ownHashes) {
  226. hashes.add(hash);
  227. }
  228. }
  229. for (const hash of referencedHashes) {
  230. hashes.add(hash);
  231. }
  232. }
  233. return hashes;
  234. };
  235. const hashInfo = hash => {
  236. const assets = hashToAssets.get(hash);
  237. return `${hash} (${Array.from(assets, a => a.name)})`;
  238. };
  239. const hashesInOrder = new Set();
  240. for (const hash of hashToAssets.keys()) {
  241. const add = (hash, stack) => {
  242. const deps = getDependencies(hash);
  243. if (!deps) return;
  244. stack.add(hash);
  245. for (const dep of deps) {
  246. if (hashesInOrder.has(dep)) continue;
  247. if (stack.has(dep)) {
  248. throw new Error(
  249. `Circular hash dependency ${Array.from(
  250. stack,
  251. hashInfo
  252. ).join(" -> ")} -> ${hashInfo(dep)}`
  253. );
  254. }
  255. add(dep, stack);
  256. }
  257. hashesInOrder.add(hash);
  258. stack.delete(hash);
  259. };
  260. if (hashesInOrder.has(hash)) continue;
  261. add(hash, new Set());
  262. }
  263. const hashToNewHash = new Map();
  264. const getEtag = asset =>
  265. cacheGenerate.mergeEtags(
  266. cacheGenerate.getLazyHashedEtag(asset.source),
  267. Array.from(asset.referencedHashes, hash =>
  268. hashToNewHash.get(hash)
  269. ).join("|")
  270. );
  271. const computeNewContent = asset => {
  272. if (asset.contentComputePromise) return asset.contentComputePromise;
  273. return (asset.contentComputePromise = (async () => {
  274. if (
  275. asset.ownHashes.size > 0 ||
  276. Array.from(asset.referencedHashes).some(
  277. hash => hashToNewHash.get(hash) !== hash
  278. )
  279. ) {
  280. const identifier = asset.name;
  281. const etag = getEtag(asset);
  282. asset.newSource = await cacheGenerate.providePromise(
  283. identifier,
  284. etag,
  285. () => {
  286. const newContent = asset.content.replace(hashRegExp, hash =>
  287. hashToNewHash.get(hash)
  288. );
  289. return new RawSource(newContent);
  290. }
  291. );
  292. }
  293. })());
  294. };
  295. const computeNewContentWithoutOwn = asset => {
  296. if (asset.contentComputeWithoutOwnPromise)
  297. return asset.contentComputeWithoutOwnPromise;
  298. return (asset.contentComputeWithoutOwnPromise = (async () => {
  299. if (
  300. asset.ownHashes.size > 0 ||
  301. Array.from(asset.referencedHashes).some(
  302. hash => hashToNewHash.get(hash) !== hash
  303. )
  304. ) {
  305. const identifier = asset.name + "|without-own";
  306. const etag = getEtag(asset);
  307. asset.newSourceWithoutOwn = await cacheGenerate.providePromise(
  308. identifier,
  309. etag,
  310. () => {
  311. const newContent = asset.content.replace(
  312. hashRegExp,
  313. hash => {
  314. if (asset.ownHashes.has(hash)) {
  315. return "";
  316. }
  317. return hashToNewHash.get(hash);
  318. }
  319. );
  320. return new RawSource(newContent);
  321. }
  322. );
  323. }
  324. })());
  325. };
  326. const comparator = compareSelect(a => a.name, compareStrings);
  327. for (const oldHash of hashesInOrder) {
  328. const assets = hashToAssets.get(oldHash);
  329. assets.sort(comparator);
  330. const hash = createHash(this._hashFunction);
  331. await Promise.all(
  332. assets.map(asset =>
  333. asset.ownHashes.has(oldHash)
  334. ? computeNewContentWithoutOwn(asset)
  335. : computeNewContent(asset)
  336. )
  337. );
  338. const assetsContent = mapAndDeduplicateBuffers(assets, asset => {
  339. if (asset.ownHashes.has(oldHash)) {
  340. return asset.newSourceWithoutOwn
  341. ? asset.newSourceWithoutOwn.buffer()
  342. : asset.source.buffer();
  343. } else {
  344. return asset.newSource
  345. ? asset.newSource.buffer()
  346. : asset.source.buffer();
  347. }
  348. });
  349. let newHash = hooks.updateHash.call(assetsContent, oldHash);
  350. if (!newHash) {
  351. for (const content of assetsContent) {
  352. hash.update(content);
  353. }
  354. const digest = hash.digest(this._hashDigest);
  355. newHash = /** @type {string} */ (digest.slice(0, oldHash.length));
  356. }
  357. hashToNewHash.set(oldHash, newHash);
  358. }
  359. await Promise.all(
  360. assetsWithInfo.map(async asset => {
  361. await computeNewContent(asset);
  362. const newName = asset.name.replace(hashRegExp, hash =>
  363. hashToNewHash.get(hash)
  364. );
  365. const infoUpdate = {};
  366. const hash = asset.info.contenthash;
  367. infoUpdate.contenthash = Array.isArray(hash)
  368. ? hash.map(hash => hashToNewHash.get(hash))
  369. : hashToNewHash.get(hash);
  370. if (asset.newSource !== undefined) {
  371. compilation.updateAsset(
  372. asset.name,
  373. asset.newSource,
  374. infoUpdate
  375. );
  376. } else {
  377. compilation.updateAsset(asset.name, asset.source, infoUpdate);
  378. }
  379. if (asset.name !== newName) {
  380. compilation.renameAsset(asset.name, newName);
  381. }
  382. })
  383. );
  384. }
  385. );
  386. });
  387. }
  388. }
  389. module.exports = RealContentHashPlugin;