Compiler.js 35 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230
  1. /*
  2. MIT License http://www.opensource.org/licenses/mit-license.php
  3. Author Tobias Koppers @sokra
  4. */
  5. "use strict";
  6. const parseJson = require("json-parse-even-better-errors");
  7. const asyncLib = require("neo-async");
  8. const {
  9. SyncHook,
  10. SyncBailHook,
  11. AsyncParallelHook,
  12. AsyncSeriesHook
  13. } = require("tapable");
  14. const { SizeOnlySource } = require("webpack-sources");
  15. const webpack = require("./");
  16. const Cache = require("./Cache");
  17. const CacheFacade = require("./CacheFacade");
  18. const ChunkGraph = require("./ChunkGraph");
  19. const Compilation = require("./Compilation");
  20. const ConcurrentCompilationError = require("./ConcurrentCompilationError");
  21. const ContextModuleFactory = require("./ContextModuleFactory");
  22. const ModuleGraph = require("./ModuleGraph");
  23. const NormalModuleFactory = require("./NormalModuleFactory");
  24. const RequestShortener = require("./RequestShortener");
  25. const ResolverFactory = require("./ResolverFactory");
  26. const Stats = require("./Stats");
  27. const Watching = require("./Watching");
  28. const WebpackError = require("./WebpackError");
  29. const { Logger } = require("./logging/Logger");
  30. const { join, dirname, mkdirp } = require("./util/fs");
  31. const { makePathsRelative } = require("./util/identifier");
  32. const { isSourceEqual } = require("./util/source");
  33. /** @typedef {import("webpack-sources").Source} Source */
  34. /** @typedef {import("../declarations/WebpackOptions").EntryNormalized} Entry */
  35. /** @typedef {import("../declarations/WebpackOptions").OutputNormalized} OutputOptions */
  36. /** @typedef {import("../declarations/WebpackOptions").WatchOptions} WatchOptions */
  37. /** @typedef {import("../declarations/WebpackOptions").WebpackOptionsNormalized} WebpackOptions */
  38. /** @typedef {import("../declarations/WebpackOptions").WebpackPluginInstance} WebpackPluginInstance */
  39. /** @typedef {import("./Chunk")} Chunk */
  40. /** @typedef {import("./Dependency")} Dependency */
  41. /** @typedef {import("./FileSystemInfo").FileSystemInfoEntry} FileSystemInfoEntry */
  42. /** @typedef {import("./Module")} Module */
  43. /** @typedef {import("./util/WeakTupleMap")} WeakTupleMap */
  44. /** @typedef {import("./util/fs").InputFileSystem} InputFileSystem */
  45. /** @typedef {import("./util/fs").IntermediateFileSystem} IntermediateFileSystem */
  46. /** @typedef {import("./util/fs").OutputFileSystem} OutputFileSystem */
  47. /** @typedef {import("./util/fs").WatchFileSystem} WatchFileSystem */
  48. /**
  49. * @typedef {Object} CompilationParams
  50. * @property {NormalModuleFactory} normalModuleFactory
  51. * @property {ContextModuleFactory} contextModuleFactory
  52. */
  53. /**
  54. * @template T
  55. * @callback Callback
  56. * @param {(Error | null)=} err
  57. * @param {T=} result
  58. */
  59. /**
  60. * @callback RunAsChildCallback
  61. * @param {(Error | null)=} err
  62. * @param {Chunk[]=} entries
  63. * @param {Compilation=} compilation
  64. */
  65. /**
  66. * @typedef {Object} AssetEmittedInfo
  67. * @property {Buffer} content
  68. * @property {Source} source
  69. * @property {Compilation} compilation
  70. * @property {string} outputPath
  71. * @property {string} targetPath
  72. */
  73. /**
  74. * @param {string[]} array an array
  75. * @returns {boolean} true, if the array is sorted
  76. */
  77. const isSorted = array => {
  78. for (let i = 1; i < array.length; i++) {
  79. if (array[i - 1] > array[i]) return false;
  80. }
  81. return true;
  82. };
  83. /**
  84. * @param {Object} obj an object
  85. * @param {string[]} keys the keys of the object
  86. * @returns {Object} the object with properties sorted by property name
  87. */
  88. const sortObject = (obj, keys) => {
  89. const o = {};
  90. for (const k of keys.sort()) {
  91. o[k] = obj[k];
  92. }
  93. return o;
  94. };
  95. /**
  96. * @param {string} filename filename
  97. * @param {string | string[] | undefined} hashes list of hashes
  98. * @returns {boolean} true, if the filename contains any hash
  99. */
  100. const includesHash = (filename, hashes) => {
  101. if (!hashes) return false;
  102. if (Array.isArray(hashes)) {
  103. return hashes.some(hash => filename.includes(hash));
  104. } else {
  105. return filename.includes(hashes);
  106. }
  107. };
  108. class Compiler {
  109. /**
  110. * @param {string} context the compilation path
  111. * @param {WebpackOptions} options options
  112. */
  113. constructor(context, options = /** @type {WebpackOptions} */ ({})) {
  114. this.hooks = Object.freeze({
  115. /** @type {SyncHook<[]>} */
  116. initialize: new SyncHook([]),
  117. /** @type {SyncBailHook<[Compilation], boolean>} */
  118. shouldEmit: new SyncBailHook(["compilation"]),
  119. /** @type {AsyncSeriesHook<[Stats]>} */
  120. done: new AsyncSeriesHook(["stats"]),
  121. /** @type {SyncHook<[Stats]>} */
  122. afterDone: new SyncHook(["stats"]),
  123. /** @type {AsyncSeriesHook<[]>} */
  124. additionalPass: new AsyncSeriesHook([]),
  125. /** @type {AsyncSeriesHook<[Compiler]>} */
  126. beforeRun: new AsyncSeriesHook(["compiler"]),
  127. /** @type {AsyncSeriesHook<[Compiler]>} */
  128. run: new AsyncSeriesHook(["compiler"]),
  129. /** @type {AsyncSeriesHook<[Compilation]>} */
  130. emit: new AsyncSeriesHook(["compilation"]),
  131. /** @type {AsyncSeriesHook<[string, AssetEmittedInfo]>} */
  132. assetEmitted: new AsyncSeriesHook(["file", "info"]),
  133. /** @type {AsyncSeriesHook<[Compilation]>} */
  134. afterEmit: new AsyncSeriesHook(["compilation"]),
  135. /** @type {SyncHook<[Compilation, CompilationParams]>} */
  136. thisCompilation: new SyncHook(["compilation", "params"]),
  137. /** @type {SyncHook<[Compilation, CompilationParams]>} */
  138. compilation: new SyncHook(["compilation", "params"]),
  139. /** @type {SyncHook<[NormalModuleFactory]>} */
  140. normalModuleFactory: new SyncHook(["normalModuleFactory"]),
  141. /** @type {SyncHook<[ContextModuleFactory]>} */
  142. contextModuleFactory: new SyncHook(["contextModuleFactory"]),
  143. /** @type {AsyncSeriesHook<[CompilationParams]>} */
  144. beforeCompile: new AsyncSeriesHook(["params"]),
  145. /** @type {SyncHook<[CompilationParams]>} */
  146. compile: new SyncHook(["params"]),
  147. /** @type {AsyncParallelHook<[Compilation]>} */
  148. make: new AsyncParallelHook(["compilation"]),
  149. /** @type {AsyncParallelHook<[Compilation]>} */
  150. finishMake: new AsyncSeriesHook(["compilation"]),
  151. /** @type {AsyncSeriesHook<[Compilation]>} */
  152. afterCompile: new AsyncSeriesHook(["compilation"]),
  153. /** @type {AsyncSeriesHook<[]>} */
  154. readRecords: new AsyncSeriesHook([]),
  155. /** @type {AsyncSeriesHook<[]>} */
  156. emitRecords: new AsyncSeriesHook([]),
  157. /** @type {AsyncSeriesHook<[Compiler]>} */
  158. watchRun: new AsyncSeriesHook(["compiler"]),
  159. /** @type {SyncHook<[Error]>} */
  160. failed: new SyncHook(["error"]),
  161. /** @type {SyncHook<[string | null, number]>} */
  162. invalid: new SyncHook(["filename", "changeTime"]),
  163. /** @type {SyncHook<[]>} */
  164. watchClose: new SyncHook([]),
  165. /** @type {AsyncSeriesHook<[]>} */
  166. shutdown: new AsyncSeriesHook([]),
  167. /** @type {SyncBailHook<[string, string, any[]], true>} */
  168. infrastructureLog: new SyncBailHook(["origin", "type", "args"]),
  169. // TODO the following hooks are weirdly located here
  170. // TODO move them for webpack 5
  171. /** @type {SyncHook<[]>} */
  172. environment: new SyncHook([]),
  173. /** @type {SyncHook<[]>} */
  174. afterEnvironment: new SyncHook([]),
  175. /** @type {SyncHook<[Compiler]>} */
  176. afterPlugins: new SyncHook(["compiler"]),
  177. /** @type {SyncHook<[Compiler]>} */
  178. afterResolvers: new SyncHook(["compiler"]),
  179. /** @type {SyncBailHook<[string, Entry], boolean>} */
  180. entryOption: new SyncBailHook(["context", "entry"])
  181. });
  182. this.webpack = webpack;
  183. /** @type {string=} */
  184. this.name = undefined;
  185. /** @type {Compilation=} */
  186. this.parentCompilation = undefined;
  187. /** @type {Compiler} */
  188. this.root = this;
  189. /** @type {string} */
  190. this.outputPath = "";
  191. /** @type {Watching} */
  192. this.watching = undefined;
  193. /** @type {OutputFileSystem} */
  194. this.outputFileSystem = null;
  195. /** @type {IntermediateFileSystem} */
  196. this.intermediateFileSystem = null;
  197. /** @type {InputFileSystem} */
  198. this.inputFileSystem = null;
  199. /** @type {WatchFileSystem} */
  200. this.watchFileSystem = null;
  201. /** @type {string|null} */
  202. this.recordsInputPath = null;
  203. /** @type {string|null} */
  204. this.recordsOutputPath = null;
  205. this.records = {};
  206. /** @type {Set<string | RegExp>} */
  207. this.managedPaths = new Set();
  208. /** @type {Set<string | RegExp>} */
  209. this.immutablePaths = new Set();
  210. /** @type {ReadonlySet<string>} */
  211. this.modifiedFiles = undefined;
  212. /** @type {ReadonlySet<string>} */
  213. this.removedFiles = undefined;
  214. /** @type {ReadonlyMap<string, FileSystemInfoEntry | "ignore" | null>} */
  215. this.fileTimestamps = undefined;
  216. /** @type {ReadonlyMap<string, FileSystemInfoEntry | "ignore" | null>} */
  217. this.contextTimestamps = undefined;
  218. /** @type {number} */
  219. this.fsStartTime = undefined;
  220. /** @type {ResolverFactory} */
  221. this.resolverFactory = new ResolverFactory();
  222. this.infrastructureLogger = undefined;
  223. this.options = options;
  224. this.context = context;
  225. this.requestShortener = new RequestShortener(context, this.root);
  226. this.cache = new Cache();
  227. /** @type {Map<Module, { buildInfo: object, references: WeakMap<Dependency, Module>, memCache: WeakTupleMap }> | undefined} */
  228. this.moduleMemCaches = undefined;
  229. this.compilerPath = "";
  230. /** @type {boolean} */
  231. this.running = false;
  232. /** @type {boolean} */
  233. this.idle = false;
  234. /** @type {boolean} */
  235. this.watchMode = false;
  236. this._backCompat = this.options.experiments.backCompat !== false;
  237. /** @type {Compilation} */
  238. this._lastCompilation = undefined;
  239. /** @type {NormalModuleFactory} */
  240. this._lastNormalModuleFactory = undefined;
  241. /** @private @type {WeakMap<Source, { sizeOnlySource: SizeOnlySource, writtenTo: Map<string, number> }>} */
  242. this._assetEmittingSourceCache = new WeakMap();
  243. /** @private @type {Map<string, number>} */
  244. this._assetEmittingWrittenFiles = new Map();
  245. /** @private @type {Set<string>} */
  246. this._assetEmittingPreviousFiles = new Set();
  247. }
  248. /**
  249. * @param {string} name cache name
  250. * @returns {CacheFacade} the cache facade instance
  251. */
  252. getCache(name) {
  253. return new CacheFacade(
  254. this.cache,
  255. `${this.compilerPath}${name}`,
  256. this.options.output.hashFunction
  257. );
  258. }
  259. /**
  260. * @param {string | (function(): string)} name name of the logger, or function called once to get the logger name
  261. * @returns {Logger} a logger with that name
  262. */
  263. getInfrastructureLogger(name) {
  264. if (!name) {
  265. throw new TypeError(
  266. "Compiler.getInfrastructureLogger(name) called without a name"
  267. );
  268. }
  269. return new Logger(
  270. (type, args) => {
  271. if (typeof name === "function") {
  272. name = name();
  273. if (!name) {
  274. throw new TypeError(
  275. "Compiler.getInfrastructureLogger(name) called with a function not returning a name"
  276. );
  277. }
  278. }
  279. if (this.hooks.infrastructureLog.call(name, type, args) === undefined) {
  280. if (this.infrastructureLogger !== undefined) {
  281. this.infrastructureLogger(name, type, args);
  282. }
  283. }
  284. },
  285. childName => {
  286. if (typeof name === "function") {
  287. if (typeof childName === "function") {
  288. return this.getInfrastructureLogger(() => {
  289. if (typeof name === "function") {
  290. name = name();
  291. if (!name) {
  292. throw new TypeError(
  293. "Compiler.getInfrastructureLogger(name) called with a function not returning a name"
  294. );
  295. }
  296. }
  297. if (typeof childName === "function") {
  298. childName = childName();
  299. if (!childName) {
  300. throw new TypeError(
  301. "Logger.getChildLogger(name) called with a function not returning a name"
  302. );
  303. }
  304. }
  305. return `${name}/${childName}`;
  306. });
  307. } else {
  308. return this.getInfrastructureLogger(() => {
  309. if (typeof name === "function") {
  310. name = name();
  311. if (!name) {
  312. throw new TypeError(
  313. "Compiler.getInfrastructureLogger(name) called with a function not returning a name"
  314. );
  315. }
  316. }
  317. return `${name}/${childName}`;
  318. });
  319. }
  320. } else {
  321. if (typeof childName === "function") {
  322. return this.getInfrastructureLogger(() => {
  323. if (typeof childName === "function") {
  324. childName = childName();
  325. if (!childName) {
  326. throw new TypeError(
  327. "Logger.getChildLogger(name) called with a function not returning a name"
  328. );
  329. }
  330. }
  331. return `${name}/${childName}`;
  332. });
  333. } else {
  334. return this.getInfrastructureLogger(`${name}/${childName}`);
  335. }
  336. }
  337. }
  338. );
  339. }
  340. // TODO webpack 6: solve this in a better way
  341. // e.g. move compilation specific info from Modules into ModuleGraph
  342. _cleanupLastCompilation() {
  343. if (this._lastCompilation !== undefined) {
  344. for (const module of this._lastCompilation.modules) {
  345. ChunkGraph.clearChunkGraphForModule(module);
  346. ModuleGraph.clearModuleGraphForModule(module);
  347. module.cleanupForCache();
  348. }
  349. for (const chunk of this._lastCompilation.chunks) {
  350. ChunkGraph.clearChunkGraphForChunk(chunk);
  351. }
  352. this._lastCompilation = undefined;
  353. }
  354. }
  355. // TODO webpack 6: solve this in a better way
  356. _cleanupLastNormalModuleFactory() {
  357. if (this._lastNormalModuleFactory !== undefined) {
  358. this._lastNormalModuleFactory.cleanupForCache();
  359. this._lastNormalModuleFactory = undefined;
  360. }
  361. }
  362. /**
  363. * @param {WatchOptions} watchOptions the watcher's options
  364. * @param {Callback<Stats>} handler signals when the call finishes
  365. * @returns {Watching} a compiler watcher
  366. */
  367. watch(watchOptions, handler) {
  368. if (this.running) {
  369. return handler(new ConcurrentCompilationError());
  370. }
  371. this.running = true;
  372. this.watchMode = true;
  373. this.watching = new Watching(this, watchOptions, handler);
  374. return this.watching;
  375. }
  376. /**
  377. * @param {Callback<Stats>} callback signals when the call finishes
  378. * @returns {void}
  379. */
  380. run(callback) {
  381. if (this.running) {
  382. return callback(new ConcurrentCompilationError());
  383. }
  384. let logger;
  385. const finalCallback = (err, stats) => {
  386. if (logger) logger.time("beginIdle");
  387. this.idle = true;
  388. this.cache.beginIdle();
  389. this.idle = true;
  390. if (logger) logger.timeEnd("beginIdle");
  391. this.running = false;
  392. if (err) {
  393. this.hooks.failed.call(err);
  394. }
  395. if (callback !== undefined) callback(err, stats);
  396. this.hooks.afterDone.call(stats);
  397. };
  398. const startTime = Date.now();
  399. this.running = true;
  400. const onCompiled = (err, compilation) => {
  401. if (err) return finalCallback(err);
  402. if (this.hooks.shouldEmit.call(compilation) === false) {
  403. compilation.startTime = startTime;
  404. compilation.endTime = Date.now();
  405. const stats = new Stats(compilation);
  406. this.hooks.done.callAsync(stats, err => {
  407. if (err) return finalCallback(err);
  408. return finalCallback(null, stats);
  409. });
  410. return;
  411. }
  412. process.nextTick(() => {
  413. logger = compilation.getLogger("webpack.Compiler");
  414. logger.time("emitAssets");
  415. this.emitAssets(compilation, err => {
  416. logger.timeEnd("emitAssets");
  417. if (err) return finalCallback(err);
  418. if (compilation.hooks.needAdditionalPass.call()) {
  419. compilation.needAdditionalPass = true;
  420. compilation.startTime = startTime;
  421. compilation.endTime = Date.now();
  422. logger.time("done hook");
  423. const stats = new Stats(compilation);
  424. this.hooks.done.callAsync(stats, err => {
  425. logger.timeEnd("done hook");
  426. if (err) return finalCallback(err);
  427. this.hooks.additionalPass.callAsync(err => {
  428. if (err) return finalCallback(err);
  429. this.compile(onCompiled);
  430. });
  431. });
  432. return;
  433. }
  434. logger.time("emitRecords");
  435. this.emitRecords(err => {
  436. logger.timeEnd("emitRecords");
  437. if (err) return finalCallback(err);
  438. compilation.startTime = startTime;
  439. compilation.endTime = Date.now();
  440. logger.time("done hook");
  441. const stats = new Stats(compilation);
  442. this.hooks.done.callAsync(stats, err => {
  443. logger.timeEnd("done hook");
  444. if (err) return finalCallback(err);
  445. this.cache.storeBuildDependencies(
  446. compilation.buildDependencies,
  447. err => {
  448. if (err) return finalCallback(err);
  449. return finalCallback(null, stats);
  450. }
  451. );
  452. });
  453. });
  454. });
  455. });
  456. };
  457. const run = () => {
  458. this.hooks.beforeRun.callAsync(this, err => {
  459. if (err) return finalCallback(err);
  460. this.hooks.run.callAsync(this, err => {
  461. if (err) return finalCallback(err);
  462. this.readRecords(err => {
  463. if (err) return finalCallback(err);
  464. this.compile(onCompiled);
  465. });
  466. });
  467. });
  468. };
  469. if (this.idle) {
  470. this.cache.endIdle(err => {
  471. if (err) return finalCallback(err);
  472. this.idle = false;
  473. run();
  474. });
  475. } else {
  476. run();
  477. }
  478. }
  479. /**
  480. * @param {RunAsChildCallback} callback signals when the call finishes
  481. * @returns {void}
  482. */
  483. runAsChild(callback) {
  484. const startTime = Date.now();
  485. const finalCallback = (err, entries, compilation) => {
  486. try {
  487. callback(err, entries, compilation);
  488. } catch (e) {
  489. const err = new WebpackError(
  490. `compiler.runAsChild callback error: ${e}`
  491. );
  492. err.details = e.stack;
  493. this.parentCompilation.errors.push(err);
  494. }
  495. };
  496. this.compile((err, compilation) => {
  497. if (err) return finalCallback(err);
  498. this.parentCompilation.children.push(compilation);
  499. for (const { name, source, info } of compilation.getAssets()) {
  500. this.parentCompilation.emitAsset(name, source, info);
  501. }
  502. const entries = [];
  503. for (const ep of compilation.entrypoints.values()) {
  504. entries.push(...ep.chunks);
  505. }
  506. compilation.startTime = startTime;
  507. compilation.endTime = Date.now();
  508. return finalCallback(null, entries, compilation);
  509. });
  510. }
  511. purgeInputFileSystem() {
  512. if (this.inputFileSystem && this.inputFileSystem.purge) {
  513. this.inputFileSystem.purge();
  514. }
  515. }
  516. /**
  517. * @param {Compilation} compilation the compilation
  518. * @param {Callback<void>} callback signals when the assets are emitted
  519. * @returns {void}
  520. */
  521. emitAssets(compilation, callback) {
  522. let outputPath;
  523. const emitFiles = err => {
  524. if (err) return callback(err);
  525. const assets = compilation.getAssets();
  526. compilation.assets = { ...compilation.assets };
  527. /** @type {Map<string, { path: string, source: Source, size: number, waiting: { cacheEntry: any, file: string }[] }>} */
  528. const caseInsensitiveMap = new Map();
  529. /** @type {Set<string>} */
  530. const allTargetPaths = new Set();
  531. asyncLib.forEachLimit(
  532. assets,
  533. 15,
  534. ({ name: file, source, info }, callback) => {
  535. let targetFile = file;
  536. let immutable = info.immutable;
  537. const queryStringIdx = targetFile.indexOf("?");
  538. if (queryStringIdx >= 0) {
  539. targetFile = targetFile.slice(0, queryStringIdx);
  540. // We may remove the hash, which is in the query string
  541. // So we recheck if the file is immutable
  542. // This doesn't cover all cases, but immutable is only a performance optimization anyway
  543. immutable =
  544. immutable &&
  545. (includesHash(targetFile, info.contenthash) ||
  546. includesHash(targetFile, info.chunkhash) ||
  547. includesHash(targetFile, info.modulehash) ||
  548. includesHash(targetFile, info.fullhash));
  549. }
  550. const writeOut = err => {
  551. if (err) return callback(err);
  552. const targetPath = join(
  553. this.outputFileSystem,
  554. outputPath,
  555. targetFile
  556. );
  557. allTargetPaths.add(targetPath);
  558. // check if the target file has already been written by this Compiler
  559. const targetFileGeneration =
  560. this._assetEmittingWrittenFiles.get(targetPath);
  561. // create an cache entry for this Source if not already existing
  562. let cacheEntry = this._assetEmittingSourceCache.get(source);
  563. if (cacheEntry === undefined) {
  564. cacheEntry = {
  565. sizeOnlySource: undefined,
  566. writtenTo: new Map()
  567. };
  568. this._assetEmittingSourceCache.set(source, cacheEntry);
  569. }
  570. let similarEntry;
  571. const checkSimilarFile = () => {
  572. const caseInsensitiveTargetPath = targetPath.toLowerCase();
  573. similarEntry = caseInsensitiveMap.get(caseInsensitiveTargetPath);
  574. if (similarEntry !== undefined) {
  575. const { path: other, source: otherSource } = similarEntry;
  576. if (isSourceEqual(otherSource, source)) {
  577. // Size may or may not be available at this point.
  578. // If it's not available add to "waiting" list and it will be updated once available
  579. if (similarEntry.size !== undefined) {
  580. updateWithReplacementSource(similarEntry.size);
  581. } else {
  582. if (!similarEntry.waiting) similarEntry.waiting = [];
  583. similarEntry.waiting.push({ file, cacheEntry });
  584. }
  585. alreadyWritten();
  586. } else {
  587. const err =
  588. new WebpackError(`Prevent writing to file that only differs in casing or query string from already written file.
  589. This will lead to a race-condition and corrupted files on case-insensitive file systems.
  590. ${targetPath}
  591. ${other}`);
  592. err.file = file;
  593. callback(err);
  594. }
  595. return true;
  596. } else {
  597. caseInsensitiveMap.set(
  598. caseInsensitiveTargetPath,
  599. (similarEntry = {
  600. path: targetPath,
  601. source,
  602. size: undefined,
  603. waiting: undefined
  604. })
  605. );
  606. return false;
  607. }
  608. };
  609. /**
  610. * get the binary (Buffer) content from the Source
  611. * @returns {Buffer} content for the source
  612. */
  613. const getContent = () => {
  614. if (typeof source.buffer === "function") {
  615. return source.buffer();
  616. } else {
  617. const bufferOrString = source.source();
  618. if (Buffer.isBuffer(bufferOrString)) {
  619. return bufferOrString;
  620. } else {
  621. return Buffer.from(bufferOrString, "utf8");
  622. }
  623. }
  624. };
  625. const alreadyWritten = () => {
  626. // cache the information that the Source has been already been written to that location
  627. if (targetFileGeneration === undefined) {
  628. const newGeneration = 1;
  629. this._assetEmittingWrittenFiles.set(targetPath, newGeneration);
  630. cacheEntry.writtenTo.set(targetPath, newGeneration);
  631. } else {
  632. cacheEntry.writtenTo.set(targetPath, targetFileGeneration);
  633. }
  634. callback();
  635. };
  636. /**
  637. * Write the file to output file system
  638. * @param {Buffer} content content to be written
  639. * @returns {void}
  640. */
  641. const doWrite = content => {
  642. this.outputFileSystem.writeFile(targetPath, content, err => {
  643. if (err) return callback(err);
  644. // information marker that the asset has been emitted
  645. compilation.emittedAssets.add(file);
  646. // cache the information that the Source has been written to that location
  647. const newGeneration =
  648. targetFileGeneration === undefined
  649. ? 1
  650. : targetFileGeneration + 1;
  651. cacheEntry.writtenTo.set(targetPath, newGeneration);
  652. this._assetEmittingWrittenFiles.set(targetPath, newGeneration);
  653. this.hooks.assetEmitted.callAsync(
  654. file,
  655. {
  656. content,
  657. source,
  658. outputPath,
  659. compilation,
  660. targetPath
  661. },
  662. callback
  663. );
  664. });
  665. };
  666. const updateWithReplacementSource = size => {
  667. updateFileWithReplacementSource(file, cacheEntry, size);
  668. similarEntry.size = size;
  669. if (similarEntry.waiting !== undefined) {
  670. for (const { file, cacheEntry } of similarEntry.waiting) {
  671. updateFileWithReplacementSource(file, cacheEntry, size);
  672. }
  673. }
  674. };
  675. const updateFileWithReplacementSource = (
  676. file,
  677. cacheEntry,
  678. size
  679. ) => {
  680. // Create a replacement resource which only allows to ask for size
  681. // This allows to GC all memory allocated by the Source
  682. // (expect when the Source is stored in any other cache)
  683. if (!cacheEntry.sizeOnlySource) {
  684. cacheEntry.sizeOnlySource = new SizeOnlySource(size);
  685. }
  686. compilation.updateAsset(file, cacheEntry.sizeOnlySource, {
  687. size
  688. });
  689. };
  690. const processExistingFile = stats => {
  691. // skip emitting if it's already there and an immutable file
  692. if (immutable) {
  693. updateWithReplacementSource(stats.size);
  694. return alreadyWritten();
  695. }
  696. const content = getContent();
  697. updateWithReplacementSource(content.length);
  698. // if it exists and content on disk matches content
  699. // skip writing the same content again
  700. // (to keep mtime and don't trigger watchers)
  701. // for a fast negative match file size is compared first
  702. if (content.length === stats.size) {
  703. compilation.comparedForEmitAssets.add(file);
  704. return this.outputFileSystem.readFile(
  705. targetPath,
  706. (err, existingContent) => {
  707. if (
  708. err ||
  709. !content.equals(/** @type {Buffer} */ (existingContent))
  710. ) {
  711. return doWrite(content);
  712. } else {
  713. return alreadyWritten();
  714. }
  715. }
  716. );
  717. }
  718. return doWrite(content);
  719. };
  720. const processMissingFile = () => {
  721. const content = getContent();
  722. updateWithReplacementSource(content.length);
  723. return doWrite(content);
  724. };
  725. // if the target file has already been written
  726. if (targetFileGeneration !== undefined) {
  727. // check if the Source has been written to this target file
  728. const writtenGeneration = cacheEntry.writtenTo.get(targetPath);
  729. if (writtenGeneration === targetFileGeneration) {
  730. // if yes, we may skip writing the file
  731. // if it's already there
  732. // (we assume one doesn't modify files while the Compiler is running, other then removing them)
  733. if (this._assetEmittingPreviousFiles.has(targetPath)) {
  734. // We assume that assets from the last compilation say intact on disk (they are not removed)
  735. compilation.updateAsset(file, cacheEntry.sizeOnlySource, {
  736. size: cacheEntry.sizeOnlySource.size()
  737. });
  738. return callback();
  739. } else {
  740. // Settings immutable will make it accept file content without comparing when file exist
  741. immutable = true;
  742. }
  743. } else if (!immutable) {
  744. if (checkSimilarFile()) return;
  745. // We wrote to this file before which has very likely a different content
  746. // skip comparing and assume content is different for performance
  747. // This case happens often during watch mode.
  748. return processMissingFile();
  749. }
  750. }
  751. if (checkSimilarFile()) return;
  752. if (this.options.output.compareBeforeEmit) {
  753. this.outputFileSystem.stat(targetPath, (err, stats) => {
  754. const exists = !err && stats.isFile();
  755. if (exists) {
  756. processExistingFile(stats);
  757. } else {
  758. processMissingFile();
  759. }
  760. });
  761. } else {
  762. processMissingFile();
  763. }
  764. };
  765. if (targetFile.match(/\/|\\/)) {
  766. const fs = this.outputFileSystem;
  767. const dir = dirname(fs, join(fs, outputPath, targetFile));
  768. mkdirp(fs, dir, writeOut);
  769. } else {
  770. writeOut();
  771. }
  772. },
  773. err => {
  774. // Clear map to free up memory
  775. caseInsensitiveMap.clear();
  776. if (err) {
  777. this._assetEmittingPreviousFiles.clear();
  778. return callback(err);
  779. }
  780. this._assetEmittingPreviousFiles = allTargetPaths;
  781. this.hooks.afterEmit.callAsync(compilation, err => {
  782. if (err) return callback(err);
  783. return callback();
  784. });
  785. }
  786. );
  787. };
  788. this.hooks.emit.callAsync(compilation, err => {
  789. if (err) return callback(err);
  790. outputPath = compilation.getPath(this.outputPath, {});
  791. mkdirp(this.outputFileSystem, outputPath, emitFiles);
  792. });
  793. }
  794. /**
  795. * @param {Callback<void>} callback signals when the call finishes
  796. * @returns {void}
  797. */
  798. emitRecords(callback) {
  799. if (this.hooks.emitRecords.isUsed()) {
  800. if (this.recordsOutputPath) {
  801. asyncLib.parallel(
  802. [
  803. cb => this.hooks.emitRecords.callAsync(cb),
  804. this._emitRecords.bind(this)
  805. ],
  806. err => callback(err)
  807. );
  808. } else {
  809. this.hooks.emitRecords.callAsync(callback);
  810. }
  811. } else {
  812. if (this.recordsOutputPath) {
  813. this._emitRecords(callback);
  814. } else {
  815. callback();
  816. }
  817. }
  818. }
  819. /**
  820. * @param {Callback<void>} callback signals when the call finishes
  821. * @returns {void}
  822. */
  823. _emitRecords(callback) {
  824. const writeFile = () => {
  825. this.outputFileSystem.writeFile(
  826. this.recordsOutputPath,
  827. JSON.stringify(
  828. this.records,
  829. (n, value) => {
  830. if (
  831. typeof value === "object" &&
  832. value !== null &&
  833. !Array.isArray(value)
  834. ) {
  835. const keys = Object.keys(value);
  836. if (!isSorted(keys)) {
  837. return sortObject(value, keys);
  838. }
  839. }
  840. return value;
  841. },
  842. 2
  843. ),
  844. callback
  845. );
  846. };
  847. const recordsOutputPathDirectory = dirname(
  848. this.outputFileSystem,
  849. this.recordsOutputPath
  850. );
  851. if (!recordsOutputPathDirectory) {
  852. return writeFile();
  853. }
  854. mkdirp(this.outputFileSystem, recordsOutputPathDirectory, err => {
  855. if (err) return callback(err);
  856. writeFile();
  857. });
  858. }
  859. /**
  860. * @param {Callback<void>} callback signals when the call finishes
  861. * @returns {void}
  862. */
  863. readRecords(callback) {
  864. if (this.hooks.readRecords.isUsed()) {
  865. if (this.recordsInputPath) {
  866. asyncLib.parallel([
  867. cb => this.hooks.readRecords.callAsync(cb),
  868. this._readRecords.bind(this)
  869. ]);
  870. } else {
  871. this.records = {};
  872. this.hooks.readRecords.callAsync(callback);
  873. }
  874. } else {
  875. if (this.recordsInputPath) {
  876. this._readRecords(callback);
  877. } else {
  878. this.records = {};
  879. callback();
  880. }
  881. }
  882. }
  883. /**
  884. * @param {Callback<void>} callback signals when the call finishes
  885. * @returns {void}
  886. */
  887. _readRecords(callback) {
  888. if (!this.recordsInputPath) {
  889. this.records = {};
  890. return callback();
  891. }
  892. this.inputFileSystem.stat(this.recordsInputPath, err => {
  893. // It doesn't exist
  894. // We can ignore this.
  895. if (err) return callback();
  896. this.inputFileSystem.readFile(this.recordsInputPath, (err, content) => {
  897. if (err) return callback(err);
  898. try {
  899. this.records = parseJson(content.toString("utf-8"));
  900. } catch (e) {
  901. e.message = "Cannot parse records: " + e.message;
  902. return callback(e);
  903. }
  904. return callback();
  905. });
  906. });
  907. }
  908. /**
  909. * @param {Compilation} compilation the compilation
  910. * @param {string} compilerName the compiler's name
  911. * @param {number} compilerIndex the compiler's index
  912. * @param {OutputOptions=} outputOptions the output options
  913. * @param {WebpackPluginInstance[]=} plugins the plugins to apply
  914. * @returns {Compiler} a child compiler
  915. */
  916. createChildCompiler(
  917. compilation,
  918. compilerName,
  919. compilerIndex,
  920. outputOptions,
  921. plugins
  922. ) {
  923. const childCompiler = new Compiler(this.context, {
  924. ...this.options,
  925. output: {
  926. ...this.options.output,
  927. ...outputOptions
  928. }
  929. });
  930. childCompiler.name = compilerName;
  931. childCompiler.outputPath = this.outputPath;
  932. childCompiler.inputFileSystem = this.inputFileSystem;
  933. childCompiler.outputFileSystem = null;
  934. childCompiler.resolverFactory = this.resolverFactory;
  935. childCompiler.modifiedFiles = this.modifiedFiles;
  936. childCompiler.removedFiles = this.removedFiles;
  937. childCompiler.fileTimestamps = this.fileTimestamps;
  938. childCompiler.contextTimestamps = this.contextTimestamps;
  939. childCompiler.fsStartTime = this.fsStartTime;
  940. childCompiler.cache = this.cache;
  941. childCompiler.compilerPath = `${this.compilerPath}${compilerName}|${compilerIndex}|`;
  942. childCompiler._backCompat = this._backCompat;
  943. const relativeCompilerName = makePathsRelative(
  944. this.context,
  945. compilerName,
  946. this.root
  947. );
  948. if (!this.records[relativeCompilerName]) {
  949. this.records[relativeCompilerName] = [];
  950. }
  951. if (this.records[relativeCompilerName][compilerIndex]) {
  952. childCompiler.records = this.records[relativeCompilerName][compilerIndex];
  953. } else {
  954. this.records[relativeCompilerName].push((childCompiler.records = {}));
  955. }
  956. childCompiler.parentCompilation = compilation;
  957. childCompiler.root = this.root;
  958. if (Array.isArray(plugins)) {
  959. for (const plugin of plugins) {
  960. plugin.apply(childCompiler);
  961. }
  962. }
  963. for (const name in this.hooks) {
  964. if (
  965. ![
  966. "make",
  967. "compile",
  968. "emit",
  969. "afterEmit",
  970. "invalid",
  971. "done",
  972. "thisCompilation"
  973. ].includes(name)
  974. ) {
  975. if (childCompiler.hooks[name]) {
  976. childCompiler.hooks[name].taps = this.hooks[name].taps.slice();
  977. }
  978. }
  979. }
  980. compilation.hooks.childCompiler.call(
  981. childCompiler,
  982. compilerName,
  983. compilerIndex
  984. );
  985. return childCompiler;
  986. }
  987. isChild() {
  988. return !!this.parentCompilation;
  989. }
  990. createCompilation(params) {
  991. this._cleanupLastCompilation();
  992. return (this._lastCompilation = new Compilation(this, params));
  993. }
  994. /**
  995. * @param {CompilationParams} params the compilation parameters
  996. * @returns {Compilation} the created compilation
  997. */
  998. newCompilation(params) {
  999. const compilation = this.createCompilation(params);
  1000. compilation.name = this.name;
  1001. compilation.records = this.records;
  1002. this.hooks.thisCompilation.call(compilation, params);
  1003. this.hooks.compilation.call(compilation, params);
  1004. return compilation;
  1005. }
  1006. createNormalModuleFactory() {
  1007. this._cleanupLastNormalModuleFactory();
  1008. const normalModuleFactory = new NormalModuleFactory({
  1009. context: this.options.context,
  1010. fs: this.inputFileSystem,
  1011. resolverFactory: this.resolverFactory,
  1012. options: this.options.module,
  1013. associatedObjectForCache: this.root,
  1014. layers: this.options.experiments.layers
  1015. });
  1016. this._lastNormalModuleFactory = normalModuleFactory;
  1017. this.hooks.normalModuleFactory.call(normalModuleFactory);
  1018. return normalModuleFactory;
  1019. }
  1020. createContextModuleFactory() {
  1021. const contextModuleFactory = new ContextModuleFactory(this.resolverFactory);
  1022. this.hooks.contextModuleFactory.call(contextModuleFactory);
  1023. return contextModuleFactory;
  1024. }
  1025. newCompilationParams() {
  1026. const params = {
  1027. normalModuleFactory: this.createNormalModuleFactory(),
  1028. contextModuleFactory: this.createContextModuleFactory()
  1029. };
  1030. return params;
  1031. }
  1032. /**
  1033. * @param {Callback<Compilation>} callback signals when the compilation finishes
  1034. * @returns {void}
  1035. */
  1036. compile(callback) {
  1037. const params = this.newCompilationParams();
  1038. this.hooks.beforeCompile.callAsync(params, err => {
  1039. if (err) return callback(err);
  1040. this.hooks.compile.call(params);
  1041. const compilation = this.newCompilation(params);
  1042. const logger = compilation.getLogger("webpack.Compiler");
  1043. logger.time("make hook");
  1044. this.hooks.make.callAsync(compilation, err => {
  1045. logger.timeEnd("make hook");
  1046. if (err) return callback(err);
  1047. logger.time("finish make hook");
  1048. this.hooks.finishMake.callAsync(compilation, err => {
  1049. logger.timeEnd("finish make hook");
  1050. if (err) return callback(err);
  1051. process.nextTick(() => {
  1052. logger.time("finish compilation");
  1053. compilation.finish(err => {
  1054. logger.timeEnd("finish compilation");
  1055. if (err) return callback(err);
  1056. logger.time("seal compilation");
  1057. compilation.seal(err => {
  1058. logger.timeEnd("seal compilation");
  1059. if (err) return callback(err);
  1060. logger.time("afterCompile hook");
  1061. this.hooks.afterCompile.callAsync(compilation, err => {
  1062. logger.timeEnd("afterCompile hook");
  1063. if (err) return callback(err);
  1064. return callback(null, compilation);
  1065. });
  1066. });
  1067. });
  1068. });
  1069. });
  1070. });
  1071. });
  1072. }
  1073. /**
  1074. * @param {Callback<void>} callback signals when the compiler closes
  1075. * @returns {void}
  1076. */
  1077. close(callback) {
  1078. if (this.watching) {
  1079. // When there is still an active watching, close this first
  1080. this.watching.close(err => {
  1081. this.close(callback);
  1082. });
  1083. return;
  1084. }
  1085. this.hooks.shutdown.callAsync(err => {
  1086. if (err) return callback(err);
  1087. // Get rid of reference to last compilation to avoid leaking memory
  1088. // We can't run this._cleanupLastCompilation() as the Stats to this compilation
  1089. // might be still in use. We try to get rid of the reference to the cache instead.
  1090. this._lastCompilation = undefined;
  1091. this._lastNormalModuleFactory = undefined;
  1092. this.cache.shutdown(callback);
  1093. });
  1094. }
  1095. }
  1096. module.exports = Compiler;