index.js 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429
  1. 'use strict'
  2. const BB = require('bluebird')
  3. const binLink = require('bin-links')
  4. const buildLogicalTree = require('npm-logical-tree')
  5. const extract = require('./lib/extract.js')
  6. const figgyPudding = require('figgy-pudding')
  7. const fs = require('graceful-fs')
  8. const getPrefix = require('find-npm-prefix')
  9. const lifecycle = require('npm-lifecycle')
  10. const lockVerify = require('lock-verify')
  11. const mkdirp = BB.promisify(require('mkdirp'))
  12. const npa = require('npm-package-arg')
  13. const path = require('path')
  14. const readPkgJson = BB.promisify(require('read-package-json'))
  15. const rimraf = BB.promisify(require('rimraf'))
  16. const readFileAsync = BB.promisify(fs.readFile)
  17. const statAsync = BB.promisify(fs.stat)
  18. const symlinkAsync = BB.promisify(fs.symlink)
  19. const writeFileAsync = BB.promisify(fs.writeFile)
  20. const LifecycleOpts = figgyPudding({
  21. config: {},
  22. 'script-shell': {},
  23. scriptShell: 'script-shell',
  24. 'ignore-scripts': {},
  25. ignoreScripts: 'ignore-scripts',
  26. 'ignore-prepublish': {},
  27. ignorePrepublish: 'ignore-prepublish',
  28. 'scripts-prepend-node-path': {},
  29. scriptsPrependNodePath: 'scripts-prepend-node-path',
  30. 'unsafe-perm': {},
  31. unsafePerm: 'unsafe-perm',
  32. prefix: {},
  33. dir: 'prefix',
  34. failOk: { default: false }
  35. }, { other () { return true } })
  36. class Installer {
  37. constructor (opts) {
  38. this.opts = opts
  39. // Stats
  40. this.startTime = Date.now()
  41. this.runTime = 0
  42. this.timings = { scripts: 0 }
  43. this.pkgCount = 0
  44. // Misc
  45. this.log = this.opts.log || require('./lib/silentlog.js')
  46. this.pkg = null
  47. this.tree = null
  48. this.failedDeps = new Set()
  49. }
  50. timedStage (name) {
  51. const start = Date.now()
  52. return BB.resolve(this[name].apply(this, [].slice.call(arguments, 1)))
  53. .tap(() => {
  54. this.timings[name] = Date.now() - start
  55. this.log.info(name, `Done in ${this.timings[name] / 1000}s`)
  56. })
  57. }
  58. run () {
  59. return this.timedStage('prepare')
  60. .then(() => this.timedStage('extractTree', this.tree))
  61. .then(() => this.timedStage('updateJson', this.tree))
  62. .then(pkgJsons => this.timedStage('buildTree', this.tree, pkgJsons))
  63. .then(() => this.timedStage('garbageCollect', this.tree))
  64. .then(() => this.timedStage('runScript', 'prepublish', this.pkg, this.prefix))
  65. .then(() => this.timedStage('runScript', 'prepare', this.pkg, this.prefix))
  66. .then(() => this.timedStage('teardown'))
  67. .then(() => {
  68. this.runTime = Date.now() - this.startTime
  69. this.log.info(
  70. 'run-scripts',
  71. `total script time: ${this.timings.scripts / 1000}s`
  72. )
  73. this.log.info(
  74. 'run-time',
  75. `total run time: ${this.runTime / 1000}s`
  76. )
  77. })
  78. .catch(err => {
  79. this.timedStage('teardown')
  80. if (err.message.match(/aggregate error/)) {
  81. throw err[0]
  82. } else {
  83. throw err
  84. }
  85. })
  86. .then(() => this)
  87. }
  88. prepare () {
  89. this.log.info('prepare', 'initializing installer')
  90. this.log.level = this.opts.loglevel
  91. this.log.verbose('prepare', 'starting workers')
  92. extract.startWorkers()
  93. return (
  94. this.opts.prefix && this.opts.global
  95. ? BB.resolve(this.opts.prefix)
  96. // There's some Special™ logic around the `--prefix` config when it
  97. // comes from a config file or env vs when it comes from the CLI
  98. : process.argv.some(arg => arg.match(/^\s*--prefix\s*/i))
  99. ? BB.resolve(this.opts.prefix)
  100. : getPrefix(process.cwd())
  101. )
  102. .then(prefix => {
  103. this.prefix = prefix
  104. this.log.verbose('prepare', 'installation prefix: ' + prefix)
  105. return BB.join(
  106. readJson(prefix, 'package.json'),
  107. readJson(prefix, 'package-lock.json', true),
  108. readJson(prefix, 'npm-shrinkwrap.json', true),
  109. (pkg, lock, shrink) => {
  110. if (shrink) {
  111. this.log.verbose('prepare', 'using npm-shrinkwrap.json')
  112. } else if (lock) {
  113. this.log.verbose('prepare', 'using package-lock.json')
  114. }
  115. pkg._shrinkwrap = shrink || lock
  116. this.pkg = pkg
  117. }
  118. )
  119. })
  120. .then(() => statAsync(
  121. path.join(this.prefix, 'node_modules')
  122. ).catch(err => { if (err.code !== 'ENOENT') { throw err } }))
  123. .then(stat => {
  124. stat && this.log.warn(
  125. 'prepare', 'removing existing node_modules/ before installation'
  126. )
  127. return BB.join(
  128. this.checkLock(),
  129. stat && rimraf(path.join(this.prefix, 'node_modules/*'))
  130. )
  131. }).then(() => {
  132. // This needs to happen -after- we've done checkLock()
  133. this.tree = buildLogicalTree(this.pkg, this.pkg._shrinkwrap)
  134. this.log.silly('tree', this.tree)
  135. this.expectedTotal = 0
  136. this.tree.forEach((dep, next) => {
  137. this.expectedTotal++
  138. next()
  139. })
  140. })
  141. }
  142. teardown () {
  143. this.log.verbose('teardown', 'shutting down workers.')
  144. return extract.stopWorkers()
  145. }
  146. checkLock () {
  147. this.log.verbose('checkLock', 'verifying package-lock data')
  148. const pkg = this.pkg
  149. const prefix = this.prefix
  150. if (!pkg._shrinkwrap || !pkg._shrinkwrap.lockfileVersion) {
  151. return BB.reject(
  152. new Error(`cipm can only install packages with an existing package-lock.json or npm-shrinkwrap.json with lockfileVersion >= 1. Run an install with npm@5 or later to generate it, then try again.`)
  153. )
  154. }
  155. return lockVerify(prefix).then(result => {
  156. if (result.status) {
  157. result.warnings.forEach(w => this.log.warn('lockfile', w))
  158. } else {
  159. throw new Error(
  160. 'cipm can only install packages when your package.json and package-lock.json or ' +
  161. 'npm-shrinkwrap.json are in sync. Please update your lock file with `npm install` ' +
  162. 'before continuing.\n\n' +
  163. result.warnings.map(w => 'Warning: ' + w).join('\n') + '\n' +
  164. result.errors.join('\n') + '\n'
  165. )
  166. }
  167. }).catch(err => {
  168. throw err
  169. })
  170. }
  171. extractTree (tree) {
  172. this.log.verbose('extractTree', 'extracting dependencies to node_modules/')
  173. const cg = this.log.newItem('extractTree', this.expectedTotal)
  174. return tree.forEachAsync((dep, next) => {
  175. if (!this.checkDepEnv(dep)) { return }
  176. const depPath = dep.path(this.prefix)
  177. const spec = npa.resolve(dep.name, dep.version, this.prefix)
  178. if (dep.isRoot) {
  179. return next()
  180. } else if (spec.type === 'directory') {
  181. const relative = path.relative(path.dirname(depPath), spec.fetchSpec)
  182. this.log.silly('extractTree', `${dep.name}@${spec.fetchSpec} -> ${depPath} (symlink)`)
  183. return mkdirp(path.dirname(depPath))
  184. .then(() => symlinkAsync(relative, depPath, 'junction'))
  185. .catch(
  186. () => rimraf(depPath)
  187. .then(() => symlinkAsync(relative, depPath, 'junction'))
  188. ).then(() => next())
  189. .then(() => {
  190. this.pkgCount++
  191. cg.completeWork(1)
  192. })
  193. } else {
  194. this.log.silly('extractTree', `${dep.name}@${dep.version} -> ${depPath}`)
  195. return (
  196. dep.bundled
  197. ? statAsync(path.join(depPath, 'package.json')).catch(err => {
  198. if (err.code !== 'ENOENT') { throw err }
  199. })
  200. : BB.resolve(false)
  201. )
  202. .then(wasBundled => {
  203. // Don't extract if a bundled dep is actually present
  204. if (wasBundled) {
  205. cg.completeWork(1)
  206. return next()
  207. } else {
  208. return BB.resolve(extract.child(
  209. dep.name, dep, depPath, this.opts
  210. ))
  211. .then(() => cg.completeWork(1))
  212. .then(() => { this.pkgCount++ })
  213. .then(next)
  214. }
  215. })
  216. }
  217. }, {concurrency: 50, Promise: BB})
  218. .then(() => cg.finish())
  219. }
  220. checkDepEnv (dep) {
  221. const includeDev = (
  222. // Covers --dev and --development (from npm config itself)
  223. this.opts.dev ||
  224. (
  225. !/^prod(uction)?$/.test(this.opts.only) &&
  226. !this.opts.production
  227. ) ||
  228. /^dev(elopment)?$/.test(this.opts.only) ||
  229. /^dev(elopment)?$/.test(this.opts.also)
  230. )
  231. const includeProd = !/^dev(elopment)?$/.test(this.opts.only)
  232. const includeOptional = includeProd && this.opts.optional
  233. return (dep.dev && includeDev) ||
  234. (dep.optional && includeOptional) ||
  235. (!dep.dev && !dep.optional && includeProd)
  236. }
  237. updateJson (tree) {
  238. this.log.verbose('updateJson', 'updating json deps to include _from')
  239. const pkgJsons = new Map()
  240. return tree.forEachAsync((dep, next) => {
  241. if (!this.checkDepEnv(dep)) { return }
  242. const spec = npa.resolve(dep.name, dep.version)
  243. const depPath = dep.path(this.prefix)
  244. return next()
  245. .then(() => readJson(depPath, 'package.json'))
  246. .then(pkg => (spec.registry || spec.type === 'directory')
  247. ? pkg
  248. : this.updateFromField(dep, pkg).then(() => pkg)
  249. )
  250. .then(pkg => (pkg.scripts && pkg.scripts.install)
  251. ? pkg
  252. : this.updateInstallScript(dep, pkg).then(() => pkg)
  253. )
  254. .tap(pkg => { pkgJsons.set(dep, pkg) })
  255. }, {concurrency: 100, Promise: BB})
  256. .then(() => pkgJsons)
  257. }
  258. buildTree (tree, pkgJsons) {
  259. this.log.verbose('buildTree', 'finalizing tree and running scripts')
  260. return tree.forEachAsync((dep, next) => {
  261. if (!this.checkDepEnv(dep)) { return }
  262. const spec = npa.resolve(dep.name, dep.version)
  263. const depPath = dep.path(this.prefix)
  264. const pkg = pkgJsons.get(dep)
  265. this.log.silly('buildTree', `linking ${spec}`)
  266. return this.runScript('preinstall', pkg, depPath)
  267. .then(next) // build children between preinstall and binLink
  268. // Don't link root bins
  269. .then(() => {
  270. if (
  271. dep.isRoot ||
  272. !(pkg.bin || pkg.man || (pkg.directories && pkg.directories.bin))
  273. ) {
  274. // We skip the relatively expensive readPkgJson if there's no way
  275. // we'll actually be linking any bins or mans
  276. return
  277. }
  278. return readPkgJson(path.join(depPath, 'package.json'))
  279. .then(pkg => binLink(pkg, depPath, false, {
  280. force: this.opts.force,
  281. ignoreScripts: this.opts['ignore-scripts'],
  282. log: Object.assign({}, this.log, { info: () => {} }),
  283. name: pkg.name,
  284. pkgId: pkg.name + '@' + pkg.version,
  285. prefix: this.prefix,
  286. prefixes: [this.prefix],
  287. umask: this.opts.umask
  288. }), e => {
  289. this.log.verbose('buildTree', `error linking ${spec}: ${e.message} ${e.stack}`)
  290. })
  291. })
  292. .then(() => this.runScript('install', pkg, depPath))
  293. .then(() => this.runScript('postinstall', pkg, depPath))
  294. .then(() => this)
  295. .catch(e => {
  296. if (dep.optional) {
  297. this.failedDeps.add(dep)
  298. } else {
  299. throw e
  300. }
  301. })
  302. }, {concurrency: 1, Promise: BB})
  303. }
  304. updateFromField (dep, pkg) {
  305. const depPath = dep.path(this.prefix)
  306. const depPkgPath = path.join(depPath, 'package.json')
  307. const parent = dep.requiredBy.values().next().value
  308. return readJson(parent.path(this.prefix), 'package.json')
  309. .then(ppkg =>
  310. (ppkg.dependencies && ppkg.dependencies[dep.name]) ||
  311. (ppkg.devDependencies && ppkg.devDependencies[dep.name]) ||
  312. (ppkg.optionalDependencies && ppkg.optionalDependencies[dep.name])
  313. )
  314. .then(from => npa.resolve(dep.name, from))
  315. .then(from => { pkg._from = from.toString() })
  316. .then(() => writeFileAsync(depPkgPath, JSON.stringify(pkg, null, 2)))
  317. .then(() => pkg)
  318. }
  319. updateInstallScript (dep, pkg) {
  320. const depPath = dep.path(this.prefix)
  321. return statAsync(path.join(depPath, 'binding.gyp'))
  322. .catch(err => { if (err.code !== 'ENOENT') { throw err } })
  323. .then(stat => {
  324. if (stat) {
  325. if (!pkg.scripts) {
  326. pkg.scripts = {}
  327. }
  328. pkg.scripts.install = 'node-gyp rebuild'
  329. }
  330. })
  331. .then(() => pkg)
  332. }
  333. // A cute little mark-and-sweep collector!
  334. garbageCollect (tree) {
  335. if (!this.failedDeps.size) { return }
  336. return sweep(
  337. tree,
  338. this.prefix,
  339. mark(tree, this.failedDeps)
  340. )
  341. .then(purged => {
  342. this.purgedDeps = purged
  343. this.pkgCount -= purged.size
  344. })
  345. }
  346. runScript (stage, pkg, pkgPath) {
  347. const start = Date.now()
  348. if (!this.opts['ignore-scripts']) {
  349. // TODO(mikesherov): remove pkg._id when npm-lifecycle no longer relies on it
  350. pkg._id = pkg.name + '@' + pkg.version
  351. return BB.resolve(lifecycle(
  352. pkg, stage, pkgPath, LifecycleOpts(this.opts).concat({
  353. // TODO: can be removed once npm-lifecycle is updated to modern
  354. // config practices.
  355. config: Object.assign({}, this.opts, {
  356. log: null,
  357. dirPacker: null
  358. }),
  359. dir: this.prefix
  360. }))
  361. ).tap(() => { this.timings.scripts += Date.now() - start })
  362. }
  363. return BB.resolve()
  364. }
  365. }
  366. module.exports = Installer
  367. function mark (tree, failed) {
  368. const liveDeps = new Set()
  369. tree.forEach((dep, next) => {
  370. if (!failed.has(dep)) {
  371. liveDeps.add(dep)
  372. next()
  373. }
  374. })
  375. return liveDeps
  376. }
  377. function sweep (tree, prefix, liveDeps) {
  378. const purged = new Set()
  379. return tree.forEachAsync((dep, next) => {
  380. return next().then(() => {
  381. if (
  382. !dep.isRoot && // never purge root! 🙈
  383. !liveDeps.has(dep) &&
  384. !purged.has(dep)
  385. ) {
  386. purged.add(dep)
  387. return rimraf(dep.path(prefix))
  388. }
  389. })
  390. }, {concurrency: 50, Promise: BB}).then(() => purged)
  391. }
  392. function stripBOM (str) {
  393. return str.replace(/^\uFEFF/, '')
  394. }
  395. module.exports._readJson = readJson
  396. function readJson (jsonPath, name, ignoreMissing) {
  397. return readFileAsync(path.join(jsonPath, name), 'utf8')
  398. .then(str => JSON.parse(stripBOM(str)))
  399. .catch({code: 'ENOENT'}, err => {
  400. if (!ignoreMissing) {
  401. throw err
  402. }
  403. })
  404. }