graceful-fs.js 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442
  1. // this keeps a queue of opened file descriptors, and will make
  2. // fs operations wait until some have closed before trying to open more.
  3. var fs = exports = module.exports = {}
  4. fs._originalFs = require("fs")
  5. Object.getOwnPropertyNames(fs._originalFs).forEach(function(prop) {
  6. var desc = Object.getOwnPropertyDescriptor(fs._originalFs, prop)
  7. Object.defineProperty(fs, prop, desc)
  8. })
  9. var queue = []
  10. , constants = require("constants")
  11. fs._curOpen = 0
  12. fs.MIN_MAX_OPEN = 64
  13. fs.MAX_OPEN = 1024
  14. // prevent EMFILE errors
  15. function OpenReq (path, flags, mode, cb) {
  16. this.path = path
  17. this.flags = flags
  18. this.mode = mode
  19. this.cb = cb
  20. }
  21. function noop () {}
  22. fs.open = gracefulOpen
  23. function gracefulOpen (path, flags, mode, cb) {
  24. if (typeof mode === "function") cb = mode, mode = null
  25. if (typeof cb !== "function") cb = noop
  26. if (fs._curOpen >= fs.MAX_OPEN) {
  27. queue.push(new OpenReq(path, flags, mode, cb))
  28. setTimeout(flush)
  29. return
  30. }
  31. open(path, flags, mode, function (er, fd) {
  32. if (er && er.code === "EMFILE" && fs._curOpen > fs.MIN_MAX_OPEN) {
  33. // that was too many. reduce max, get back in queue.
  34. // this should only happen once in a great while, and only
  35. // if the ulimit -n is set lower than 1024.
  36. fs.MAX_OPEN = fs._curOpen - 1
  37. return fs.open(path, flags, mode, cb)
  38. }
  39. cb(er, fd)
  40. })
  41. }
  42. function open (path, flags, mode, cb) {
  43. cb = cb || noop
  44. fs._curOpen ++
  45. fs._originalFs.open.call(fs, path, flags, mode, function (er, fd) {
  46. if (er) onclose()
  47. cb(er, fd)
  48. })
  49. }
  50. fs.openSync = function (path, flags, mode) {
  51. var ret
  52. ret = fs._originalFs.openSync.call(fs, path, flags, mode)
  53. fs._curOpen ++
  54. return ret
  55. }
  56. function onclose () {
  57. fs._curOpen --
  58. flush()
  59. }
  60. function flush () {
  61. while (fs._curOpen < fs.MAX_OPEN) {
  62. var req = queue.shift()
  63. if (!req) return
  64. switch (req.constructor.name) {
  65. case 'OpenReq':
  66. open(req.path, req.flags || "r", req.mode || 0777, req.cb)
  67. break
  68. case 'ReaddirReq':
  69. readdir(req.path, req.cb)
  70. break
  71. case 'ReadFileReq':
  72. readFile(req.path, req.options, req.cb)
  73. break
  74. case 'WriteFileReq':
  75. writeFile(req.path, req.data, req.options, req.cb)
  76. break
  77. default:
  78. throw new Error('Unknown req type: ' + req.constructor.name)
  79. }
  80. }
  81. }
  82. fs.close = function (fd, cb) {
  83. cb = cb || noop
  84. fs._originalFs.close.call(fs, fd, function (er) {
  85. onclose()
  86. cb(er)
  87. })
  88. }
  89. fs.closeSync = function (fd) {
  90. try {
  91. return fs._originalFs.closeSync.call(fs, fd)
  92. } finally {
  93. onclose()
  94. }
  95. }
  96. // readdir takes a fd as well.
  97. // however, the sync version closes it right away, so
  98. // there's no need to wrap.
  99. // It would be nice to catch when it throws an EMFILE,
  100. // but that's relatively rare anyway.
  101. fs.readdir = gracefulReaddir
  102. function gracefulReaddir (path, cb) {
  103. if (fs._curOpen >= fs.MAX_OPEN) {
  104. queue.push(new ReaddirReq(path, cb))
  105. setTimeout(flush)
  106. return
  107. }
  108. readdir(path, function (er, files) {
  109. if (er && er.code === "EMFILE" && fs._curOpen > fs.MIN_MAX_OPEN) {
  110. fs.MAX_OPEN = fs._curOpen - 1
  111. return fs.readdir(path, cb)
  112. }
  113. cb(er, files)
  114. })
  115. }
  116. function readdir (path, cb) {
  117. cb = cb || noop
  118. fs._curOpen ++
  119. fs._originalFs.readdir.call(fs, path, function (er, files) {
  120. onclose()
  121. cb(er, files)
  122. })
  123. }
  124. function ReaddirReq (path, cb) {
  125. this.path = path
  126. this.cb = cb
  127. }
  128. fs.readFile = gracefulReadFile
  129. function gracefulReadFile(path, options, cb) {
  130. if (typeof options === "function") cb = options, options = null
  131. if (typeof cb !== "function") cb = noop
  132. if (fs._curOpen >= fs.MAX_OPEN) {
  133. queue.push(new ReadFileReq(path, options, cb))
  134. setTimeout(flush)
  135. return
  136. }
  137. readFile(path, options, function (er, data) {
  138. if (er && er.code === "EMFILE" && fs._curOpen > fs.MIN_MAX_OPEN) {
  139. fs.MAX_OPEN = fs._curOpen - 1
  140. return fs.readFile(path, options, cb)
  141. }
  142. cb(er, data)
  143. })
  144. }
  145. function readFile (path, options, cb) {
  146. cb = cb || noop
  147. fs._curOpen ++
  148. fs._originalFs.readFile.call(fs, path, options, function (er, data) {
  149. onclose()
  150. cb(er, data)
  151. })
  152. }
  153. function ReadFileReq (path, options, cb) {
  154. this.path = path
  155. this.options = options
  156. this.cb = cb
  157. }
  158. fs.writeFile = gracefulWriteFile
  159. function gracefulWriteFile(path, data, options, cb) {
  160. if (typeof options === "function") cb = options, options = null
  161. if (typeof cb !== "function") cb = noop
  162. if (fs._curOpen >= fs.MAX_OPEN) {
  163. queue.push(new WriteFileReq(path, data, options, cb))
  164. setTimeout(flush)
  165. return
  166. }
  167. writeFile(path, data, options, function (er) {
  168. if (er && er.code === "EMFILE" && fs._curOpen > fs.MIN_MAX_OPEN) {
  169. fs.MAX_OPEN = fs._curOpen - 1
  170. return fs.writeFile(path, data, options, cb)
  171. }
  172. cb(er)
  173. })
  174. }
  175. function writeFile (path, data, options, cb) {
  176. cb = cb || noop
  177. fs._curOpen ++
  178. fs._originalFs.writeFile.call(fs, path, data, options, function (er) {
  179. onclose()
  180. cb(er)
  181. })
  182. }
  183. function WriteFileReq (path, data, options, cb) {
  184. this.path = path
  185. this.data = data
  186. this.options = options
  187. this.cb = cb
  188. }
  189. // (re-)implement some things that are known busted or missing.
  190. var constants = require("constants")
  191. // lchmod, broken prior to 0.6.2
  192. // back-port the fix here.
  193. if (constants.hasOwnProperty('O_SYMLINK') &&
  194. process.version.match(/^v0\.6\.[0-2]|^v0\.5\./)) {
  195. fs.lchmod = function (path, mode, callback) {
  196. callback = callback || noop
  197. fs.open( path
  198. , constants.O_WRONLY | constants.O_SYMLINK
  199. , mode
  200. , function (err, fd) {
  201. if (err) {
  202. callback(err)
  203. return
  204. }
  205. // prefer to return the chmod error, if one occurs,
  206. // but still try to close, and report closing errors if they occur.
  207. fs.fchmod(fd, mode, function (err) {
  208. fs.close(fd, function(err2) {
  209. callback(err || err2)
  210. })
  211. })
  212. })
  213. }
  214. fs.lchmodSync = function (path, mode) {
  215. var fd = fs.openSync(path, constants.O_WRONLY | constants.O_SYMLINK, mode)
  216. // prefer to return the chmod error, if one occurs,
  217. // but still try to close, and report closing errors if they occur.
  218. var err, err2
  219. try {
  220. var ret = fs.fchmodSync(fd, mode)
  221. } catch (er) {
  222. err = er
  223. }
  224. try {
  225. fs.closeSync(fd)
  226. } catch (er) {
  227. err2 = er
  228. }
  229. if (err || err2) throw (err || err2)
  230. return ret
  231. }
  232. }
  233. // lutimes implementation, or no-op
  234. if (!fs.lutimes) {
  235. if (constants.hasOwnProperty("O_SYMLINK")) {
  236. fs.lutimes = function (path, at, mt, cb) {
  237. fs.open(path, constants.O_SYMLINK, function (er, fd) {
  238. cb = cb || noop
  239. if (er) return cb(er)
  240. fs.futimes(fd, at, mt, function (er) {
  241. fs.close(fd, function (er2) {
  242. return cb(er || er2)
  243. })
  244. })
  245. })
  246. }
  247. fs.lutimesSync = function (path, at, mt) {
  248. var fd = fs.openSync(path, constants.O_SYMLINK)
  249. , err
  250. , err2
  251. , ret
  252. try {
  253. var ret = fs.futimesSync(fd, at, mt)
  254. } catch (er) {
  255. err = er
  256. }
  257. try {
  258. fs.closeSync(fd)
  259. } catch (er) {
  260. err2 = er
  261. }
  262. if (err || err2) throw (err || err2)
  263. return ret
  264. }
  265. } else if (fs.utimensat && constants.hasOwnProperty("AT_SYMLINK_NOFOLLOW")) {
  266. // maybe utimensat will be bound soonish?
  267. fs.lutimes = function (path, at, mt, cb) {
  268. fs.utimensat(path, at, mt, constants.AT_SYMLINK_NOFOLLOW, cb)
  269. }
  270. fs.lutimesSync = function (path, at, mt) {
  271. return fs.utimensatSync(path, at, mt, constants.AT_SYMLINK_NOFOLLOW)
  272. }
  273. } else {
  274. fs.lutimes = function (_a, _b, _c, cb) { process.nextTick(cb) }
  275. fs.lutimesSync = function () {}
  276. }
  277. }
  278. // https://github.com/isaacs/node-graceful-fs/issues/4
  279. // Chown should not fail on einval or eperm if non-root.
  280. fs.chown = chownFix(fs.chown)
  281. fs.fchown = chownFix(fs.fchown)
  282. fs.lchown = chownFix(fs.lchown)
  283. fs.chownSync = chownFixSync(fs.chownSync)
  284. fs.fchownSync = chownFixSync(fs.fchownSync)
  285. fs.lchownSync = chownFixSync(fs.lchownSync)
  286. function chownFix (orig) {
  287. if (!orig) return orig
  288. return function (target, uid, gid, cb) {
  289. return orig.call(fs, target, uid, gid, function (er, res) {
  290. if (chownErOk(er)) er = null
  291. cb(er, res)
  292. })
  293. }
  294. }
  295. function chownFixSync (orig) {
  296. if (!orig) return orig
  297. return function (target, uid, gid) {
  298. try {
  299. return orig.call(fs, target, uid, gid)
  300. } catch (er) {
  301. if (!chownErOk(er)) throw er
  302. }
  303. }
  304. }
  305. function chownErOk (er) {
  306. // if there's no getuid, or if getuid() is something other than 0,
  307. // and the error is EINVAL or EPERM, then just ignore it.
  308. // This specific case is a silent failure in cp, install, tar,
  309. // and most other unix tools that manage permissions.
  310. // When running as root, or if other types of errors are encountered,
  311. // then it's strict.
  312. if (!er || (!process.getuid || process.getuid() !== 0)
  313. && (er.code === "EINVAL" || er.code === "EPERM")) return true
  314. }
  315. // if lchmod/lchown do not exist, then make them no-ops
  316. if (!fs.lchmod) {
  317. fs.lchmod = function (path, mode, cb) {
  318. process.nextTick(cb)
  319. }
  320. fs.lchmodSync = function () {}
  321. }
  322. if (!fs.lchown) {
  323. fs.lchown = function (path, uid, gid, cb) {
  324. process.nextTick(cb)
  325. }
  326. fs.lchownSync = function () {}
  327. }
  328. // on Windows, A/V software can lock the directory, causing this
  329. // to fail with an EACCES or EPERM if the directory contains newly
  330. // created files. Try again on failure, for up to 1 second.
  331. if (process.platform === "win32") {
  332. var rename_ = fs.rename
  333. fs.rename = function rename (from, to, cb) {
  334. var start = Date.now()
  335. rename_(from, to, function CB (er) {
  336. if (er
  337. && (er.code === "EACCES" || er.code === "EPERM")
  338. && Date.now() - start < 1000) {
  339. return rename_(from, to, CB)
  340. }
  341. cb(er)
  342. })
  343. }
  344. }
  345. // if read() returns EAGAIN, then just try it again.
  346. var read = fs.read
  347. fs.read = function (fd, buffer, offset, length, position, callback_) {
  348. var callback
  349. if (callback_ && typeof callback_ === 'function') {
  350. var eagCounter = 0
  351. callback = function (er, _, __) {
  352. if (er && er.code === 'EAGAIN' && eagCounter < 10) {
  353. eagCounter ++
  354. return read.call(fs, fd, buffer, offset, length, position, callback)
  355. }
  356. callback_.apply(this, arguments)
  357. }
  358. }
  359. return read.call(fs, fd, buffer, offset, length, position, callback)
  360. }
  361. var readSync = fs.readSync
  362. fs.readSync = function (fd, buffer, offset, length, position) {
  363. var eagCounter = 0
  364. while (true) {
  365. try {
  366. return readSync.call(fs, fd, buffer, offset, length, position)
  367. } catch (er) {
  368. if (er.code === 'EAGAIN' && eagCounter < 10) {
  369. eagCounter ++
  370. continue
  371. }
  372. throw er
  373. }
  374. }
  375. }