// this keeps a queue of opened file descriptors, and will make // fs operations wait until some have closed before trying to open more. var fs = exports = module.exports = {} fs._originalFs = require("fs") Object.getOwnPropertyNames(fs._originalFs).forEach(function(prop) { var desc = Object.getOwnPropertyDescriptor(fs._originalFs, prop) Object.defineProperty(fs, prop, desc) }) var queue = [] , constants = require("constants") fs._curOpen = 0 fs.MIN_MAX_OPEN = 64 fs.MAX_OPEN = 1024 // prevent EMFILE errors function OpenReq (path, flags, mode, cb) { this.path = path this.flags = flags this.mode = mode this.cb = cb } function noop () {} fs.open = gracefulOpen function gracefulOpen (path, flags, mode, cb) { if (typeof mode === "function") cb = mode, mode = null if (typeof cb !== "function") cb = noop if (fs._curOpen >= fs.MAX_OPEN) { queue.push(new OpenReq(path, flags, mode, cb)) setTimeout(flush) return } open(path, flags, mode, function (er, fd) { if (er && er.code === "EMFILE" && fs._curOpen > fs.MIN_MAX_OPEN) { // that was too many. reduce max, get back in queue. // this should only happen once in a great while, and only // if the ulimit -n is set lower than 1024. fs.MAX_OPEN = fs._curOpen - 1 return fs.open(path, flags, mode, cb) } cb(er, fd) }) } function open (path, flags, mode, cb) { cb = cb || noop fs._curOpen ++ fs._originalFs.open.call(fs, path, flags, mode, function (er, fd) { if (er) onclose() cb(er, fd) }) } fs.openSync = function (path, flags, mode) { var ret ret = fs._originalFs.openSync.call(fs, path, flags, mode) fs._curOpen ++ return ret } function onclose () { fs._curOpen -- flush() } function flush () { while (fs._curOpen < fs.MAX_OPEN) { var req = queue.shift() if (!req) return switch (req.constructor.name) { case 'OpenReq': open(req.path, req.flags || "r", req.mode || 0777, req.cb) break case 'ReaddirReq': readdir(req.path, req.cb) break case 'ReadFileReq': readFile(req.path, req.options, req.cb) break case 'WriteFileReq': writeFile(req.path, req.data, req.options, req.cb) break default: throw new Error('Unknown req type: ' + req.constructor.name) } } } fs.close = function (fd, cb) { cb = cb || noop fs._originalFs.close.call(fs, fd, function (er) { onclose() cb(er) }) } fs.closeSync = function (fd) { try { return fs._originalFs.closeSync.call(fs, fd) } finally { onclose() } } // readdir takes a fd as well. // however, the sync version closes it right away, so // there's no need to wrap. // It would be nice to catch when it throws an EMFILE, // but that's relatively rare anyway. fs.readdir = gracefulReaddir function gracefulReaddir (path, cb) { if (fs._curOpen >= fs.MAX_OPEN) { queue.push(new ReaddirReq(path, cb)) setTimeout(flush) return } readdir(path, function (er, files) { if (er && er.code === "EMFILE" && fs._curOpen > fs.MIN_MAX_OPEN) { fs.MAX_OPEN = fs._curOpen - 1 return fs.readdir(path, cb) } cb(er, files) }) } function readdir (path, cb) { cb = cb || noop fs._curOpen ++ fs._originalFs.readdir.call(fs, path, function (er, files) { onclose() cb(er, files) }) } function ReaddirReq (path, cb) { this.path = path this.cb = cb } fs.readFile = gracefulReadFile function gracefulReadFile(path, options, cb) { if (typeof options === "function") cb = options, options = null if (typeof cb !== "function") cb = noop if (fs._curOpen >= fs.MAX_OPEN) { queue.push(new ReadFileReq(path, options, cb)) setTimeout(flush) return } readFile(path, options, function (er, data) { if (er && er.code === "EMFILE" && fs._curOpen > fs.MIN_MAX_OPEN) { fs.MAX_OPEN = fs._curOpen - 1 return fs.readFile(path, options, cb) } cb(er, data) }) } function readFile (path, options, cb) { cb = cb || noop fs._curOpen ++ fs._originalFs.readFile.call(fs, path, options, function (er, data) { onclose() cb(er, data) }) } function ReadFileReq (path, options, cb) { this.path = path this.options = options this.cb = cb } fs.writeFile = gracefulWriteFile function gracefulWriteFile(path, data, options, cb) { if (typeof options === "function") cb = options, options = null if (typeof cb !== "function") cb = noop if (fs._curOpen >= fs.MAX_OPEN) { queue.push(new WriteFileReq(path, data, options, cb)) setTimeout(flush) return } writeFile(path, data, options, function (er) { if (er && er.code === "EMFILE" && fs._curOpen > fs.MIN_MAX_OPEN) { fs.MAX_OPEN = fs._curOpen - 1 return fs.writeFile(path, data, options, cb) } cb(er) }) } function writeFile (path, data, options, cb) { cb = cb || noop fs._curOpen ++ fs._originalFs.writeFile.call(fs, path, data, options, function (er) { onclose() cb(er) }) } function WriteFileReq (path, data, options, cb) { this.path = path this.data = data this.options = options this.cb = cb } // (re-)implement some things that are known busted or missing. var constants = require("constants") // lchmod, broken prior to 0.6.2 // back-port the fix here. if (constants.hasOwnProperty('O_SYMLINK') && process.version.match(/^v0\.6\.[0-2]|^v0\.5\./)) { fs.lchmod = function (path, mode, callback) { callback = callback || noop fs.open( path , constants.O_WRONLY | constants.O_SYMLINK , mode , function (err, fd) { if (err) { callback(err) return } // prefer to return the chmod error, if one occurs, // but still try to close, and report closing errors if they occur. fs.fchmod(fd, mode, function (err) { fs.close(fd, function(err2) { callback(err || err2) }) }) }) } fs.lchmodSync = function (path, mode) { var fd = fs.openSync(path, constants.O_WRONLY | constants.O_SYMLINK, mode) // prefer to return the chmod error, if one occurs, // but still try to close, and report closing errors if they occur. var err, err2 try { var ret = fs.fchmodSync(fd, mode) } catch (er) { err = er } try { fs.closeSync(fd) } catch (er) { err2 = er } if (err || err2) throw (err || err2) return ret } } // lutimes implementation, or no-op if (!fs.lutimes) { if (constants.hasOwnProperty("O_SYMLINK")) { fs.lutimes = function (path, at, mt, cb) { fs.open(path, constants.O_SYMLINK, function (er, fd) { cb = cb || noop if (er) return cb(er) fs.futimes(fd, at, mt, function (er) { fs.close(fd, function (er2) { return cb(er || er2) }) }) }) } fs.lutimesSync = function (path, at, mt) { var fd = fs.openSync(path, constants.O_SYMLINK) , err , err2 , ret try { var ret = fs.futimesSync(fd, at, mt) } catch (er) { err = er } try { fs.closeSync(fd) } catch (er) { err2 = er } if (err || err2) throw (err || err2) return ret } } else if (fs.utimensat && constants.hasOwnProperty("AT_SYMLINK_NOFOLLOW")) { // maybe utimensat will be bound soonish? fs.lutimes = function (path, at, mt, cb) { fs.utimensat(path, at, mt, constants.AT_SYMLINK_NOFOLLOW, cb) } fs.lutimesSync = function (path, at, mt) { return fs.utimensatSync(path, at, mt, constants.AT_SYMLINK_NOFOLLOW) } } else { fs.lutimes = function (_a, _b, _c, cb) { process.nextTick(cb) } fs.lutimesSync = function () {} } } // https://github.com/isaacs/node-graceful-fs/issues/4 // Chown should not fail on einval or eperm if non-root. fs.chown = chownFix(fs.chown) fs.fchown = chownFix(fs.fchown) fs.lchown = chownFix(fs.lchown) fs.chownSync = chownFixSync(fs.chownSync) fs.fchownSync = chownFixSync(fs.fchownSync) fs.lchownSync = chownFixSync(fs.lchownSync) function chownFix (orig) { if (!orig) return orig return function (target, uid, gid, cb) { return orig.call(fs, target, uid, gid, function (er, res) { if (chownErOk(er)) er = null cb(er, res) }) } } function chownFixSync (orig) { if (!orig) return orig return function (target, uid, gid) { try { return orig.call(fs, target, uid, gid) } catch (er) { if (!chownErOk(er)) throw er } } } function chownErOk (er) { // if there's no getuid, or if getuid() is something other than 0, // and the error is EINVAL or EPERM, then just ignore it. // This specific case is a silent failure in cp, install, tar, // and most other unix tools that manage permissions. // When running as root, or if other types of errors are encountered, // then it's strict. if (!er || (!process.getuid || process.getuid() !== 0) && (er.code === "EINVAL" || er.code === "EPERM")) return true } // if lchmod/lchown do not exist, then make them no-ops if (!fs.lchmod) { fs.lchmod = function (path, mode, cb) { process.nextTick(cb) } fs.lchmodSync = function () {} } if (!fs.lchown) { fs.lchown = function (path, uid, gid, cb) { process.nextTick(cb) } fs.lchownSync = function () {} } // on Windows, A/V software can lock the directory, causing this // to fail with an EACCES or EPERM if the directory contains newly // created files. Try again on failure, for up to 1 second. if (process.platform === "win32") { var rename_ = fs.rename fs.rename = function rename (from, to, cb) { var start = Date.now() rename_(from, to, function CB (er) { if (er && (er.code === "EACCES" || er.code === "EPERM") && Date.now() - start < 1000) { return rename_(from, to, CB) } cb(er) }) } } // if read() returns EAGAIN, then just try it again. var read = fs.read fs.read = function (fd, buffer, offset, length, position, callback_) { var callback if (callback_ && typeof callback_ === 'function') { var eagCounter = 0 callback = function (er, _, __) { if (er && er.code === 'EAGAIN' && eagCounter < 10) { eagCounter ++ return read.call(fs, fd, buffer, offset, length, position, callback) } callback_.apply(this, arguments) } } return read.call(fs, fd, buffer, offset, length, position, callback) } var readSync = fs.readSync fs.readSync = function (fd, buffer, offset, length, position) { var eagCounter = 0 while (true) { try { return readSync.call(fs, fd, buffer, offset, length, position) } catch (er) { if (er.code === 'EAGAIN' && eagCounter < 10) { eagCounter ++ continue } throw er } } }