#!/usr/bin/env python # # Copyright 2006 The Closure Library Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS-IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Calculates JavaScript dependencies without requiring Google's build system. This tool is deprecated and is provided for legacy users. See build/closurebuilder.py and build/depswriter.py for the current tools. It iterates over a number of search paths and builds a dependency tree. With the inputs provided, it walks the dependency tree and outputs all the files required for compilation. """ try: import distutils.version except ImportError: # distutils is not available in all environments distutils = None import logging import optparse import os import re import subprocess import sys _BASE_REGEX_STRING = '^\s*goog\.%s\(\s*[\'"](.+)[\'"]\s*\)' req_regex = re.compile(_BASE_REGEX_STRING % 'require') prov_regex = re.compile(_BASE_REGEX_STRING % 'provide') ns_regex = re.compile('^ns:((\w+\.)*(\w+))$') version_regex = re.compile('[\.0-9]+') def IsValidFile(ref): """Returns true if the provided reference is a file and exists.""" return os.path.isfile(ref) def IsJsFile(ref): """Returns true if the provided reference is a Javascript file.""" return ref.endswith('.js') def IsNamespace(ref): """Returns true if the provided reference is a namespace.""" return re.match(ns_regex, ref) is not None def IsDirectory(ref): """Returns true if the provided reference is a directory.""" return os.path.isdir(ref) def ExpandDirectories(refs): """Expands any directory references into inputs. Description: Looks for any directories in the provided references. Found directories are recursively searched for .js files, which are then added to the result list. Args: refs: a list of references such as files, directories, and namespaces Returns: A list of references with directories removed and replaced by any .js files that are found in them. Also, the paths will be normalized. """ result = [] for ref in refs: if IsDirectory(ref): # Disable 'Unused variable' for subdirs # pylint: disable=unused-variable for (directory, subdirs, filenames) in os.walk(ref): for filename in filenames: if IsJsFile(filename): result.append(os.path.join(directory, filename)) else: result.append(ref) return map(os.path.normpath, result) class DependencyInfo(object): """Represents a dependency that is used to build and walk a tree.""" def __init__(self, filename): self.filename = filename self.provides = [] self.requires = [] def __str__(self): return '%s Provides: %s Requires: %s' % (self.filename, repr(self.provides), repr(self.requires)) def BuildDependenciesFromFiles(files): """Build a list of dependencies from a list of files. Description: Takes a list of files, extracts their provides and requires, and builds out a list of dependency objects. Args: files: a list of files to be parsed for goog.provides and goog.requires. Returns: A list of dependency objects, one for each file in the files argument. """ result = [] filenames = set() for filename in files: if filename in filenames: continue # Python 3 requires the file encoding to be specified if (sys.version_info[0] < 3): file_handle = open(filename, 'r') else: file_handle = open(filename, 'r', encoding='utf8') try: dep = CreateDependencyInfo(filename, file_handle) result.append(dep) finally: file_handle.close() filenames.add(filename) return result def CreateDependencyInfo(filename, source): """Create dependency info. Args: filename: Filename for source. source: File-like object containing source. Returns: A DependencyInfo object with provides and requires filled. """ dep = DependencyInfo(filename) for line in source: if re.match(req_regex, line): dep.requires.append(re.search(req_regex, line).group(1)) if re.match(prov_regex, line): dep.provides.append(re.search(prov_regex, line).group(1)) return dep def BuildDependencyHashFromDependencies(deps): """Builds a hash for searching dependencies by the namespaces they provide. Description: Dependency objects can provide multiple namespaces. This method enumerates the provides of each dependency and adds them to a hash that can be used to easily resolve a given dependency by a namespace it provides. Args: deps: a list of dependency objects used to build the hash. Raises: Exception: If a multiple files try to provide the same namepace. Returns: A hash table { namespace: dependency } that can be used to resolve a dependency by a namespace it provides. """ dep_hash = {} for dep in deps: for provide in dep.provides: if provide in dep_hash: raise Exception('Duplicate provide (%s) in (%s, %s)' % ( provide, dep_hash[provide].filename, dep.filename)) dep_hash[provide] = dep return dep_hash def CalculateDependencies(paths, inputs): """Calculates the dependencies for given inputs. Description: This method takes a list of paths (files, directories) and builds a searchable data structure based on the namespaces that each .js file provides. It then parses through each input, resolving dependencies against this data structure. The final output is a list of files, including the inputs, that represent all of the code that is needed to compile the given inputs. Args: paths: the references (files, directories) that are used to build the dependency hash. inputs: the inputs (files, directories, namespaces) that have dependencies that need to be calculated. Raises: Exception: if a provided input is invalid. Returns: A list of all files, including inputs, that are needed to compile the given inputs. """ deps = BuildDependenciesFromFiles(paths + inputs) search_hash = BuildDependencyHashFromDependencies(deps) result_list = [] seen_list = [] for input_file in inputs: if IsNamespace(input_file): namespace = re.search(ns_regex, input_file).group(1) if namespace not in search_hash: raise Exception('Invalid namespace (%s)' % namespace) input_file = search_hash[namespace].filename if not IsValidFile(input_file) or not IsJsFile(input_file): raise Exception('Invalid file (%s)' % input_file) seen_list.append(input_file) file_handle = open(input_file, 'r') try: for line in file_handle: if re.match(req_regex, line): require = re.search(req_regex, line).group(1) ResolveDependencies(require, search_hash, result_list, seen_list) finally: file_handle.close() result_list.append(input_file) # All files depend on base.js, so put it first. base_js_path = FindClosureBasePath(paths) if base_js_path: result_list.insert(0, base_js_path) else: logging.warning('Closure Library base.js not found.') return result_list def FindClosureBasePath(paths): """Given a list of file paths, return Closure base.js path, if any. Args: paths: A list of paths. Returns: The path to Closure's base.js file including filename, if found. """ for path in paths: pathname, filename = os.path.split(path) if filename == 'base.js': f = open(path) is_base = False # Sanity check that this is the Closure base file. Check that this # is where goog is defined. This is determined by the @provideGoog # flag. for line in f: if '@provideGoog' in line: is_base = True break f.close() if is_base: return path def ResolveDependencies(require, search_hash, result_list, seen_list): """Takes a given requirement and resolves all of the dependencies for it. Description: A given requirement may require other dependencies. This method recursively resolves all dependencies for the given requirement. Raises: Exception: when require does not exist in the search_hash. Args: require: the namespace to resolve dependencies for. search_hash: the data structure used for resolving dependencies. result_list: a list of filenames that have been calculated as dependencies. This variable is the output for this function. seen_list: a list of filenames that have been 'seen'. This is required for the dependency->dependent ordering. """ if require not in search_hash: raise Exception('Missing provider for (%s)' % require) dep = search_hash[require] if not dep.filename in seen_list: seen_list.append(dep.filename) for sub_require in dep.requires: ResolveDependencies(sub_require, search_hash, result_list, seen_list) result_list.append(dep.filename) def GetDepsLine(dep, base_path): """Returns a JS string for a dependency statement in the deps.js file. Args: dep: The dependency that we're printing. base_path: The path to Closure's base.js including filename. """ return 'goog.addDependency("%s", %s, %s);' % ( GetRelpath(dep.filename, base_path), dep.provides, dep.requires) def GetRelpath(path, start): """Return a relative path to |path| from |start|.""" # NOTE: Python 2.6 provides os.path.relpath, which has almost the same # functionality as this function. Since we want to support 2.4, we have # to implement it manually. :( path_list = os.path.abspath(os.path.normpath(path)).split(os.sep) start_list = os.path.abspath( os.path.normpath(os.path.dirname(start))).split(os.sep) common_prefix_count = 0 for i in range(0, min(len(path_list), len(start_list))): if path_list[i] != start_list[i]: break common_prefix_count += 1 # Always use forward slashes, because this will get expanded to a url, # not a file path. return '/'.join(['..'] * (len(start_list) - common_prefix_count) + path_list[common_prefix_count:]) def PrintLine(msg, out): out.write(msg) out.write('\n') def PrintDeps(source_paths, deps, out): """Print out a deps.js file from a list of source paths. Args: source_paths: Paths that we should generate dependency info for. deps: Paths that provide dependency info. Their dependency info should not appear in the deps file. out: The output file. Returns: True on success, false if it was unable to find the base path to generate deps relative to. """ base_path = FindClosureBasePath(source_paths + deps) if not base_path: return False PrintLine('// This file was autogenerated by calcdeps.py', out) excludesSet = set(deps) for dep in BuildDependenciesFromFiles(source_paths + deps): if not dep.filename in excludesSet: PrintLine(GetDepsLine(dep, base_path), out) return True def PrintScript(source_paths, out): for index, dep in enumerate(source_paths): PrintLine('// Input %d' % index, out) f = open(dep, 'r') PrintLine(f.read(), out) f.close() def GetJavaVersion(): """Returns the string for the current version of Java installed.""" proc = subprocess.Popen(['java', '-version'], stderr=subprocess.PIPE) proc.wait() version_line = proc.stderr.read().splitlines()[0] return version_regex.search(version_line.decode('utf-8')).group() def FilterByExcludes(options, files): """Filters the given files by the exlusions specified at the command line. Args: options: The flags to calcdeps. files: The files to filter. Returns: A list of files. """ excludes = [] if options.excludes: excludes = ExpandDirectories(options.excludes) excludesSet = set(excludes) return [i for i in files if not i in excludesSet] def GetPathsFromOptions(options): """Generates the path files from flag options. Args: options: The flags to calcdeps. Returns: A list of files in the specified paths. (strings). """ search_paths = options.paths if not search_paths: search_paths = ['.'] # Add default folder if no path is specified. search_paths = ExpandDirectories(search_paths) return FilterByExcludes(options, search_paths) def GetInputsFromOptions(options): """Generates the inputs from flag options. Args: options: The flags to calcdeps. Returns: A list of inputs (strings). """ inputs = options.inputs if not inputs: # Parse stdin logging.info('No inputs specified. Reading from stdin...') inputs = filter(None, [line.strip('\n') for line in sys.stdin.readlines()]) logging.info('Scanning files...') inputs = ExpandDirectories(inputs) return FilterByExcludes(options, inputs) def Compile(compiler_jar_path, source_paths, out, flags=None): """Prepares command-line call to Closure compiler. Args: compiler_jar_path: Path to the Closure compiler .jar file. source_paths: Source paths to build, in order. flags: A list of additional flags to pass on to Closure compiler. """ args = ['java', '-jar', compiler_jar_path] for path in source_paths: args += ['--js', path] if flags: args += flags logging.info('Compiling with the following command: %s', ' '.join(args)) proc = subprocess.Popen(args, stdout=subprocess.PIPE) (stdoutdata, stderrdata) = proc.communicate() if proc.returncode != 0: logging.error('JavaScript compilation failed.') sys.exit(1) else: out.write(stdoutdata.decode('utf-8')) def main(): """The entrypoint for this script.""" logging.basicConfig(format='calcdeps.py: %(message)s', level=logging.INFO) usage = 'usage: %prog [options] arg' parser = optparse.OptionParser(usage) parser.add_option('-i', '--input', dest='inputs', action='append', help='The inputs to calculate dependencies for. Valid ' 'values can be files, directories, or namespaces ' '(ns:goog.net.XhrIo). Only relevant to "list" and ' '"script" output.') parser.add_option('-p', '--path', dest='paths', action='append', help='The paths that should be traversed to build the ' 'dependencies.') parser.add_option('-d', '--dep', dest='deps', action='append', help='Directories or files that should be traversed to ' 'find required dependencies for the deps file. ' 'Does not generate dependency information for names ' 'provided by these files. Only useful in "deps" mode.') parser.add_option('-e', '--exclude', dest='excludes', action='append', help='Files or directories to exclude from the --path ' 'and --input flags') parser.add_option('-o', '--output_mode', dest='output_mode', action='store', default='list', help='The type of output to generate from this script. ' 'Options are "list" for a list of filenames, "script" ' 'for a single script containing the contents of all the ' 'file, "deps" to generate a deps.js file for all ' 'paths, or "compiled" to produce compiled output with ' 'the Closure compiler.') parser.add_option('-c', '--compiler_jar', dest='compiler_jar', action='store', help='The location of the Closure compiler .jar file.') parser.add_option('-f', '--compiler_flag', '--compiler_flags', # for backwards compatibility dest='compiler_flags', action='append', help='Additional flag to pass to the Closure compiler. ' 'May be specified multiple times to pass multiple flags.') parser.add_option('--output_file', dest='output_file', action='store', help=('If specified, write output to this path instead of ' 'writing to standard output.')) (options, args) = parser.parse_args() search_paths = GetPathsFromOptions(options) if options.output_file: out = open(options.output_file, 'w') else: out = sys.stdout if options.output_mode == 'deps': result = PrintDeps(search_paths, ExpandDirectories(options.deps or []), out) if not result: logging.error('Could not find Closure Library in the specified paths') sys.exit(1) return inputs = GetInputsFromOptions(options) logging.info('Finding Closure dependencies...') deps = CalculateDependencies(search_paths, inputs) output_mode = options.output_mode if output_mode == 'script': PrintScript(deps, out) elif output_mode == 'list': # Just print out a dep per line for dep in deps: PrintLine(dep, out) elif output_mode == 'compiled': # Make sure a .jar is specified. if not options.compiler_jar: logging.error('--compiler_jar flag must be specified if --output is ' '"compiled"') sys.exit(1) # User friendly version check. if distutils and not (distutils.version.LooseVersion(GetJavaVersion()) > distutils.version.LooseVersion('1.6')): logging.error('Closure Compiler requires Java 1.6 or higher.') logging.error('Please visit http://www.java.com/getjava') sys.exit(1) Compile(options.compiler_jar, deps, out, options.compiler_flags) else: logging.error('Invalid value for --output flag.') sys.exit(1) if __name__ == '__main__': main()