From a1449e44dcc5f5058c8a83668b03f9682e09726a Mon Sep 17 00:00:00 2001 From: Jacob Trimble Date: Wed, 15 Jun 2016 12:21:31 -0700 Subject: [PATCH] Make build files consistent with Google python style guide. Change-Id: I49d181c4dbe4fe12b73cdc4a5c78b3dcf1319b56 --- build/all.py | 11 +- build/build.py | 275 ++++++++-------- build/check.py | 98 +++--- build/checkversion.py | 31 +- build/docs.py | 28 +- build/gendeps.py | 33 +- build/shakaBuildHelpers.py | 162 ++++++---- build/stats.py | 619 +++++++++++++++++++++---------------- build/test.py | 49 +-- 9 files changed, 729 insertions(+), 577 deletions(-) diff --git a/build/all.py b/build/all.py index 8c7387081..ae6fe564a 100755 --- a/build/all.py +++ b/build/all.py @@ -1,6 +1,6 @@ #!/usr/bin/python # -# Copyright 2016 Google Inc. +# Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,13 +14,16 @@ # See the License for the specific language governing permissions and # limitations under the License. -import check +"""Builds the dependencies, runs the checks, and compiles the library.""" + import build +import check import gendeps import shakaBuildHelpers + def main(args): - code = gendeps.genDeps([]) + code = gendeps.gen_deps([]) if code != 0: return code @@ -36,4 +39,4 @@ def main(args): return build.main(build_args) if __name__ == '__main__': - shakaBuildHelpers.runMain(main) + shakaBuildHelpers.run_main(main) diff --git a/build/build.py b/build/build.py index eed1feace..7c1af9951 100755 --- a/build/build.py +++ b/build/build.py @@ -1,6 +1,6 @@ #!/usr/bin/python # -# Copyright 2016 Google Inc. +# Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,10 +14,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Creates a build from the given commands. A command is either an addition or -a subtraction. An addition is prefixed with a +; a subtraction is when -prefixed with a -. After the character, there is a name of a file or a @ sign -and the name of a build file. +"""Creates a build from the given commands. + +A command is either an addition or a subtraction. An addition is prefixed with +a +; a subtraction is when prefixed with a -. After the character, there is a +name of a file or a @ sign and the name of a build file. Build files are the files found in build/types. These files are simply a newline separated list of commands to execute. So if the "+@complete" command @@ -35,58 +36,64 @@ Examples: build.py +@complete build.py +@complete -@networking - build.py --name custom +@manifests +@networking +../my_plugin.js""" + build.py --name custom +@manifests +@networking +../my_plugin.js +""" import os import re -import shakaBuildHelpers import shutil import subprocess import sys +import shakaBuildHelpers + + closure_opts = [ - '--language_in', 'ECMASCRIPT5', - '--language_out', 'ECMASCRIPT3', + '--language_in', 'ECMASCRIPT5', + '--language_out', 'ECMASCRIPT3', - '--jscomp_error=*', + '--jscomp_error=*', - # 'deprecatedAnnotations' controls complains about @expose, but the new - # @nocollapse annotation does not do the same job for properties. - # So since we can't use the new annotations, we have to ignore complaints - # about the old one. - '--jscomp_off=deprecatedAnnotations', + # 'deprecatedAnnotations' controls complains about @expose, but the new + # @nocollapse annotation does not do the same job for properties. + # So since we can't use the new annotations, we have to ignore complaints + # about the old one. + '--jscomp_off=deprecatedAnnotations', - # 'analyzerChecks' complains about countless instances of implicitly nullable - # types, plus a few other issues. Even the closure library doesn't pass - # these checks, and the implicit nullability check in particular is over- - # zealous and unhelpful. So we disable the whole category of - # 'analyzerChecks'. - '--jscomp_off=analyzerChecks', + # 'analyzerChecks' complains about countless instances of implicitly + # nullable types, plus a few other issues. Even the closure library doesn't + # pass these checks, and the implicit nullability check in particular is + # over-zealous and unhelpful. So we disable the whole category of + # 'analyzerChecks'. + '--jscomp_off=analyzerChecks', - '--extra_annotation_name=listens', - '--extra_annotation_name=exportDoc', + '--extra_annotation_name=listens', + '--extra_annotation_name=exportDoc', - '--conformance_configs', '%s/build/conformance.textproto' % \ - shakaBuildHelpers.cygwinSafePath(shakaBuildHelpers.getSourceBase()), + '--conformance_configs', + ('%s/build/conformance.textproto' % + shakaBuildHelpers.cygwin_safe_path(shakaBuildHelpers.get_source_base())), - '-O', 'ADVANCED', - '--generate_exports', - '--output_wrapper_file=%s/build/wrapper.template.js' % \ - shakaBuildHelpers.cygwinSafePath(shakaBuildHelpers.getSourceBase()), + '-O', 'ADVANCED', + '--generate_exports', + ('--output_wrapper_file=%s/build/wrapper.template.js' % + shakaBuildHelpers.cygwin_safe_path(shakaBuildHelpers.get_source_base())), - '-D', 'COMPILED=true', - '-D', 'goog.DEBUG=false', - '-D', 'goog.STRICT_MODE_COMPATIBLE=true', - '-D', 'goog.ENABLE_DEBUG_LOADER=false', - '-D', 'goog.asserts.ENABLE_ASSERTS=false', - '-D', 'shaka.log.MAX_LOG_LEVEL=0', - '-D', 'GIT_VERSION="%s"' % shakaBuildHelpers.calculateVersion() + '-D', 'COMPILED=true', + '-D', 'goog.DEBUG=false', + '-D', 'goog.STRICT_MODE_COMPATIBLE=true', + '-D', 'goog.ENABLE_DEBUG_LOADER=false', + '-D', 'goog.asserts.ENABLE_ASSERTS=false', + '-D', 'shaka.log.MAX_LOG_LEVEL=0', + '-D', 'GIT_VERSION="%s"' % shakaBuildHelpers.calculate_version() ] -class Build: - """Defines a build that has been parsed from a build file. This has - exclude files even though it will not be used at the top-level. This allows - combining builds. A file will only exist in at most one set. + +class Build(object): + """Defines a build that has been parsed from a build file. + + This has exclude files even though it will not be used at the top-level. This + allows combining builds. A file will only exist in at most one set. Members: include - A set of files to include. @@ -97,65 +104,67 @@ class Build: self.include = include or set() self.exclude = exclude or set() - def _getBuildFilePath(self, name, root): - """Gets the full path to a build file, if it exists. Returns None if not. + def _get_build_file_path(self, name, root): + """Gets the full path to a build file, if it exists. - Arguments: - name - The string name to check. + Args: + name: The string name to check. + root: The full path to the base directory. Returns: - The full path to the build file. + The full path to the build file, or None if not found. """ - sourceBase = shakaBuildHelpers.getSourceBase() - localPath = os.path.join(root, name) - buildPath = os.path.join(sourceBase, 'build', 'types', name) - if (os.path.isfile(localPath) and os.path.isfile(buildPath) - and localPath != buildPath): + source_base = shakaBuildHelpers.get_source_base() + local_path = os.path.join(root, name) + build_path = os.path.join(source_base, 'build', 'types', name) + if (os.path.isfile(local_path) and os.path.isfile(build_path) + and local_path != build_path): print >> sys.stderr, 'Build file "%s" is ambiguous' % name return None - elif os.path.isfile(localPath): - return localPath - elif os.path.isfile(buildPath): - return buildPath + elif os.path.isfile(local_path): + return local_path + elif os.path.isfile(build_path): + return build_path else: print >> sys.stderr, 'Build file not found: ' + name return None - def _reverse(self): + def _combine(self, other): + include_all = self.include | other.include + exclude_all = self.exclude | other.exclude + self.include = include_all - exclude_all + self.exclude = exclude_all - include_all + + def reverse(self): return Build(self.exclude, self.include) - def _combine(self, other): - includeAll = self.include | other.include - excludeAll = self.exclude | other.exclude - self.include = includeAll - excludeAll - self.exclude = excludeAll - includeAll - - def _addCore(self): + def add_core(self): """Adds the core library.""" # Add externs and closure dependencies. - sourceBase = shakaBuildHelpers.getSourceBase() + source_base = shakaBuildHelpers.get_source_base() match = re.compile(r'.*\.js$') - self.include = self.include | set( - shakaBuildHelpers.getAllFiles( - os.path.join(sourceBase, 'externs'), match) + - shakaBuildHelpers.getAllFiles( - os.path.join(sourceBase, 'third_party', 'closure'), match)) + self.include |= set( + shakaBuildHelpers.get_all_files( + os.path.join(source_base, 'externs'), match) + + shakaBuildHelpers.get_all_files( + os.path.join(source_base, 'third_party', 'closure'), match)) # Check that there are no files in 'core' that are removed - coreBuild = Build() - coreBuild.parseBuild(['+@core'], os.getcwd()) - coreFiles = coreBuild.include - if len(self.exclude & coreFiles) > 0: + core_build = Build() + core_build.parse_build(['+@core'], os.getcwd()) + core_files = core_build.include + if self.exclude & core_files: print >> sys.stderr, 'Cannot exclude files from core' - self.include = self.include | coreFiles + self.include |= core_files - def parseBuild(self, lines, root): - """Parses a Build object from the given lines of commands. This will - recursively read and parse builds. + def parse_build(self, lines, root): + """Parses a Build object from the given lines of commands. - Arguments: - lines - An array of strings defining commands. - root - The full path to the base directory. + This will recursively read and parse builds. + + Args: + lines: An array of strings defining commands. + root: The full path to the base directory. Returns: True on success, False otherwise. @@ -172,11 +181,11 @@ class Build: if not line: continue - isNeg = False if line[0] == '+': + is_neg = False line = line[1:].strip() elif line[0] == '-': - isNeg = True + is_neg = True line = line[1:].strip() else: print >> sys.stderr, 'Operation (+/-) required' @@ -185,21 +194,21 @@ class Build: if line[0] == '@': line = line[1:].strip() - buildPath = self._getBuildFilePath(line, root) - if not buildPath: + build_path = self._get_build_file_path(line, root) + if not build_path: return False - lines = open(buildPath).readlines() - subRoot = os.path.dirname(buildPath) + lines = open(build_path).readlines() + sub_root = os.path.dirname(build_path) # If this is a build file, then recurse and combine the builds. - subBuild = Build() - if not subBuild.parseBuild(lines, subRoot): + sub_build = Build() + if not sub_build.parse_build(lines, sub_root): return False - if isNeg: - self._combine(subBuild._reverse()) + if is_neg: + self._combine(sub_build.reverse()) else: - self._combine(subBuild) + self._combine(sub_build) else: if not os.path.isabs(line): line = os.path.abspath(os.path.join(root, line)) @@ -207,7 +216,7 @@ class Build: print >> sys.stderr, 'Unable to find file ' + line return False - if isNeg: + if is_neg: self.include.discard(line) self.exclude.add(line) else: @@ -216,80 +225,81 @@ class Build: return True - def buildRaw(self, extra_opts): + def build_raw(self, extra_opts): """Builds the files in |self.include| using the given extra Closure options. - Arguments: - extra_opts - An array of extra options to give to Closure. + Args: + extra_opts: An array of extra options to give to Closure. Returns: True on success; False on failure. """ - jar = os.path.join(shakaBuildHelpers.getSourceBase(), - 'third_party', 'closure', 'compiler.jar') - jar = shakaBuildHelpers.cygwinSafePath(jar) - files = map(shakaBuildHelpers.cygwinSafePath, list(self.include)) + jar = os.path.join(shakaBuildHelpers.get_source_base(), + 'third_party', 'closure', 'compiler.jar') + jar = shakaBuildHelpers.cygwin_safe_path(jar) + files = [shakaBuildHelpers.cygwin_safe_path(f) for f in self.include] try: - cmdLine = ['java', '-jar', jar] + closure_opts + extra_opts + files - shakaBuildHelpers.printCmdLine(cmdLine) - subprocess.check_call(cmdLine) + cmd_line = ['java', '-jar', jar] + closure_opts + extra_opts + files + shakaBuildHelpers.print_cmd_line(cmd_line) + subprocess.check_call(cmd_line) return True except subprocess.CalledProcessError: print >> sys.stderr, 'Build failed' return False - def buildLibrary(self, name, rebuild): + def build_library(self, name, rebuild): """Builds Shaka Player using the files in |self.include|. - Arguments: - name - The name of the build. - rebuild - True to rebuild, False to ignore if no changes are detected. + Args: + name: The name of the build. + rebuild: True to rebuild, False to ignore if no changes are detected. Returns: True on success; False on failure. """ - self._addCore() + self.add_core() # In the build files, we use '/' in the paths, however Windows uses '\'. # Although Windows supports both, the source mapping will not work. So # use Linux-style paths for arguments. - sourceBase = shakaBuildHelpers.getSourceBase().replace('\\', '/') + source_base = shakaBuildHelpers.get_source_base().replace('\\', '/') - resultPrefix = shakaBuildHelpers.cygwinSafePath( - os.path.join(sourceBase, 'dist', 'shaka-player.' + name)) - resultFile = resultPrefix + '.js' - resultDebug = resultPrefix + '.debug.js' - resultMap = resultPrefix + '.debug.map' + result_prefix = shakaBuildHelpers.cygwin_safe_path( + os.path.join(source_base, 'dist', 'shaka-player.' + name)) + result_file = result_prefix + '.js' + result_debug = result_prefix + '.debug.js' + result_map = result_prefix + '.debug.map' # Detect changes to the library and only build if changes have been made. - if not rebuild and os.path.isfile(resultFile): - buildTime = os.path.getmtime(resultFile) - completeBuild = Build() - if completeBuild.parseBuild(['+@complete'], os.getcwd()): - completeBuild._addCore() + if not rebuild and os.path.isfile(result_file): + build_time = os.path.getmtime(result_file) + complete_build = Build() + if complete_build.parse_build(['+@complete'], os.getcwd()): + complete_build.add_core() # Get a list of files modified since the build file was. - editedFiles = filter(lambda x: os.path.getmtime(x) > buildTime, - completeBuild.include) - if len(editedFiles) == 0: + edited_files = [f for f in complete_build.include + if os.path.getmtime(f) > build_time] + if not edited_files: print 'No changes detected, not building. Use --force to override.' return True - opts = ['--create_source_map', resultMap, '--js_output_file', resultDebug, - '--source_map_location_mapping', sourceBase + '|..'] - if not self.buildRaw(opts): + opts = ['--create_source_map', result_map, '--js_output_file', result_debug, + '--source_map_location_mapping', source_base + '|..'] + if not self.build_raw(opts): return False - shutil.copyfile(resultDebug, resultFile) + shutil.copyfile(result_debug, result_file) # Add a special source-mapping comment so that Chrome and Firefox can map # line and character numbers from the compiled library back to the original # source locations. - with open(resultDebug, 'a') as f: + with open(result_debug, 'a') as f: f.write('//# sourceMappingURL=shaka-player.' + name + '.debug.map') return True + def usage(): print 'Usage:', sys.argv[0], """[options] [commands] @@ -300,6 +310,7 @@ Options: """ print __doc__ + def main(args): name = 'compiled' lines = [] @@ -307,7 +318,7 @@ def main(args): i = 0 while i < len(args): if args[i] == '--name': - i = i + 1 + i += 1 if i == len(args): print >> sys.stderr, '--name requires an argument' return 1 @@ -323,17 +334,17 @@ def main(args): return 1 else: lines.append(args[i]) - i = i + 1 + i += 1 - if len(lines) == 0: + if not lines: lines = ['+@complete'] print 'Compiling the library...' - customBuild = Build() - if not customBuild.parseBuild(lines, os.getcwd()): + custom_build = Build() + if not custom_build.parse_build(lines, os.getcwd()): return 1 - return 0 if customBuild.buildLibrary(name, rebuild) else 1 + return 0 if custom_build.build_library(name, rebuild) else 1 if __name__ == '__main__': - shakaBuildHelpers.runMain(main) + shakaBuildHelpers.run_main(main) diff --git a/build/check.py b/build/check.py index 1c6b06957..f7ea65208 100755 --- a/build/check.py +++ b/build/check.py @@ -1,6 +1,6 @@ #!/usr/bin/python # -# Copyright 2016 Google Inc. +# Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -19,26 +19,28 @@ This checks: * All files in lib/ appear when compiling +@complete * Runs a compiler pass over the test code to check for type errors - * Run the linter to check for style violations.""" + * Run the linter to check for style violations. +""" -import build import os import re -import shakaBuildHelpers import subprocess import sys -def getLintFiles(): - """Returns an array of absolute paths to all the files to run the linter - over. - """ +import build +import shakaBuildHelpers + + +def get_lint_files(): + """Returns the absolute paths to all the files to run the linter over.""" match = re.compile(r'.*\.js$') - base = shakaBuildHelpers.getSourceBase() + base = shakaBuildHelpers.get_source_base() def get(arg): - return shakaBuildHelpers.getAllFiles(os.path.join(base, arg), match) + return shakaBuildHelpers.get_all_files(os.path.join(base, arg), match) return get('test') + get('lib') + get('externs') + get('demo') -def checkLint(): + +def check_lint(): """Runs the linter over the library files.""" print 'Running Closure linter...' @@ -46,36 +48,43 @@ def checkLint(): 'static', 'summary', 'namespace', 'event', 'description', 'property', 'fires', 'listens', 'example', 'exportDoc']) args = ['--nobeep', '--custom_jsdoc_tags', jsdoc3_tags, '--strict'] - base = shakaBuildHelpers.getSourceBase() + base = shakaBuildHelpers.get_source_base() cmd = os.path.join(base, 'third_party', 'gjslint', 'gjslint') # Even though this is python, don't import and execute since gjslint expects # command-line arguments using argv. Have to explicitly execute python so # it works on Windows. - cmdLine = ['python', cmd] + args + getLintFiles() - shakaBuildHelpers.printCmdLine(cmdLine) - return (subprocess.call(cmdLine) == 0) + cmd_line = ['python', cmd] + args + get_lint_files() + shakaBuildHelpers.print_cmd_line(cmd_line) + return subprocess.call(cmd_line) == 0 -def checkHtmlLint(): + +def check_html_lint(): """Runs the HTML linter over the HTML files. + Skipped if htmlhint is not available. + + Returns: + True on success, False on failure. """ - htmlhint_path = shakaBuildHelpers.getNodeBinaryPath('htmlhint') + htmlhint_path = shakaBuildHelpers.get_node_binary_path('htmlhint') if not os.path.exists(htmlhint_path): return True print 'Running htmlhint...' - base = shakaBuildHelpers.getSourceBase() + base = shakaBuildHelpers.get_source_base() files = ['index.html', 'demo/index.html', 'support.html'] file_paths = [os.path.join(base, x) for x in files] - cmdLine = [htmlhint_path] + file_paths - shakaBuildHelpers.printCmdLine(cmdLine) - return (subprocess.call(cmdLine) == 0) + cmd_line = [htmlhint_path] + file_paths + shakaBuildHelpers.print_cmd_line(cmd_line) + return subprocess.call(cmd_line) == 0 -def checkComplete(): - """Checks whether the 'complete' build references every file. This is used - by the build script to ensure that every file is included in at least one - build type. + +def check_complete(): + """Checks whether the 'complete' build references every file. + + This is used by the build script to ensure that every file is included in at + least one build type. Returns: True on success, False on failure. @@ -86,24 +95,25 @@ def checkComplete(): # Normally we don't need to include @core, but because we look at the build # object directly, we need to include it here. When using main(), it will # call addCore which will ensure core is included. - if not complete.parseBuild(['+@complete', '+@core'], os.getcwd()): + if not complete.parse_build(['+@complete', '+@core'], os.getcwd()): print >> sys.stderr, 'Error parsing complete build' return False match = re.compile(r'.*\.js$') - base = shakaBuildHelpers.getSourceBase() - allFiles = shakaBuildHelpers.getAllFiles(os.path.join(base, 'lib'), match) - missingFiles = set(allFiles) - complete.include + base = shakaBuildHelpers.get_source_base() + all_files = shakaBuildHelpers.get_all_files(os.path.join(base, 'lib'), match) + missing_files = set(all_files) - complete.include - if len(missingFiles) > 0: + if missing_files: print >> sys.stderr, 'There are files missing from the complete build:' - for missing in missingFiles: + for missing in missing_files: # Convert to a path relative to source base. print >> sys.stderr, ' ' + os.path.relpath(missing, base) return False return True -def checkTests(): + +def check_tests(): """Runs an extra compile pass over the test code to check for type errors. Returns: @@ -112,23 +122,25 @@ def checkTests(): print 'Checking the tests for type errors...' match = re.compile(r'.*\.js$') - base = shakaBuildHelpers.getSourceBase() + base = shakaBuildHelpers.get_source_base() def get(*args): - return shakaBuildHelpers.getAllFiles(os.path.join(base, *args), match) + return shakaBuildHelpers.get_all_files(os.path.join(base, *args), match) files = (get('lib') + get('externs') + get('test') + get('demo') + - get('third_party', 'closure')) - testBuild = build.Build(set(files)) + get('third_party', 'closure')) + test_build = build.Build(set(files)) # Ignore missing goog.require since we assume the whole library is # already included. opts = ['--jscomp_off=missingRequire', '--checks-only', '-O', 'SIMPLE'] - return testBuild.buildRaw(opts) + return test_build.build_raw(opts) + def usage(): print 'Usage:', sys.argv[0] print print __doc__ + def main(args): for arg in args: if arg == '--help': @@ -139,17 +151,17 @@ def main(args): usage() return 1 - if not checkLint(): + if not check_lint(): return 1 - elif not checkHtmlLint(): + elif not check_html_lint(): return 1 - elif not checkComplete(): + elif not check_complete(): return 1 - elif not checkTests(): + elif not check_tests(): return 1 else: return 0 -if __name__ == '__main__': - shakaBuildHelpers.runMain(main) +if __name__ == '__main__': + shakaBuildHelpers.run_main(main) diff --git a/build/checkversion.py b/build/checkversion.py index 04594ba02..553cb3135 100755 --- a/build/checkversion.py +++ b/build/checkversion.py @@ -1,6 +1,6 @@ #!/usr/bin/python # -# Copyright 2016 Google Inc. +# Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,31 +14,37 @@ # See the License for the specific language governing permissions and # limitations under the License. +"""Checks that all the versions match.""" + import os import re -import shakaBuildHelpers import sys -def playerVersion(): +import shakaBuildHelpers + + +def player_version(): """Gets the version of the library from player.js.""" - path = os.path.join(shakaBuildHelpers.getSourceBase(), 'lib', 'player.js') + path = os.path.join(shakaBuildHelpers.get_source_base(), 'lib', 'player.js') with open(path, 'r') as f: match = re.search(r'goog\.define\(\'GIT_VERSION\', \'(.*)\'\)', f.read()) return match.group(1) if match else '' -def changelogVersion(): + +def changelog_version(): """Gets the version of the library from the CHANGELOG.""" - path = os.path.join(shakaBuildHelpers.getSourceBase(), 'CHANGELOG.md') + path = os.path.join(shakaBuildHelpers.get_source_base(), 'CHANGELOG.md') with open(path, 'r') as f: match = re.search(r'## (.*) \(', f.read()) return match.group(1) if match else '' -def checkVersion(_): + +def check_version(_): """Checks that all the versions in the library match.""" - changelog = changelogVersion() - player = playerVersion() - git = shakaBuildHelpers.gitVersion() - npm = shakaBuildHelpers.npmVersion() + changelog = changelog_version() + player = player_version() + git = shakaBuildHelpers.git_version() + npm = shakaBuildHelpers.npm_version() print 'git version:', git print 'npm version:', npm @@ -71,5 +77,6 @@ def checkVersion(_): return ret + if __name__ == '__main__': - shakaBuildHelpers.runMain(checkVersion) + shakaBuildHelpers.run_main(check_version) diff --git a/build/docs.py b/build/docs.py index b8f33128e..75e8816ad 100755 --- a/build/docs.py +++ b/build/docs.py @@ -1,6 +1,6 @@ #!/usr/bin/python # -# Copyright 2016 Google Inc. +# Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,33 +14,37 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Builds the documentation from the source code. This deletes the old -documentation first. +"""Builds the documentation from the source code. + +This deletes the old documentation first. """ import os -import shakaBuildHelpers import shutil import subprocess -import sys -def buildDocs(_): +import shakaBuildHelpers + + +def build_docs(_): + """Builds the source code documentation.""" print 'Building the docs...' - base = shakaBuildHelpers.getSourceBase() + base = shakaBuildHelpers.get_source_base() shutil.rmtree(os.path.join(base, 'docs', 'api'), ignore_errors=True) os.chdir(base) - if shakaBuildHelpers.isWindows() or shakaBuildHelpers.isCygwin(): + if shakaBuildHelpers.is_windows() or shakaBuildHelpers.is_cygwin(): # Windows has a different command name. The Unix version does not seem to # work on Cygwin, but the windows one does. jsdoc = os.path.join('third_party', 'jsdoc', 'jsdoc.cmd') else: jsdoc = os.path.join('third_party', 'jsdoc', 'jsdoc') - cmdLine = [jsdoc, '-c', 'docs/jsdoc.conf.json', '-R', 'docs/api-mainpage.md'] - shakaBuildHelpers.printCmdLine(cmdLine) - return subprocess.call(cmdLine) + cmd_line = [jsdoc, '-c', 'docs/jsdoc.conf.json', '-R', 'docs/api-mainpage.md'] + shakaBuildHelpers.print_cmd_line(cmd_line) + return subprocess.call(cmd_line) + if __name__ == '__main__': - shakaBuildHelpers.runMain(buildDocs) + shakaBuildHelpers.run_main(build_docs) diff --git a/build/gendeps.py b/build/gendeps.py index 83ac9a484..1f33e9305 100755 --- a/build/gendeps.py +++ b/build/gendeps.py @@ -1,6 +1,6 @@ #!/usr/bin/python # -# Copyright 2016 Google Inc. +# Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,40 +14,43 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Creates the Closure dependencies file required to run in uncompiled mode. -""" +"""Creates the Closure dependencies file required to run in uncompiled mode.""" import os -import shakaBuildHelpers import subprocess -import sys -depsArgs = [ - '--root_with_prefix=lib ../../../lib', - '--root_with_prefix=third_party/closure ../../../third_party/closure' +import shakaBuildHelpers + + +deps_args = [ + '--root_with_prefix=lib ../../../lib', + '--root_with_prefix=third_party/closure ../../../third_party/closure' ] -def genDeps(_): + +def gen_deps(_): + """Generates the uncompiled dependencies files.""" print 'Generating Closure dependencies...' # Make the dist/ folder, ignore errors. - base = shakaBuildHelpers.getSourceBase() + base = shakaBuildHelpers.get_source_base() try: os.mkdir(os.path.join(base, 'dist')) except OSError: pass os.chdir(base) - depsWriter = os.path.join('third_party', 'closure', 'deps', 'depswriter.py') + deps_writer = os.path.join('third_party', 'closure', 'deps', 'depswriter.py') try: - cmdLine = ['python', depsWriter] + depsArgs - shakaBuildHelpers.printCmdLine(cmdLine) - deps = subprocess.check_output(cmdLine) + cmd_line = ['python', deps_writer] + deps_args + shakaBuildHelpers.print_cmd_line(cmd_line) + deps = subprocess.check_output(cmd_line) with open(os.path.join(base, 'dist', 'deps.js'), 'w') as f: f.write(deps) return 0 except subprocess.CalledProcessError as e: return e.returncode + if __name__ == '__main__': - shakaBuildHelpers.runMain(genDeps) + shakaBuildHelpers.run_main(gen_deps) diff --git a/build/shakaBuildHelpers.py b/build/shakaBuildHelpers.py index c81132dd7..865c8a4ef 100644 --- a/build/shakaBuildHelpers.py +++ b/build/shakaBuildHelpers.py @@ -1,5 +1,6 @@ +#!/usr/bin/python # -# Copyright 2016 Google Inc. +# Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,8 +14,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Contains helper functions used in the build scripts. This uses two -environment variables to help with debugging the scripts: +"""Contains helper functions used in the build scripts. + +This uses two environment variables to help with debugging the scripts: PRINT_ARGUMENTS - If set, will print any arguments to subprocess. RAISE_INTERRUPT - Will raise keyboard interrupts rather than swallowing them. @@ -26,32 +28,48 @@ import re import subprocess import sys -def _parseVersion(s): - return tuple([int(i) for i in s.split('.')]) -def getSourceBase(): +def _parse_version(version): + """Converts the given string version to a tuple of numbers.""" + return tuple([int(i) for i in version.split('.')]) + + +def get_source_base(): """Returns the absolute path to the source code base.""" return os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) -def isLinux(): + +def is_linux(): """Determines if the system is Linux.""" return platform.uname()[0] == 'Linux' -def isDarwin(): + +def is_darwin(): """Determines if the system is a Mac.""" return platform.uname()[0] == 'Darwin' -def isWindows(): + +def is_windows(): """Determines if the system is native Windows (i.e. not Cygwin).""" return platform.uname()[0] == 'Windows' -def isCygwin(): + +def is_cygwin(): """Determines if the system is Cygwin (i.e. not native Windows).""" return 'CYGWIN' in platform.uname()[0] -def quoteArgument(arg): - """Quotes shell arguments so that printCmdLine output can be copied and pasted - into a shell.""" + +def quote_argument(arg): + """Wraps the given argument in quotes if needed. + + This is so print_cmd_line output can be copied and pasted into a shell. + + Args: + arg: The string to convert. + + Returns: + The quoted argument. + """ if '"' in arg: assert "'" not in arg return "'" + arg + "'" @@ -62,80 +80,92 @@ def quoteArgument(arg): return '"' + arg + '"' return arg -def printCmdLine(args): - """Prints the given command line if the environment variable PRINT_ARGUMENTS - is set.""" - if os.environ.get('PRINT_ARGUMENTS'): - print ' '.join([quoteArgument(x) for x in args]) -def cygwinSafePath(path): - """If the system is Cygwin, converts the given Cygwin path to a Windows path; - this does nothing if not Cygwin""" - if isCygwin(): - cmdLine = ['cygpath', '-w', path] - printCmdLine(cmdLine) - return subprocess.check_output(cmdLine).strip() +def print_cmd_line(args): + """Prints the given command line if needed. + + This uses the environment variable PRINT_ARGUMENTS. + + Args: + args: The arguments to print. + """ + if os.environ.get('PRINT_ARGUMENTS'): + print ' '.join([quote_argument(x) for x in args]) + + +def cygwin_safe_path(path): + """Converts the given path to a Cygwin path, if needed.""" + if is_cygwin(): + cmd_line = ['cygpath', '-w', path] + print_cmd_line(cmd_line) + return subprocess.check_output(cmd_line).strip() else: return path -def gitVersion(): + +def git_version(): """Gets the version of the library from git.""" try: # Check git tags for a version number, noting if the sources are dirty. - cmdLine = ['git', '-C', getSourceBase(), 'describe', '--tags', '--dirty'] - printCmdLine(cmdLine) - return subprocess.check_output(cmdLine).strip() + cmd_line = ['git', '-C', get_source_base(), 'describe', '--tags', '--dirty'] + print_cmd_line(cmd_line) + return subprocess.check_output(cmd_line).strip() except subprocess.CalledProcessError: raise RuntimeError('Unable to determine library version!') -def npmVersion(isDirty=False): + +def npm_version(is_dirty=False): """Gets the version of the library from NPM.""" try: - base = cygwinSafePath(getSourceBase()) - cmd = 'npm.cmd' if isWindows() else 'npm' - cmdLine = [cmd, '--prefix', base, 'ls', 'shaka-player'] - printCmdLine(cmdLine) - text = subprocess.check_output(cmdLine) + base = cygwin_safe_path(get_source_base()) + cmd = 'npm.cmd' if is_windows() else 'npm' + cmd_line = [cmd, '--prefix', base, 'ls', 'shaka-player'] + print_cmd_line(cmd_line) + text = subprocess.check_output(cmd_line) except subprocess.CalledProcessError as e: text = e.output match = re.search(r'shaka-player@(.*) ', text) if match: - return match.group(1) + ('-npm-dirty' if isDirty else '') + return match.group(1) + ('-npm-dirty' if is_dirty else '') raise RuntimeError('Unable to determine library version!') -def calculateVersion(): + +def calculate_version(): """Returns the version of the library.""" # Fall back to NPM's installed package version, and assume the sources # are dirty since the build scripts are being run at all after install. try: - return gitVersion() + return git_version() except RuntimeError: - # If there is an error in |gitVersion|, ignore it and try NPM. If there + # If there is an error in |git_version|, ignore it and try NPM. If there # is an error with NPM, propagate the error. - return npmVersion(isDirty=True) + return npm_version(is_dirty=True) -def getAllFiles(dirPath, exp): - """Returns an array of absolute paths to all the files at the given path that - match the given regex (if given). - Arguments: - dirPath - The string path to search. - exp - A regex to match, can be None. +def get_all_files(dir_path, exp=None): + """Returns an array of absolute paths to all the files at the given path. + + This optionally will filter the output using the given regex. + + Args: + dir_path: The string path to search. + exp: A regex to match, can be None. Returns: An array of absolute paths to all the files. """ ret = [] - for root, _, files in os.walk(dirPath): + for root, _, files in os.walk(dir_path): for f in files: if not exp or exp.match(f): ret.append(os.path.join(root, f)) ret.sort() return ret -def getNodeBinaryPath(name): + +def get_node_binary_path(name): # Try local modules first. - base = getSourceBase() + base = get_source_base() path = os.path.join(base, 'node_modules', '.bin', name) if os.path.isfile(path): return path @@ -143,28 +173,36 @@ def getNodeBinaryPath(name): # Not found locally, assume it can be found in os.environ['PATH']. return name -def updateNodeModules(): - base = cygwinSafePath(getSourceBase()) - cmd = 'npm.cmd' if isWindows() else 'npm' + +def update_node_modules(): + """Updates the node modules using 'npm'.""" + base = cygwin_safe_path(get_source_base()) + cmd = 'npm.cmd' if is_windows() else 'npm' # Check the version of npm. - cmdLine = [cmd, '-v'] - printCmdLine(cmdLine) - version = subprocess.check_output(cmdLine) - if _parseVersion(version) < _parseVersion('1.3.12'): + cmd_line = [cmd, '-v'] + print_cmd_line(cmd_line) + version = subprocess.check_output(cmd_line) + if _parse_version(version) < _parse_version('1.3.12'): print >> sys.stderr, 'npm version is too old, please upgrade. e.g.:' print >> sys.stderr, ' npm install -g npm' return False # Update the modules. - cmdLine = [cmd, '--prefix', base, 'update'] - printCmdLine(cmdLine) - subprocess.check_call(cmdLine) + cmd_line = [cmd, '--prefix', base, 'update'] + print_cmd_line(cmd_line) + subprocess.check_call(cmd_line) return True -def runMain(main): - """Executes the given function with the current command-line arguments, - calling exit with the return value. This ignores keyboard interrupts.""" + +def run_main(main): + """Executes the given function with the current command-line arguments. + + This calls exit with the return value. This ignores keyboard interrupts. + + Args: + main: The main function to call. + """ try: sys.exit(main(sys.argv[1:])) except KeyboardInterrupt: diff --git a/build/stats.py b/build/stats.py index 2df30b525..0a2344ae4 100755 --- a/build/stats.py +++ b/build/stats.py @@ -1,6 +1,6 @@ #!/usr/bin/python # -# Copyright 2016 Google Inc. +# Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,9 +14,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""A program for analyzing the Shaka compiled sources to find areas that can -be removed if not needed. This uses the source map -(i.e. shaka-player.compiled.debug.map) to find the compiled code +"""A program for analyzing the Shaka compiled sources. + +This can be used to find areas that can be removed if not needed. This uses +the source map (i.e. shaka-player.compiled.debug.map) to find the compiled code size, see: https://github.com/mattrobenolt/python-sourcemap http://www.html5rocks.com/en/tutorials/developertools/sourcemaps/ @@ -33,16 +34,35 @@ programs to display a visual layout of the dependencies. import json import math -import shakaBuildHelpers +import os import string import sys -import os -def fromVlqSigned(value): +import shakaBuildHelpers + + +# A Base64 VLQ digit can represent 5 bits, so it is Base32. +VLQ_BASE_SHIFT = 5 +VLQ_BASE = 1 << VLQ_BASE_SHIFT + +# A mask of bits for a VLQ digit (11111), 31 decimal +VLQ_BASE_MASK = VLQ_BASE - 1 + +# The continuation bit is the 6th bit +VLQ_CONTINUATION_BIT = VLQ_BASE + +# Don't use Base64 lib since it is not a real Base64 string; it simply +# decodes each character to a single Base64 number. +B64 = dict((c, i) for i, c in + enumerate('ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz' + '0123456789+/')) + + +def from_vlq_signed(value): """Converts a VLQ number to a normal signed number. - Arguments: - value - A number decoded from a VLQ string. + Args: + value: A number decoded from a VLQ string. Returns: an integer. @@ -51,45 +71,32 @@ def fromVlqSigned(value): value >>= 1 return -value if negative else value -class Segment: + +class Segment(object): """Defines an entry in the source map. Members: - dstColOffset - The offset of the destination column from the previous + dst_col_offset - The offset of the destination column from the previous segment. - nameOffset - If not None, the offset of the name index from the previous + name_offset - If not None, the offset of the name index from the previous segment. """ - def __init__(self, data): - self.dstColOffset = data[0] - self.nameOffset = data[4] if len(data) > 4 else None -def decodeSegment(segment): + def __init__(self, data): + self.dst_col_offset = data[0] + self.name_offset = data[4] if len(data) > 4 else None + + +def decode_segment(segment): """Decodes VLQ values from the given segment. - Arguments: - segment - A string containing the encoded segment text. + Args: + segment: A string containing the encoded segment text. Returns: the parsed Segment. """ - # A Base64 VLQ digit can represent 5 bits, so it is Base32. - VLQ_BASE_SHIFT = 5 - VLQ_BASE = 1 << VLQ_BASE_SHIFT - - # A mask of bits for a VLQ digit (11111), 31 decimal - VLQ_BASE_MASK = VLQ_BASE - 1 - - # The continuation bit is the 6th bit - VLQ_CONTINUATION_BIT = VLQ_BASE - - # Don't use Base64 lib since it is not a real Base64 string; it simply - # decodes each character to a single Base64 number. - B64 = dict((c, i) for i, c in - enumerate('ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz' - '0123456789+/')) - values = [] cur, shift = 0, 0 @@ -101,7 +108,7 @@ def decodeSegment(segment): shift += VLQ_BASE_SHIFT if not cont: - values.append(fromVlqSigned(cur)) + values.append(from_vlq_signed(cur)) cur, shift = 0, 0 # A valid VLQ string should not have dangling bits. @@ -109,66 +116,71 @@ def decodeSegment(segment): assert shift == 0 return Segment(values) -class Token: - """A Token represents one JavaScript symbol. For example, this can be a - variable or an equals sign. If this is a variable or the keyword 'function' - it will usually have a name which indicates what it originally was defined. - But there are also tokens such as ; and ( which appear as tokens in the - map but to not have explicit name (see isFunction). + +class Token(object): + """A Token represents one JavaScript symbol. + + For example, this can be a variable or an equals sign. If this is a variable + or the keyword 'function' it will usually have a name which indicates what it + originally was defined. But there are also tokens such as ; and ( which appear + as tokens in the map but to not have explicit name (see isFunction). Members: - dstLine - Line index in compiled code - dstCol - Column index in compiled code + dst_line - Line index in compiled code + dst_col - Column index in compiled code name - Name of the token; or None """ - def __init__(self, dstLine, dstCol, name=None): - self.dstLine = dstLine - self.dstCol = dstCol + + def __init__(self, dst_line, dst_col, name=None): + self.dst_line = dst_line + self.dst_col = dst_col self.name = name def __str__(self): return str(self.name) -def decodeMappings(lineData, names): + +def decode_mappings(line_data, names): """Decodes a mappings line of text. - Arguments: - lineData - A string containing the mapping line. - names - An array of strings containing the names of the objects. + Args: + line_data: A string containing the mapping line. + names: An array of strings containing the names of the objects. - Returns: - a list of Tokens + Returns: + a list of Tokens """ tokens = [] - lines = lineData.split(';') - nameId = 0 - for dstLine, line in enumerate(lines): - dstCol = 0 + lines = line_data.split(';') + name_id = 0 + for dst_line, line in enumerate(lines): + dst_col = 0 segments = line.split(',') for segment in segments: if not segment: continue - segment = decodeSegment(segment) - dstCol += segment.dstColOffset + segment = decode_segment(segment) + dst_col += segment.dst_col_offset - # segment.dstCol can be negative (more useful in names below); however + # segment.dst_col can be negative (more useful in names below); however # after applying a negative offset, the result must still be positive. - assert dstCol >= 0 + assert dst_col >= 0 name = None - if segment.nameOffset != None: - nameId += segment.nameOffset - assert nameId >= 0 - name = names[nameId] + if segment.name_offset is not None: + name_id += segment.name_offset + assert name_id >= 0 + name = names[name_id] - tokens.append(Token(dstLine, dstCol, name)) + tokens.append(Token(dst_line, dst_col, name)) return tokens -def isFunction(token, lines): + +def is_function(token, lines): """Determines if the given token is the start of a function. All function definitions are assumed to have a name field and the token in @@ -176,9 +188,9 @@ def isFunction(token, lines): defined on the previous semicolon and sometimes that semicolon appears on the previous line. - Arguments: - token - The Token to check. - lines - An array of compiled code lines. + Args: + token: The Token to check. + lines: An array of compiled code lines. Returns: whether the token is a function. @@ -191,59 +203,61 @@ def isFunction(token, lines): # Sometimes a function token starts with the previous ; # Also sometimes the token starts on the ; that is on the previous # line. - partialLine = lines[token.dstLine][token.dstCol:] - if partialLine == ';\n': - if len(lines) == token.dstLine + 1: + partial_line = lines[token.dst_line][token.dst_col:] + if partial_line == ';\n': + if len(lines) == token.dst_line + 1: return False else: - return lines[token.dstLine + 1].startswith('function') + return lines[token.dst_line + 1].startswith('function') else: - return (partialLine.startswith('function') or - partialLine.startswith(';function')) + return (partial_line.startswith('function') or + partial_line.startswith(';function')) -def readFunction(tokenIter, prev, prevIndex, lines, callback): - """Reads a function from the token stream. The function token should - already be consumed. - Arguments: - tokenIter - An iterator of the tokens. - prev - The token containing the function definition. - prevIndex - The index of the previous token. - lines - An array of compiled code lines. - callback - A callback type used to create the data. See traverseTokens. +def read_function(token_iter, prev, prev_index, lines, callback): + """Reads a function from the token stream. + + The function token should already be consumed. + + Args: + token_iter: An iterator of the tokens. + prev: The token containing the function definition. + prev_index: The index of the previous token. + lines: An array of compiled code lines. + callback: A callback type used to create the data. See traverse_tokens. Returns: an array of State objects in a format controlled by the callback (see - traverseTokens). + traverse_tokens). """ brackets = 0 read = False ret = [] - partialLine = lines[prev.dstLine][prev.dstCol:] - state = callback(prev, prevIndex) + partial_line = lines[prev.dst_line][prev.dst_col:] + state = callback(prev, prev_index) try: while not read or brackets > 0: - index, token = next(tokenIter) - partialLine = lines[token.dstLine][token.dstCol:] + index, token = next(token_iter) + partial_line = lines[token.dst_line][token.dst_col:] # Recursively read functions. Sometimes functions are defined nested. # This doesn't happen that often, and never for Shaka methods, so it does # not count it twice since the size of this method includes the nested # function. - if isFunction(token, lines): - ret += readFunction(tokenIter, token, index, lines, callback) + if is_function(token, lines): + ret += read_function(token_iter, token, index, lines, callback) else: state.add(token, index) - if partialLine.startswith('{}'): + if partial_line.startswith('{}'): read = True - elif partialLine[0] == '{': + elif partial_line[0] == '{': brackets += 1 read = True - elif partialLine[0] == '}': + elif partial_line[0] == '}': brackets -= 1 # When we run out of tokens, simply ignore it. A parent call will not see # this error; but it will continue and the next call to 'next' will fail @@ -258,24 +272,25 @@ def readFunction(tokenIter, prev, prevIndex, lines, callback): return ret -def traverseTokens(tokens, lines, callback): - """Traverses a list of tokens to identify functions. Then uses a callback - to perform some work on the functions. Each function seen gets a new State - object created from the given callback method; there is a single State for - global code which is given None in the constructor. Then, each token seen - is passed to the 'add' method of the State. This is used by the State to - either calculate sizes, print tokens, or detect dependencies. The 'build' - method is called at the end of the function to create a result object that - is returned as an array at the end. - Arguments: - tokens - An array of Tokens. - lines - An array of compiled code lines. - callback - A constructor that returns a state object. It takes a start - token or None if outside a function. It has two member - functions: - add - accepts the current token and the token's index. - build - returns an object to be added to the results. +def traverse_tokens(tokens, lines, callback): + """Traverses a list of tokens to identify functions. + + Then uses a callback to perform some work on the functions. Each function + seen gets a new State object created from the given callback method; there is + a single State for global code which is given None in the constructor. Then, + each token seen is passed to the 'add' method of the State. This is used by + the State to either calculate sizes, print tokens, or detect dependencies. + The 'build' method is called at the end of the function to create a result + object that is returned as an array at the end. + + Args: + tokens: An array of Tokens. + lines: An array of compiled code lines. + callback: A constructor that returns a state object. It takes a start + token or None if outside a function. It has two member functions + add - accepts the current token and the token's index. + build - returns an object to be added to the results. Returns: an array of State objects in a format controlled by the callback. @@ -285,13 +300,13 @@ def traverseTokens(tokens, lines, callback): state = callback(None, None) # Create a token iterator. This is used to read tokens from the array. We # cannot use a for loop because the iterator is passed to readFunction. - tokenIter = enumerate(tokens) + token_iter = enumerate(tokens) try: while True: - index, token = next(tokenIter) + index, token = next(token_iter) - if isFunction(token, lines): - ret += readFunction(tokenIter, token, index, lines, callback) + if is_function(token, lines): + ret += read_function(token_iter, token, index, lines, callback) else: state.add(token, index) except StopIteration: @@ -302,50 +317,63 @@ def traverseTokens(tokens, lines, callback): ret.append(temp) return ret -class FunctionSize: + +class FunctionSize(object): + """Contains information about a function's size.""" + def __init__(self, name, size): self.name = name self.size = size -def printTokens(tokens, lines, funcs): + +def print_tokens(tokens, lines, funcs): """Prints the given tokens. - Arguments: - tokens - An array of Tokens. - lines - An array of compiled code lines. - funcs - An array of FunctionSize. + Args: + tokens: An array of Tokens. + lines: An array of compiled code lines. + funcs: An array of FunctionSize. """ - class State: + + class State(object): + """Defines the current parser state.""" + def __init__(self, token, index): # The start of a function, or the global start. self.name = token.name if token else None if token: - self._printToken('>', token, index) + self._print_token('>', token, index) - def _printToken(self, prefix, token, index): - partialLine = lines[token.dstLine][token.dstCol:] + def _print_token(self, prefix, token, index): + partial_line = lines[token.dst_line][token.dst_col:] if len(tokens) > index + 1: next_ = tokens[index + 1] - if next_.dstLine == token.dstLine: - partialLine = lines[token.dstLine][token.dstCol:next_.dstCol] - tokenText = partialLine[:10].replace('\n', '').rjust(12) - print '%s %4d %4d %12s %s' % (prefix, token.dstLine, token.dstCol, - tokenText, token.name) + if next_.dst_line == token.dst_line: + partial_line = lines[token.dst_line][token.dst_col:next_.dst_col] + token_text = partial_line[:10].replace('\n', '').rjust(12) + print '%s %4d %4d %12s %s' % (prefix, token.dst_line, token.dst_col, + token_text, token.name) def add(self, token, index): - prefix = None - if not self.name: - prefix = '!' - elif lines[token.dstLine][token.dstCol:token.dstCol+2] == '{}': - prefix = ' ' - elif lines[token.dstLine][token.dstCol] == '{': - prefix = '+' - elif lines[token.dstLine][token.dstCol] == '}': - prefix = '-' - else: - prefix = ' ' + """Parses the given token. - self._printToken(prefix, token, index) + Args: + token: The token to add. + index: The index of the token in the original array. + """ + prefix = None + if not self.name: + prefix = '!' + elif lines[token.dst_line][token.dst_col:token.dst_col+2] == '{}': + prefix = ' ' + elif lines[token.dst_line][token.dst_col] == '{': + prefix = '+' + elif lines[token.dst_line][token.dst_col] == '}': + prefix = '-' + else: + prefix = ' ' + + self._print_token(prefix, token, index) def build(self): if not self.name: @@ -353,35 +381,50 @@ def printTokens(tokens, lines, funcs): # The end of a function. Print the size of this function. size = 0 - thisFunc = filter(lambda key:key.name == self.name, funcs) - if len(thisFunc) > 0: - size = thisFunc[0].size + this_func = [t for t in funcs if t.name == self.name] + if this_func: + size = this_func[0].size print 'X', self.name, size - traverseTokens(tokens, lines, State) + traverse_tokens(tokens, lines, State) + + +class FunctionDependencies(object): + """Contains information about a function's dependencies.""" -class FunctionDependencies: def __init__(self, name, deps): self.name = name self.deps = deps -def processDeps(tokens, lines, isClass): + +def process_deps(tokens, lines, is_class): """Processes the tokens into function or class dependencies. - Arguments: - tokens - An array of Tokens. - lines - An array of compiled code lines. - isClass - Whether to create a class graph instead of a function graph. + Args: + tokens: An array of Tokens. + lines: An array of compiled code lines. + is_class: Whether to create a class graph instead of a function graph. Returns: an array of FunctionDependencies. """ - class State: + + class State(object): + """Defines the current parser state.""" + def __init__(self, token, _): self.deps = [] - self.name, self.parts = self._createParts(token) + self.name, self.parts = self._create_parts(token) - def _createParts(self, token): + def _create_parts(self, token): + """Creates an array of name parts. + + Args: + token: The token to create the name from. + + Returns: + A tuple of the name and the array of name parts. + """ if not token or not token.name: return (None, None) parts = token.name.split('.') @@ -392,7 +435,7 @@ def processDeps(tokens, lines, isClass): del parts[-2] # Strip function names if class graph; also remove it from the name. - if isClass: + if is_class: if parts[-1][0] in string.lowercase: del parts[-1] name = '.'.join(parts) @@ -400,71 +443,85 @@ def processDeps(tokens, lines, isClass): return (name, parts) def add(self, token, _): + """Parses the given token. + + Args: + token: The token to parse. + """ # Ignore symbols outside a function. Only care about function # references and only those that reference our code. if not self.name or not token.name or not token.name.startswith('shaka.'): return - name, otherParts = self._createParts(token) + name, other_parts = self._create_parts(token) # Get the index of the first different namespace. - count = min(len(self.parts), len(otherParts)) + count = min(len(self.parts), len(other_parts)) i = 0 - while i < count and self.parts[i] == otherParts[i]: + while i < count and self.parts[i] == other_parts[i]: i += 1 # Ignore use of members of the same object: # OfflineVideoSource.configure and OfflineVideoSource.store - if (i == count - 1 or i == count) and len(self.parts) == len(otherParts): + if (i == count - 1 or i == count) and len(self.parts) == len(other_parts): return # Ignore use of the constructor of the same type: # OfflineVideoSource and OfflineVideoSource.store - if i == count and abs(len(self.parts) - len(otherParts)) == 1: + if i == count and abs(len(self.parts) - len(other_parts)) == 1: return # Add the dependency. - if not (name in self.deps): + if name not in self.deps: self.deps.append(name) def build(self): return FunctionDependencies(self.name, self.deps) if self.name else None - ret = traverseTokens(tokens, lines, State) - assert len(ret) > 0 - ret = sorted(ret, key=lambda key:key.name) + ret = traverse_tokens(tokens, lines, State) + assert ret + ret = sorted(ret, key=lambda key: key.name) # We need to collapse duplicates. i = 0 while i + 1 < len(ret): if ret[i].name == ret[i + 1].name: for dep in ret[i + 1].deps: - if not dep in ret[i].deps: + if dep not in ret[i].deps: ret[i].deps.append(dep) del ret[i + 1] else: - i += 1 + i += 1 return ret -def processSizes(tokens, lines): + +def process_sizes(tokens, lines): """Processes an array of tokens into function lengths. - Arguments: - tokens - An array of Tokens. - lines - An array of compiled code lines. + Args: + tokens: An array of Tokens. + lines: An array of compiled code lines. Returns: an array of FunctionSizes sorted on name. """ - class State: + + class State(object): + """Defines the current parser state.""" + def __init__(self, token, _): self.name = token.name if token else None self.size = 0 - self.start = token.dstCol if token else None - self.line = token.dstLine if token else None + self.start = token.dst_col if token else None + self.line = token.dst_line if token else None def add(self, token, _): + """Parses the given token. + + Args: + token: The token to parse. + """ # Ignore outside a function if not self.name: return @@ -472,26 +529,26 @@ def processSizes(tokens, lines): # If we skipped multiple lines, include the whole line. This will most # likely never happen since the compiled code usually has new lines on # function boundaries. - assert token.dstLine >= self.line - while token.dstLine != self.line: + assert token.dst_line >= self.line + while token.dst_line != self.line: self.size += len(lines[self.line]) - self.start self.line += 1 self.start = 0 # Keep increasing the size. We can't simply keep the start and measure # at the end since we are not given the end token in build(). - self.size += token.dstCol - self.start - self.start = token.dstCol + self.size += token.dst_col - self.start + self.start = token.dst_col def build(self): return FunctionSize(self.name, self.size) if self.name else None - ret = traverseTokens(tokens, lines, State) - assert len(ret) > 0 + ret = traverse_tokens(tokens, lines, State) + assert ret - ret = filter(lambda key:key.name and - (key.name.startswith('shaka.') or key.name.startswith('goog.')), ret) - ret = sorted(ret, key=lambda key:key.name) + ret = [k for k in ret if k.name and + (k.name.startswith('shaka.') or k.name.startswith('goog.'))] + ret = sorted(ret, key=lambda key: key.name) # We need to collapse duplicates. i = 0 @@ -500,19 +557,20 @@ def processSizes(tokens, lines): ret[i].size += ret[i + 1].size del ret[i + 1] else: - i += 1 + i += 1 return ret -def printTree(results, indent, callback, endCallback): + +def print_tree(results, indent, callback, end_callback): """Prints the results in an indented format. - Arguments: - results - An array of FunctionSizes sorted on name. - indent - A number to indent. - callback - A callback function to print the data. Accepts a title, an - indentation, and a sublist of the items in that group. - endCallback - A callback function called after a group; can be None. + Args: + results: An array of FunctionSizes sorted on name. + indent: A number to indent. + callback: A callback function to print the data. Accepts a title, an + indentation, and a sublist of the items in that group. + end_callback: A callback function called after a group; can be None. """ # This is used both when printing sizes and when printing dependencies in @@ -535,72 +593,76 @@ def printTree(results, indent, callback, endCallback): last = results[-1].name.split('.') prefix = 0 while (prefix < len(first) and prefix < len(last) - and first[prefix] == last[prefix]): + and first[prefix] == last[prefix]): prefix += 1 group = 0 - groupItems = first + group_items = first if prefix == len(first): # This happens when the group has a first element of a class name and the # remaining are member functions. Remove the first element from this # group. - groupItems = results[1].name.split('.') + group_items = results[1].name.split('.') group = 1 # Start with second element, and go one more so we make sure to process the # last group. for i in range(1, len(results) + 1): - items = (results[i].name.split('.') if i != len(results) else - [''] * (prefix + 1)) - if items[prefix] != groupItems[prefix]: - title = '.'.join(groupItems[:(prefix + 1)]) + if i == len(results): + items = [''] * (prefix + 1) + else: + items = results[i].name.split('.') + if items[prefix] != group_items[prefix]: + title = '.'.join(group_items[:(prefix + 1)]) callback(title, indent, results[group:i]) - printTree(results[group:i], indent + 1, callback, endCallback) + print_tree(results[group:i], indent + 1, callback, end_callback) # Set the start of the next group to the current element. group = i - groupItems = items + group_items = items - if endCallback: - endCallback(indent) + if end_callback: + end_callback(indent) -def printSizes(sizes): + +def print_sizes(sizes): """Prints the sizes in an indented format. - Arguments: - sizes - An array of FunctionSizes sorted on name. + Args: + sizes: An array of FunctionSizes sorted on name. """ # This callback is used to print the total sizes of each of the sub-groups. # Using the indent as padding allows to print a tree-like structure to # show how big each section is. - def callbackFactory(padding): + def callback_factory(padding): # Use a factory so we capture the padding. def callback(title, indent, results): if title: - size = sum(map(lambda key:key.size, results)) + size = sum([k.size for k in results]) print '%s %*d %s' % (indent * ' ', padding, size, title) return callback - total = sum(map(lambda key:key.size, sizes)) + total = sum([k.size for k in sizes]) padding = int(math.ceil(math.log10(total))) print '%*d %s' % (padding, total, 'TOTAL') - printTree(sizes, 0, callbackFactory(padding), None) + print_tree(sizes, 0, callback_factory(padding), None) -def printDeps(results, inDot): + +def print_deps(results, in_dot): """Prints the dependencies. Arguments: - results - A sorted array of FunctionDependencies. - inDot - Whether to print in DOT format. + results: A sorted array of FunctionDependencies. + in_dot: Whether to print in DOT format. """ - if not inDot: + if not in_dot: for func in results: name, deps = func.name, func.deps # Ignore items with no dependencies. - if len(deps) > 0: + if deps: print name for dep in deps: @@ -608,87 +670,95 @@ def printDeps(results, inDot): return - depMap = dict() + dep_map = dict() # Use the printTree to produce clusters for each namespace and type. This # will print boxes around each class and show dependencies between types. print 'digraph {' - def callbackFactory(depMap, temp): + def callback_factory(dep_map, temp): + """Creates a callback function.""" def callback(title, indent, results): if title: if len(results) > 1: print '\t' * indent, 'subgraph', 'cluster' + str(len(temp)), '{' temp.append(1) else: - print '\t' * indent, len(depMap), '[', \ - 'label="' + results[0].name + '"', ']', ';' - depMap[results[0].name] = len(depMap) + print('\t' * indent, len(dep_map), '[', + 'label="' + results[0].name + '"', ']', ';') + dep_map[results[0].name] = len(dep_map) return callback - def endCallback(indent): + + def end_callback(indent): if indent > 1: print '\t' * (indent - 1), '}' - printTree(results, 1, callbackFactory(depMap, []), endCallback) + + print_tree(results, 1, callback_factory(dep_map, []), end_callback) for func in results: name, deps = func.name, func.deps # Ignore items with no dependencies. - if len(deps) > 0: - if not name in depMap: - depMap[name] = len(depMap) - print '\t', depMap[name], '[', 'label="' + name + '"', ']', ';' + if deps: + if name not in dep_map: + dep_map[name] = len(dep_map) + print '\t', dep_map[name], '[', 'label="' + name + '"', ']', ';' for dep in deps: - if not dep in depMap: - depMap[dep] = len(depMap) - print '\t', depMap[dep], '[', 'label="' + dep + '"', ']', ';' + if dep not in dep_map: + dep_map[dep] = len(dep_map) + print '\t', dep_map[dep], '[', 'label="' + dep + '"', ']', ';' - print '\t', depMap[name], '->', depMap[dep], ';' + print '\t', dep_map[name], '->', dep_map[dep], ';' print '}' -class Options: + +class Options(object): + """Defines options to the script.""" + def __init__(self): - self.printDeps = False - self.printSizes = False - self.printTokens = False - self.inDot = False - self.isClass = False + self.print_deps = False + self.print_sizes = False + self.print_tokens = False + self.in_dot = False + self.is_class = False + def process(text, options): """Decodes a JSON string containing source map data. - Arguments: - text - A JSON string containing source map data. - options - An object containing the command-line options. + Args: + text: A JSON string containing source map data. + options: An object containing the command-line options. """ # The spec allows a map file to start with )]} to prevent javascript from # including it. if text.startswith(')]}\'\n') or text.startswith(')]}\n'): - _, text = text.split('\n', 1) + _, text = text.split('\n', 1) # Decode the JSON data and get the parts we need. data = json.loads(text) - # Paths are relative to the source code root. - base = shakaBuildHelpers.getSourceBase() - fileLines = open(os.path.join(base, data['file'])).readlines() + # Paths are relative to the output directory. + base = os.path.join(shakaBuildHelpers.get_source_base(), 'dist') + file_lines = open(os.path.join(base, data['file'])).readlines() names = data['names'] mappings = data['mappings'] - tokens = decodeMappings(mappings, names) - sizes = processSizes(tokens, fileLines) + tokens = decode_mappings(mappings, names) + sizes = process_sizes(tokens, file_lines) # Print out one of the results. - if options.printTokens: - printTokens(tokens, fileLines, sizes) - elif options.printSizes: - printSizes(sizes) - elif options.printDeps or options.isClass: - temp = processDeps(tokens, fileLines, options.isClass) - printDeps(temp, options.inDot) + if options.print_tokens: + print_tokens(tokens, file_lines, sizes) + elif options.print_sizes: + print_sizes(sizes) + elif options.print_deps or options.is_class: + temp = process_deps(tokens, file_lines, options.is_class) + print_deps(temp, options.in_dot) -def printHelp(): + +def print_help(): """Prints the help docs. """ @@ -726,70 +796,71 @@ DOT Format: """, sys.argv[0], """-c -d | fdp -Goverlap=prism | neato -n2 -Tsvg > out.svg""" + def main(args): options = Options() - doneArgs = False + done_args = False name = 'shaka-player.compiled.debug.map' # Process the command-line arguments. for arg in args: - if doneArgs or arg[0] != '-': + if done_args or arg[0] != '-': name = arg elif arg == '-f' or arg == '--function-deps': - options.printDeps = True + options.print_deps = True elif arg == '-t' or arg == '--all-tokens': - options.printTokens = True + options.print_tokens = True elif arg == '-s' or arg == '--function-sizes': - options.printSizes = True + options.print_sizes = True elif arg == '-c' or arg == '--class-deps': - options.isClass = True + options.is_class = True elif arg == '-d' or arg == '--dot-format': - options.inDot = True + options.in_dot = True elif arg == '--': - doneArgs = True + done_args = True elif arg == '-h' or arg == '--help': - printHelp() + print_help() return 0 else: print >> sys.stderr, 'Unrecognized argument:', arg - printHelp() + print_help() return 1 # Try to find the file if not os.path.isfile(name): # Get the source code base directory - base = shakaBuildHelpers.getSourceBase() + base = shakaBuildHelpers.get_source_base() # Supports the following searches: # * File name given, map in dist/ # * Type given, map in working directory # * Type given, map in dist/ - if os.path.isfile(os.path.join(base, 'dist' , name)): + if os.path.isfile(os.path.join(base, 'dist', name)): name = os.path.join(base, 'dist', name) elif os.path.isfile( - os.path.join('shaka-player.' + name + '.debug.map')): + os.path.join('shaka-player.' + name + '.debug.map')): name = os.path.join('shaka-player.' + name + '.debug.map') elif os.path.isfile( - os.path.join(base, 'dist', 'shaka-player.' + name + '.debug.map')): + os.path.join(base, 'dist', 'shaka-player.' + name + '.debug.map')): name = os.path.join(base, 'dist', 'shaka-player.' + name + '.debug.map') else: print >> sys.stderr, name, 'not found; build Shaka first.' return 1 # Verify arguments are correct. - if (options.printSizes + options.printDeps + options.printTokens + - options.isClass) != 1: + if (options.print_sizes + options.print_deps + options.print_tokens + + options.is_class) != 1: print >> sys.stderr, 'Must include exactly one output type.' - printHelp() + print_help() return 1 - elif options.inDot and not options.printDeps and not options.isClass: - print >> sys.stderr, '--dot-format only valid with --function-deps or \ ---class-deps.' + elif options.in_dot and not options.print_deps and not options.is_class: + line = '--dot-format only valid with --function-deps or --class-deps.' + print >> sys.stderr, line return 1 else: process(open(name).read(), options) return 0 -if __name__ == '__main__': - shakaBuildHelpers.runMain(main) +if __name__ == '__main__': + shakaBuildHelpers.run_main(main) diff --git a/build/test.py b/build/test.py index a04fd2648..9afb5bbac 100755 --- a/build/test.py +++ b/build/test.py @@ -1,6 +1,6 @@ #!/usr/bin/python # -# Copyright 2016 Google Inc. +# Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,23 +14,26 @@ # See the License for the specific language governing permissions and # limitations under the License. -import build -import gendeps -import os +"""Runs unit and integrations tests on the library.""" + import platform -import shakaBuildHelpers import subprocess import sys -def runTests(args): +import build +import gendeps +import shakaBuildHelpers + + +def run_tests(args): """Runs all the karma tests.""" # Update node modules if needed. - if not shakaBuildHelpers.updateNodeModules(): + if not shakaBuildHelpers.update_node_modules(): return 1 # Generate dependencies and compile library. # This is required for the tests. - if gendeps.genDeps([]) != 0: + if gendeps.gen_deps([]) != 0: return 1 build_args = [] @@ -45,50 +48,50 @@ def runTests(args): return 1 karma_command_name = 'karma' - if shakaBuildHelpers.isWindows(): + if shakaBuildHelpers.is_windows(): # Windows karma program has a different name karma_command_name = 'karma.cmd' - karma_path = shakaBuildHelpers.getNodeBinaryPath(karma_command_name) + karma_path = shakaBuildHelpers.get_node_binary_path(karma_command_name) cmd = [karma_path, 'start'] # Get the browsers supported on the local system. - browsers = _GetBrowsers() + browsers = _get_browsers() if not browsers: print >> sys.stderr, 'Unrecognized system "%s"' % platform.uname()[0] return 1 print 'Starting tests...' - if len(args) == 0: + if not args: # Run tests in all available browsers. print 'Running with platform default:', '--browsers', browsers - cmdLine = cmd + ['--browsers', browsers] - shakaBuildHelpers.printCmdLine(cmdLine) - return subprocess.call(cmdLine) + cmd_line = cmd + ['--browsers', browsers] + shakaBuildHelpers.print_cmd_line(cmd_line) + return subprocess.call(cmd_line) else: # Run with command-line arguments from the user. if '--browsers' not in args: print 'No --browsers specified.' print 'In this mode, browsers must be manually connected to karma.' - cmdLine = cmd + args - shakaBuildHelpers.printCmdLine(cmdLine) - return subprocess.call(cmdLine) + cmd_line = cmd + args + shakaBuildHelpers.print_cmd_line(cmd_line) + return subprocess.call(cmd_line) -def _GetBrowsers(): +def _get_browsers(): """Uses the platform name to configure which browsers will be tested.""" browsers = None - if shakaBuildHelpers.isLinux(): + if shakaBuildHelpers.is_linux(): # For MP4 support on Linux Firefox, install gstreamer1.0-libav. # Opera on Linux only supports MP4 for Ubuntu 15.04+, so it is not in the # default list of browsers for Linux at this time. browsers = 'Chrome,Firefox' - elif shakaBuildHelpers.isDarwin(): + elif shakaBuildHelpers.is_darwin(): browsers = 'Chrome,Firefox,Safari' - elif shakaBuildHelpers.isWindows() or shakaBuildHelpers.isCygwin(): + elif shakaBuildHelpers.is_windows() or shakaBuildHelpers.is_cygwin(): browsers = 'Chrome,Firefox,IE' return browsers if __name__ == '__main__': - shakaBuildHelpers.runMain(runTests) + shakaBuildHelpers.run_main(run_tests)