shaka-player/build/build.py

#!/usr/bin/env python3
#
# Copyright 2016 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Creates a build from the given commands.

A command is either an addition or a subtraction.  An addition is prefixed with
a +; a subtraction is when prefixed with a -.  After the character, there is a
name of a file or a @ sign and the name of a build file.

Build files are the files found in build/types.  These files are simply a
newline separated list of commands to execute.  So if the "+@complete" command
is given, it will open the complete file and run it (which may in turn open
other build files).  Subtracting a build file will reverse all actions applied
by the given file.  So "-@networking" will remove all the networking plugins,
and "-@ui" will remove the UI.

The core library is always included so does not have to be listed.  The default
is to use the name 'experimental'; if no commands are given, it will build the
complete build, including the UI.

Examples:
  # Equivalent to +@complete
  build.py

  build.py +@complete
  build.py +@complete -@networking
  build.py +@complete -@ui
  build.py --name custom +@manifests +@networking +../my_plugin.js
"""

import argparse
import hashlib
import json
import logging
import os
import re
import shutil
import sys

from contextlib import contextmanager

import compiler
import generateLocalizations
import shakaBuildHelpers


shaka_version = shakaBuildHelpers.calculate_version()

common_closure_opts = [
    '--jscomp_error=*',

    # Turn off complaints like:
    #   "Object is a reference type with no nullability modifier that is
    #   explicitly set to null."
    # and:
    #   "Property defineClass of type goog has been deprecated"
    # Even the Closure Library's base.js doesn't pass these checks yet as of
    # the 20200406 release.
    '--jscomp_off=lintChecks',
    '--jscomp_off=deprecated',
    # Turn off complaints like:
    #   "Built-in 'Reflect.setPrototypeOf' not supported in output version es3."
    '--jscomp_off=missingPolyfill',
    '--extra_annotation_name=listens',
    '--extra_annotation_name=exportDoc',
    '--extra_annotation_name=exportInterface',

    '--conformance_configs',
    ('%s/build/conformance.textproto' %
     shakaBuildHelpers.cygwin_safe_path(shakaBuildHelpers.get_source_base())),

    '--generate_exports',
]
common_closure_defines = [
    '-D', 'COMPILED=true',
    '-D', 'goog.ENABLE_DEBUG_LOADER=false',
]

debug_closure_opts = [
    '-O', 'SIMPLE',
]
debug_closure_defines = [
    '-D', 'goog.DEBUG=true',
    '-D', 'goog.asserts.ENABLE_ASSERTS=true',
    '-D', 'shaka.log.MAX_LOG_LEVEL=4',  # shaka.log.Level.DEBUG
    '-D', 'shaka.Player.version="%s-debug"' % shaka_version,
]

release_closure_opts = [
    '-O', 'ADVANCED',
]
release_closure_defines = [
    '-D', 'goog.DEBUG=false',
    '-D', 'goog.asserts.ENABLE_ASSERTS=false',
    '-D', 'shaka.log.MAX_LOG_LEVEL=0',
    '-D', 'shaka.Player.version="%s"' % shaka_version,
]


class Build(object):
  """Defines a build that has been parsed from a build file.

  This has exclude files even though it will not be used at the top-level.  This
  allows combining builds.  A file will only exist in at most one set.

  Members:
    include - A set of files to include.
    exclude - A set of files to remove.
  """

  def __init__(self, include=None, exclude=None):
    self.include = include or set()
    self.exclude = exclude or set()

  def _get_build_file_path(self, name, root):
    """Gets the full path to a build file, if it exists.

    Args:
      name: The string name to check.
      root: The full path to the base directory.

    Returns:
      The full path to the build file, or None if not found.
    """
    source_base = shakaBuildHelpers.get_source_base()
    local_path = os.path.join(root, name)
    build_path = os.path.join(source_base, 'build', 'types', name)
    if (os.path.isfile(local_path) and os.path.isfile(build_path)
        and local_path != build_path):
      logging.error('Build file "%s" is ambiguous', name)
      return None
    elif os.path.isfile(local_path):
      return local_path
    elif os.path.isfile(build_path):
      return build_path
    else:
      logging.error('Build file not found: %s', name)
      return None

  def _combine(self, other):
    include_all = self.include | other.include
    exclude_all = self.exclude | other.exclude
    self.include = include_all - exclude_all
    self.exclude = exclude_all - include_all

  def reverse(self):
    return Build(self.exclude, self.include)

  def add_closure(self):
    """Adds the closure library and externs."""
    # Add externs and closure dependencies.
    self.include |= set(
        [shakaBuildHelpers.get_closure_base_js_path()] +
        shakaBuildHelpers.get_all_js_files('externs'))

  def add_core(self):
    """Adds the core library."""
    # Check that there are no files in 'core' that are removed
    core_build = Build()
    core_build.parse_build(['+@core'], os.getcwd())
    core_files = core_build.include
    if self.exclude & core_files:
      logging.error('Cannot exclude files from core')
      return False
    self.include |= core_files
    return True

  def has_ui(self):
    """Returns True if the UI library is in the build."""
    for path in self.include:
      if 'ui' in path.split(os.path.sep):
        return True
    return False

  def has_cast(self):
    """Returns True if the cast system is in the build."""
    for path in self.include:
      if 'cast' in path.split(os.path.sep):
        return True
    return False

  def has_transmuxer_proxy(self):
    """Returns True if the transmuxer proxy is in the build."""
    for path in self.include:
      if path.endswith('transmuxer_proxy.js'):
        return True
    return False

  def generate_localizations(self, locales, force):
    localizations = compiler.GenerateLocalizations(locales)
    localizations.generate(force)
    self.include.add(os.path.abspath(localizations.output))

  def parse_build(self, lines, root):
    """Parses a Build object from the given lines of commands.

    This will recursively read and parse builds.

    Args:
      lines: An array of strings defining commands.
      root: The full path to the base directory.

    Returns:
      True on success, False otherwise.
    """
    for line in lines:
      # Strip comments
      try:
        line = line[:line.index('#')]
      except ValueError:
        pass

      # Strip whitespace and ignore empty lines.
      line = line.strip()
      if not line:
        continue

      if line[0] == '+':
        is_neg = False
        line = line[1:].strip()
      elif line[0] == '-':
        is_neg = True
        line = line[1:].strip()
      else:
        logging.error('Operation (+/-) required')
        return False

      if line[0] == '@':
        line = line[1:].strip()

        build_path = self._get_build_file_path(line, root)
        if not build_path:
          return False
        lines = shakaBuildHelpers.open_file(build_path).readlines()
        sub_root = os.path.dirname(build_path)

        # If this is a build file, then recurse and combine the builds.
        sub_build = Build()
        if not sub_build.parse_build(lines, sub_root):
          return False

        if is_neg:
          self._combine(sub_build.reverse())
        else:
          self._combine(sub_build)
      else:
        if not os.path.isabs(line):
          line = os.path.abspath(os.path.join(root, line))
        if not os.path.isfile(line):
          logging.error('Unable to find file: %s', line)
          return False

        if is_neg:
          self.include.discard(line)
          self.exclude.add(line)
        else:
          self.include.add(line)
          self.exclude.discard(line)

    return True

  def build_worker_bundle(self, langout, force, is_debug):
    """Compiles the transmuxer worker bundle.

    Returns True on success; False on failure.
    """
    worker_build = Build()
    if not worker_build.parse_build(['+@transmuxer-worker'], os.getcwd()):
      return False
    worker_build.add_closure()
    if not worker_build.add_core():
      return False

    build_name = 'shaka-player.transmuxer-worker'
    if is_debug:
      build_name += '.debug'
    closure = compiler.ClosureCompiler(worker_build.include, build_name)
    closure.add_wrapper = False
    closure.add_source_map = False

    closure_opts = common_closure_opts + common_closure_defines
    closure_opts += ['--language_out', langout]
    if is_debug:
      closure_opts += debug_closure_opts + debug_closure_defines
    else:
      closure_opts += release_closure_opts + release_closure_defines

    closure_opts += [
        '--dependency_mode=PRUNE',
        '--entry_point=goog:shaka.transmuxer.TransmuxerWorker',
        # Each transmuxer plugin registers itself at load time (side effect),
        # so we must list them as entry points too.
        '--entry_point=goog:shaka.transmuxer.AacTransmuxer',
        '--entry_point=goog:shaka.transmuxer.Ac3Transmuxer',
        '--entry_point=goog:shaka.transmuxer.Ec3Transmuxer',
        '--entry_point=goog:shaka.transmuxer.Mp3Transmuxer',
        '--entry_point=goog:shaka.transmuxer.MpegTsTransmuxer',
        '--entry_point=goog:shaka.transmuxer.TsTransmuxer',
    ]

    # Suppress type errors caused by dependency pruning; the main build
    # already validates all types.
    closure_opts += [
        '--jscomp_off=checkTypes',
        '--jscomp_off=unknownDefines',
    ]

    return closure.compile(closure_opts, force)

  def build_library(self, name, langout, locales, force, is_debug, skip_ts,
      build_worker):
    """Builds Shaka Player using the files in |self.include|.

    Args:
      name: The name of the build.
      langout: Closure Compiler output language.
      locales: A list of strings of locale identifiers.
      force: True to rebuild, False to ignore if no changes are detected.
      is_debug: True to compile for debugging, false for release.
      skip_ts: True to skip generation of TypeScript definitions.
      build_worker: True to build the standalone transmuxer worker if needed.

    Returns:
      True on success; False on failure.
    """
    self.add_closure()
    if not self.add_core():
      return False
    if self.has_ui():
      self.generate_localizations(locales, force)
      # So that the UI will correctly build if the cast is disabled, add the
      # dummy cast proxy.
      if not self.has_cast():
        self.include.add(os.path.abspath('conditional/dummy_cast_proxy.js'))

    if build_worker and self.has_transmuxer_proxy():
      logging.info('Compiling transmuxer worker bundle...')
      if not self.build_worker_bundle(langout, force, is_debug):
        return False

    if is_debug:
      name += '.debug'

    build_name = 'shaka-player.' + name
    closure = compiler.ClosureCompiler(self.include, build_name)

    closure_opts = common_closure_opts + common_closure_defines
    closure_opts += ['--language_out', langout]
    if is_debug:
      closure_opts += debug_closure_opts + debug_closure_defines
    else:
      closure_opts += release_closure_opts + release_closure_defines

    if not closure.compile(closure_opts, force):
      return False

    source_base = shakaBuildHelpers.get_source_base()

    # Don't pass local node modules to the extern generator.  But don't simply
    # exclude the string 'node_modules', either, since Shaka Player could be
    # rebuilt after installing it as a node module.
    node_modules_path = os.path.join(source_base, 'node_modules')
    local_include = set([f for f in self.include if node_modules_path not in f])
    extern_generator = compiler.ExternGenerator(local_include, build_name)

    if not extern_generator.generate(force):
      return False

    generated_externs = [extern_generator.output]
    shaka_externs = shakaBuildHelpers.get_all_js_files('externs/shaka')
    if self.has_ui():
      shaka_externs += shakaBuildHelpers.get_all_js_files('ui/externs')

    if not skip_ts:
      ts_def_generator = compiler.TsDefGenerator(
          generated_externs + shaka_externs, build_name)

      if not ts_def_generator.generate(force):
        return False

    # Copy this file to dist/ where support.html can use it
    shutil.copy(
        os.path.join(source_base, 'test', 'test', 'cast-boot.js'),
        os.path.join(source_base, 'dist', 'cast-boot.js'))

    return True


def main(args):
  parser = argparse.ArgumentParser(
      description=__doc__,
      formatter_class=argparse.RawDescriptionHelpFormatter)

  parser.add_argument(
      '--locales',
      type=str,
      nargs='+',
      default=generateLocalizations.DEFAULT_LOCALES,
      help='The list of locales to compile in (requires UI, default %(default)r)')

  parser.add_argument(
      '--force',
      '-f',
      help='Force building the library even if no files have changed.',
      action='store_true')

  parser.add_argument(
      '--mode',
      help='Specify which build mode to use.',
      choices=['debug', 'release'],
      default='release')

  parser.add_argument(
      '--debug',
      help='Same as using "--mode debug".',
      action='store_const',
      dest='mode',
      const='debug')

  parser.add_argument(
      '--name',
      help='Set the name of the build. Uses "ui" if not given.',
      type=str,
      default='experimental')

  parser.add_argument(
      '--langout',
      help='Set closure compiler output language. Defaults to ECMASCRIPT5.',
      type=str,
      default='ECMASCRIPT5')

  parser.add_argument(
    '--skip-ts',
    help='Skips generation of TypeScript definition files (.d.ts).',
    action='store_true')

  parser.add_argument(
    '--worker',
    help='Build only the standalone transmuxer worker script.',
    action='store_true')

  parser.add_argument(
    '--skip-worker',
    help='Do not build the standalone transmuxer worker alongside the library.',
    action='store_true')

  parsed_args, commands = parser.parse_known_args(args)

  # Make the dist/ folder, ignore errors.
  base = shakaBuildHelpers.get_source_base()
  dist_path = os.path.join(base, 'dist')
  try:
    os.mkdir(dist_path)
  except OSError:
    pass

  # Update node modules if needed.
  if not shakaBuildHelpers.update_node_modules():
    return 1

  # If no commands are given then use complete  by default.
  if len(commands) == 0:
    commands.append('+@complete')

  logging.info('Compiling the library (%s, %s)...',
               parsed_args.name, parsed_args.mode)

  custom_build = Build()

  if not custom_build.parse_build(commands, os.getcwd()):
    return 1

  # Global shared state file; parallel processes will serialize access via locks.
  hash_file_path = os.path.join(dist_path, 'build_state.json')

  def compute_build_hash(include, exclude, commands, mode, locales, skip_ts):
    state = {
      'include': sorted(include),
      'exclude': sorted(exclude),
      'commands': commands,
      'mode': mode,
      'locales': locales,
      'skip_ts': skip_ts,
      'langout': parsed_args.langout,
    }
    state_json = json.dumps(state, sort_keys=True)
    return hashlib.sha256(state_json.encode('utf-8')).hexdigest()

  current_hash = compute_build_hash(
    custom_build.include,
    custom_build.exclude,
    commands,
    parsed_args.mode,
    parsed_args.locales,
    parsed_args.skip_ts
  )

  build_key = f"{parsed_args.name}_{parsed_args.mode}"

  force = parsed_args.force

  # --- Cross‑platform file lock helpers (defined inline to keep changes local) ---
  if os.name == "posix":
    import fcntl

    @contextmanager
    def locked_file_rw(path, create=False):
      """Open file for read/write and acquire an exclusive advisory lock (POSIX)."""
      mode = 'r+' if os.path.isfile(path) or not create else 'w+'
      with open(path, mode) as f:
        # Acquire exclusive lock on the entire file.
        fcntl.flock(f.fileno(), fcntl.LOCK_EX)
        try:
          yield f
        finally:
          fcntl.flock(f.fileno(), fcntl.LOCK_UN)
  else:
    import msvcrt

    @contextmanager
    def locked_file_rw(path, create=False):
      """Open file for read/write and acquire an exclusive lock (Windows).
      This uses msvcrt.locking over the first byte as a process‑wide mutex.
      """
      mode = 'r+' if os.path.isfile(path) or not create else 'w+'
      with open(path, mode) as f:
        # Always lock the first byte as a mutex region.
        f.seek(0, os.SEEK_SET)
        msvcrt.locking(f.fileno(), msvcrt.LK_LOCK, 1)
        try:
          yield f
        finally:
          f.seek(0, os.SEEK_SET)
          msvcrt.locking(f.fileno(), msvcrt.LK_UNLCK, 1)

  # --- Safe read of the global state under lock (if the file exists) ---
  previous_state = {}
  if os.path.isfile(hash_file_path):
    try:
      with locked_file_rw(hash_file_path):
        # Re-opened with lock: parse safely
        with open(hash_file_path, 'r') as f:
          try:
            previous_state = json.load(f)
          except json.JSONDecodeError:
            logging.warning("Detected a corrupted build_state.json; continuing with empty state.")
            previous_state = {}
    except OSError as e:
      logging.warning("Could not lock/read build_state.json (%s); proceeding with empty state.", e)
      previous_state = {}

  previous_hash = previous_state.get(build_key)
  if previous_hash != current_hash:
    logging.info('Build parameters changed for "%s"; forcing rebuild.', parsed_args.name)
    force = True

  name = parsed_args.name
  langout = parsed_args.langout
  locales = parsed_args.locales
  is_debug = parsed_args.mode == 'debug'
  skip_ts = parsed_args.skip_ts

  if parsed_args.worker:
    if not custom_build.build_worker_bundle(langout, force, is_debug):
      return 1
  elif not custom_build.build_library(name, langout, locales, force, is_debug,
      skip_ts, not parsed_args.skip_worker):
    return 1

  # Persist (merge) the updated state under lock so we don't clobber parallel updates.
  # Use create=True so the file is created if it didn't exist.
  try:
    with locked_file_rw(hash_file_path, create=True) as f:
      # Read current on-disk state (may have been updated by other processes)
      try:
        f.seek(0, os.SEEK_SET)
        on_disk = json.load(f)
      except json.JSONDecodeError:
        on_disk = {}
      except Exception:
        on_disk = {}
      # Merge and write back atomically under the same lock
      on_disk[build_key] = current_hash
      f.seek(0, os.SEEK_SET)
      f.truncate()
      json.dump(on_disk, f, indent=2, sort_keys=True)
      f.flush()
      try:
        os.fsync(f.fileno())
      except Exception:
        # fsync may be unavailable on some platforms; ignore safely.
        pass
  except OSError as e:
    logging.warning("Failed to persist build_state.json safely: %s", e)

  return 0


if __name__ == '__main__':
  shakaBuildHelpers.run_main(main)