shaka-player/build/generateExterns.js

#!/usr/bin/env node
/**
 * @license
 * Copyright 2016 Google Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

/**
 * @fileoverview
 *
 * A node script that generates externs automatically from the uncompiled
 * source of a Closure project.  Designed for Shaka Player, but may be usable
 * in other projects as well.  Does not depend on the Closure compiler itself.
 *
 * We were not able to get our externs generated by the Closure compiler.  There
 * were many issues with the Closure-generated externs, including the order of
 * the externs and the replacement of record types and enums with their
 * underlying types.
 *
 * This uses a node module called esprima to parse JavaScript, then explores the
 * abstract syntax tree from esprima.  It finds exported symbols and generates
 * an appropriate extern definition for it.
 *
 * The generated externs are then topologically sorted according to the
 * goog.provide and goog.require calls in the sources.  No sorting is done
 * within source files, and no sorting is done based on parameter types.
 * Circular deps between source files will not be resolved, and deps not
 * represented in goog.provide/goog.require will not be discovered.
 *
 * Arguments: --output <EXTERNS> <INPUT> [<INPUT> ...]
 */

// Load required modules.
const esprima = require('esprima');
const fs = require('fs');

// The annotations we will consider "exporting" a symbol.
const EXPORT_REGEX = /@(?:export|exportInterface|expose)\b/;

// Install ES6/ES7 polyfills for old versions of nodejs.
if (!global.Set) {
  require('es6-shim');
}
if (!Array.prototype.includes) {
  require('array-includes').shim();
}


/**
 * Topological sort of general objects using a DFS approach.
 * Will add a __mark field to each object as part of the sorting process.
 * @param {!Array.<T>} list
 * @param {function(T):!Array.<T>} getDeps
 * @return {!Array.<T>}
 * @template T
 * @see https://en.wikipedia.org/wiki/Topological_sorting#Depth-first_search
 */
function topologicalSort(list, getDeps) {
  var sorted = [];
  const NOT_VISITED = 0;
  const MID_VISIT = 1;
  const COMPLETELY_VISITED = 2;

  // Mark all objects as not visited.
  list.forEach(function(object) {
    object.__mark = NOT_VISITED;
  });

  // Visit each object.
  list.forEach(function(object) {
    visit(object);
  });

  // Return the sorted list.
  return sorted;

  /**
   * @param {T} object
   * @template T
   */
  function visit(object) {
    if (object.__mark == MID_VISIT) {
      console.assert(false, 'Dependency cycle detected!');
    } else if (object.__mark == NOT_VISITED) {
      object.__mark = MID_VISIT;

      // Visit all dependencies.
      getDeps(object).forEach(visit);

      object.__mark = COMPLETELY_VISITED;

      // Push this object onto the list.  All transitive dependencies have
      // already been added to the list.
      sorted.push(object);
    }
  }
}

/**
 * @param {ASTNode} node A node from the abstract syntax tree.
 * @return {boolean} true if this is a call node.
 */
function isCallNode(node) {
  // Example node: {
  //   type: 'ExpressionStatement',
  //   expression: { type: 'CallExpression', callee: {...}, arguments: [...] },
  // }
  return node.type == 'ExpressionStatement' &&
         node.expression.type == 'CallExpression';
}


/**
 * Pretty-print a node via console.log.  Useful for debugging and development
 * to see what the AST looks like.
 * @param {ASTNode} node A node from the abstract syntax tree.
 */
function dumpNode(node) {
  console.log(JSON.stringify(node, null, '  '));
}


/**
 * @param {ASTNode} node A node from the abstract syntax tree.
 * @return {boolean} true if this is a call to goog.provide.
 */
function isProvideNode(node) {
  return isCallNode(node) &&
         getIdentifierString(node.expression.callee) == 'goog.provide';
}


/**
 * @param {ASTNode} node A node from the abstract syntax tree.
 * @return {boolean} true if this is a call to goog.require.
 */
function isRequireNode(node) {
  return isCallNode(node) &&
         getIdentifierString(node.expression.callee) == 'goog.require';
}


/**
 * @param {ASTNode} node A node from the abstract syntax tree.
 * @return {boolean} true if this is an exported symbol or property.
 */
function isExportNode(node) {
  var doc = getLeadingBlockComment(node);
  return doc && EXPORT_REGEX.test(doc);
}


/**
 * @param {ASTNode} node A node from the abstract syntax tree.
 * @return {string} A reconstructed leading comment block for the node.
 *   If there are multiple comments before this node, we will take the most
 *   recent block comment, as that is the one that would contain any applicable
 *   jsdoc/closure annotations for this symbol.
 */
function getLeadingBlockComment(node) {
  // Example code: /** @summary blah */ /** @export */ foo.bar = ...;
  // Example node: {
  //   type: 'ExpressionStatement',
  //   expression: { ... },
  //   leadingComments: [
  //     { type: 'Block', value: '* @summary blah ' },
  //     { type: 'Block', value: '* @export ' },
  //   ],
  // }
  if (!node.leadingComments || !node.leadingComments.length) return null;

  // Ignore non-block comments, since those are not jsdoc/Closure comments.
  var blockComments = node.leadingComments.filter(function(comment) {
    return comment.type == 'Block';
  });
  if (!blockComments.length) return null;

  // In case there are multiple (for example, a file-level comment that also
  // preceeds the node), take the most recent one, which is closest to the node.
  var mostRecentComment = blockComments[blockComments.length - 1];

  // Reconstruct the original block comment by adding back /* and */.
  return '/*' + mostRecentComment.value + '*/';
}


/**
 * @param {number} idx An argument index from the call node.
 * @param {ASTNode} node A node from the abstract syntax tree.
 * @return {string} The argument value as a string.
 */
function getArgumentFromCallNode(idx, node) {
  // Example node: {
  //   type: 'ExpressionStatement',
  //   expression: { type: 'CallExpression', callee: {...}, arguments: [...] },
  // }
  console.assert(isCallNode(node));
  return node.expression.arguments[idx].value;
}


/**
 * @param {ASTNode} node An identifier or member node from the abstract syntax
 *   tree.
 * @return {string} The identifier as a string.
 */
function getIdentifierString(node) {
  if (node.type == 'Identifier') {
    // Example code: foo
    // Example node: { type: 'Identifier', name: 'foo' }
    return node.name;
  }

  console.assert(node.type == 'MemberExpression');
  // Example code: foo.bar.baz
  // Example node: {
  //   type: 'MemberExpression',
  //   object: {
  //     type: 'MemberExpression',
  //     object: { type: 'Identifier', name: 'foo' },
  //     property: { type: 'Identifier', name: 'bar' },
  //   },
  //   property: { type: 'Identifier', name: 'baz' },
  // }
  return getIdentifierString(node.object) + '.' + getIdentifierString(node.property);
}


/**
 * @param {ASTNode} node A function definition node from the abstract syntax
 *   tree.
 * @return {!Array.<string>} a list of the parameter names.
 */
function getFunctionParameters(node) {
  console.assert(node.type == 'FunctionExpression');
  // Example code: function(x, y) {...}
  // Example node: {
  //   params: [
  //     { type: 'Identifier', name: 'x' },
  //     { type: 'Identifier', name: 'y' },
  //   ],
  //   body: {...},
  // }
  return node.params.map(function(param) {
    console.assert(param.type == 'Identifier');
    return param.name;
  });
}


/**
 * @param {!Set.<string>} names A set of the names of exported nodes.
 * @param {ASTNode} node An exported node from the abstract syntax tree.
 * @return {string} An extern string for this node.
 */
function createExternFromExportNode(names, node) {
  console.assert(node.type == 'ExpressionStatement',
                 'Unknown node type: ' + node.type);

  var comment = getLeadingBlockComment(node);

  // Remove @export annotations.
  comment = comment.replace(EXPORT_REGEX, '')

  // Split into lines, remove empty comment lines, then recombine.
  comment = comment.split('\n')
      .filter(function(line) { return !/^ *\*? *$/.test(line); })
      .join('\n');

  var name;
  var assignment;

  switch (node.expression.type) {
    case 'AssignmentExpression':
      // Example code: /** @export */ foo.bar = function(...) { ... };
      // Example node.expression: {
      //   operator: '=',
      //   left: {
      //     type: 'MemberExpression',
      //     object: { type: 'Identifier', name: 'foo' },
      //     property: { type: 'Identifier', name: 'bar' },
      //   }, right: {
      //     type: 'FunctionExpression', params: [ ... ], body: {...}
      //   }
      // }
      name = getIdentifierString(node.expression.left);
      assignment = createExternAssignment(node.expression.right);
      break;

    case 'MemberExpression':
      // Example code: /** @export */ foo.bar;
      // Example node.expression: {
      //   object: { type: 'Identifier', name: 'foo' },
      //   property: { type: 'Identifier', name: 'bar' },
      // }
      name = getIdentifierString(node.expression);
      assignment = '';
      break;

    default:
      console.assert(
          false, 'Unexpected expression type: ' + node.expression.type);
  }

  // Keep track of the names we've externed.
  names.add(name);
  // Generate the actual extern string.
  return comment + '\n' + name + assignment + ';\n';
}


/**
 * @param {ASTNode} node An assignment node from the abstract syntax tree.
 * @return {string} The assignment part of the extern string for this node.
 */
function createExternAssignment(node) {
  switch (node.type) {
    case 'FunctionExpression':
      // Example code: foo.square = function(x) { return x * x; };
      // Example node: { params: [ { type: 'Identifier', name: 'x' } ] }
      var params = getFunctionParameters(node);
      return ' = function(' + params.join(', ') + ') {}';

    case 'ObjectExpression':
      // Example code: foo.Bar = { 'ABC': 1, DEF: 2 };
      // Example node: {
      //   properties: [ {
      //     kind: 'init',
      //     key: { type: 'Literal', value: 'ABC' }
      //     value: { type: 'Literal', value: 1 }
      //   }, {
      //     kind: 'init',
      //     key: { type: 'Identifier', name: 'DEF' }
      //     value: { type: 'Literal', value: 2 }
      //   } ]
      // }
      var propertyStrings = node.properties.map(function(prop) {
        console.assert(prop.kind == 'init');
        console.assert(prop.key.type == 'Literal' || prop.key.type == 'Identifier');
        // Literal indicates a quoted name in the source, while Identifier is
        // an unquoted name.  In the case of Literal, key.raw gets us the
        // unquoted name, we end up with an unquoted name in both cases.
        var name = prop.key.type == 'Literal' ? prop.key.raw : prop.key.name;
        console.assert(prop.value.type == 'Literal');
        return '  ' + name + ': ' + prop.value.raw;
      });
      return ' = {\n' + propertyStrings.join(',\n') + '\n}';

    case 'Identifier':
      // Example code: /** @const {string} @export */ foo.version = VERSION;
      // Example extern: /** @const {string} */ foo.version;
      return '';

    case 'Literal':
      // Example code: /** @const {string} @export */ foo.version = 'v1.0.0';
      // Example extern: /** @const {string} */ foo.version;
      return '';

    default:
      console.assert(false, 'unknown export type: ' + node.type);
  }
}


/**
 * @param {!Set.<string>} names A set of the names of exported nodes.
 * @param {string} inputPath
 * @return {{
 *   path: string,
 *   provides: !Array.<string>,
 *   requires: !Array.<string>,
 *   externs: string,
 * }}
 */
function generateExterns(names, inputPath) {
  // Load and parse the code, with comments attached to the nodes.
  var code = fs.readFileSync(inputPath, 'utf-8');
  var program = esprima.parse(code, {attachComment: true});
  console.assert(program.type == 'Program');

  var body = program.body;
  var provides = program.body
      .filter(isProvideNode).map(getArgumentFromCallNode.bind(null, 0));
  var requires = program.body
      .filter(isRequireNode).map(getArgumentFromCallNode.bind(null, 0));

  var rawExterns = program.body
      .filter(isExportNode).map(createExternFromExportNode.bind(null, names));
  var externs = rawExterns.join('');

  return {
    path: inputPath,
    provides: provides,
    requires: requires,
    externs: externs,
  };
}


/**
 * Generate externs from exported code.
 * Arguments: --output <EXTERNS> <INPUT> [<INPUT> ...]
 *
 * @param {!Array.<string>} args The args to this script, not counting node and
 *   the script name itself.
 */
function main(args) {
  var inputPaths = [];
  var outputPath;

  for (var i = 0; i < args.length; ++i) {
    if (args[i] == '--output') {
      outputPath = args[i + 1];
      ++i;
    } else {
      inputPaths.push(args[i]);
    }
  }
  console.assert(outputPath,
                 'You must specify output file with --output <EXTERNS>');
  console.assert(inputPaths.length,
                 'You must specify at least one input file.');

  // Generate externs for all input paths.
  var names = new Set();
  var results = inputPaths.map(generateExterns.bind(null, names));

  // Sort them in dependency order.
  var sorted = topologicalSort(results, /* getDeps */ function(object) {
    return object.requires.map(function(id) {
      var dep = results.find(function(x) { return x.provides.includes(id); });
      console.assert(dep, 'Cannot find dependency: ' + id);
      return dep;
    });
  });

  // Generate namespaces for all externs.  For example, if we extern foo.bar.baz,
  // foo and foo.bar will both need to be declared first.
  var namespaces = new Set();
  var namespaceDeclarations = [];
  names.forEach(function(name) {
    // Add the full name "foo.bar.baz" and its prototype ahead of time.  We should
    // never generate these as namespaces.
    namespaces.add(name);
    namespaces.add(name + '.prototype');

    // For name "foo.bar.baz", iterate over partialName "foo" and "foo.bar".
    var pieces = name.split('.');
    for (var i = 1; i < pieces.length; ++i) {
      var partialName = pieces.slice(0, i).join('.');
      if (!namespaces.has(partialName)) {
        var declaration = '/** @const */\n';
        if (i == 1) declaration += 'var ';
        declaration += partialName + ' = {};\n';
        namespaceDeclarations.push(declaration);
        namespaces.add(partialName);
      }
    }
  });

  // Get externs.
  var externs = sorted.map(function(x) { return x.externs; }).join('');

  // Output generated externs, with an appropriate header.
  fs.writeFileSync(outputPath,
      '/**\n' +
      ' * @fileoverview Generated externs.  DO NOT EDIT!\n' +
      ' * @externs\n' +
      ' */\n\n' +
      namespaceDeclarations.join('') + '\n' + externs);
}


// Skip argv[0], which is the node binary, and argv[1], which is the script.
main(process.argv.slice(2));