From 2eca4b2ef52edea98977edb068e48890ce895ae3 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Tue, 12 Apr 2022 10:10:41 -0700 Subject: [PATCH] ci: Generate static content for demo index at deploy time (#4119) Rather than runtime-querying of appengine versions within the appengine environment, we can instead generate the index at deployment time (from git tags) and just serve static content. This simplifies the system and avoids dependence on Google Cloud. This was less feasible before we adopted GitHub Actions, but is now relatively simple. The index will be regenerated when the index code is updated or when a new release is created. Closes #4074 --- .github/workflows/demo-version-index.yaml | 5 ++ app-engine/demo-version-index/.gitignore | 1 + app-engine/demo-version-index/README.md | 6 +- app-engine/demo-version-index/app.yaml | 22 ++++- .../{main.py => generate.py} | 80 ++++++++++--------- .../demo-version-index/requirements.txt | 1 - 6 files changed, 71 insertions(+), 44 deletions(-) create mode 100644 app-engine/demo-version-index/.gitignore rename app-engine/demo-version-index/{main.py => generate.py} (72%) mode change 100644 => 100755 diff --git a/.github/workflows/demo-version-index.yaml b/.github/workflows/demo-version-index.yaml index 300a6621c..05af9c496 100644 --- a/.github/workflows/demo-version-index.yaml +++ b/.github/workflows/demo-version-index.yaml @@ -9,6 +9,8 @@ on: paths: - .github/workflows/demo-version-index.yaml - app-engine/demo-version-index/** + release: + types: [published] jobs: appspot: @@ -16,6 +18,9 @@ jobs: steps: - uses: actions/checkout@v2 + - name: Generate static content + run: python3 app-engine/demo-version-index/generate.py + - uses: google-github-actions/auth@v0 with: credentials_json: '${{ secrets.APPENGINE_DEPLOY_KEY }}' diff --git a/app-engine/demo-version-index/.gitignore b/app-engine/demo-version-index/.gitignore new file mode 100644 index 000000000..980c85122 --- /dev/null +++ b/app-engine/demo-version-index/.gitignore @@ -0,0 +1 @@ +static/ diff --git a/app-engine/demo-version-index/README.md b/app-engine/demo-version-index/README.md index bc760d854..eddc84b10 100644 --- a/app-engine/demo-version-index/README.md +++ b/app-engine/demo-version-index/README.md @@ -5,10 +5,10 @@ releases and demos at https://index-dot-shaka-player-demo.appspot.com/ - app.yaml: App Engine config file. Defines the runtime (Python 3). - - main.py: A python service that queries available versions and generates the - index from a template. + - generate.py: A python script that generates the index's static content from + a template. - requirements.txt: Used by App Engine to install the necessary Python server - requirements (Flask, App Engine API). + requirements (Flask). - templates/index.html: A Jinja2 template used to generate the index HTML. diff --git a/app-engine/demo-version-index/app.yaml b/app-engine/demo-version-index/app.yaml index 017d92393..87808b49c 100644 --- a/app-engine/demo-version-index/app.yaml +++ b/app-engine/demo-version-index/app.yaml @@ -1,2 +1,22 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + runtime: python39 -default_expiration: 1s +default_expiration: 1h + +# Serve the static/ folder directly. +- url: /(.+) + secure: always + static_files: static/\1 + upload: static/.+ diff --git a/app-engine/demo-version-index/main.py b/app-engine/demo-version-index/generate.py old mode 100644 new mode 100755 similarity index 72% rename from app-engine/demo-version-index/main.py rename to app-engine/demo-version-index/generate.py index bc0122b87..14dd8b80b --- a/app-engine/demo-version-index/main.py +++ b/app-engine/demo-version-index/generate.py @@ -1,17 +1,16 @@ -# Shaka Player Version Index - Appspot Entrypoint +#!/usr/bin/env python3 + +# Shaka Player Version Index Generator # Copyright 2022 Google LLC # SPDX-License-Identifier: Apache-2.0 # Generate an index of Shaka Player versions on appspot. import collections -import google.appengine.api.modules import jinja2 import os -import random import re - -from flask import Flask, render_template +import subprocess DEMO_URL_TEMPLATE = 'https://{0}-dot-shaka-player-demo.appspot.com/' @@ -90,13 +89,20 @@ def version_to_metadata(v): 'ui_lib_defs': version_to_ui_lib_defs_url(v), } -def is_release_version(name): - return re.match(r'v\d+-\d+-\d+(?:-.+)?', name) - -def appengine_version_to_package_version(version): - # Replace the first two dashes with dots. More dashes indicate a prerelease - # version, as seen in "v2.0.0-beta3". - return version.replace('-', '.', 2) +def is_release_tag(name): + matches = re.match(r'v\d+\.\d+\.\d+(-.+)?', name) + # Doesn't match, not a version tag. + if not matches: + return False + # A "master" (old) or "main" (new) tag, indicating the state of the main + # branch at the time of a release. Not an actual version in itself, though. + if matches.group(1) == '-master' or matches.group(1) == '-main': + return False + # For historical reasons, these oldest few tags are not currently deployed to + # appengine. All other matching tags are, though. + if name in ['v1.2.0', 'v1.2.1', 'v1.2.2', 'v1.2.3']: + return False + return True def version_key(version): if version == 'nightly': @@ -111,32 +117,13 @@ def version_key(version): version_tuple = [int(x) for x in main_version.split('.')] return version_tuple + [suffix] -def get_appengine_versions(): - if os.getenv('GAE_ENV', '').startswith('standard'): - # NOTE: this doesn't return anything useful in a local dev server. - return google.appengine.api.modules.modules.get_versions() +def get_release_tags(): + output = subprocess.check_output(['git', 'tag'], text=True) + return list(filter(is_release_tag, output.split('\n'))) - # For a local dev server, fake it so we can test sorting. - fake_versions = [ - 'v1-6-0', 'v1-6-1', 'v1-6-2', 'v1-6-3', 'v1-6-4', 'v1-6-5', - 'v2-0-0-beta', 'v2-0-0-beta2', 'v2-0-0-beta3', 'v2-0-0', - 'v2-1-0', 'v2-1-1', 'v2-1-2', - 'v2-2-1', 'v2-2-2-beta', 'v2-2-2-beta2', 'v2-2-2', 'v2-2-9', 'v2-2-10', - ] - random.shuffle(fake_versions) # in-place shuffle - return fake_versions - - -app = Flask(__name__) - -@app.route('/') -def root(): - appengine_versions = get_appengine_versions() - # Filter for release versions only. - appengine_versions = filter(is_release_version, appengine_versions) - - # Now convert from appengine versions (v2-0-0) to package versions (v2.0.0). - versions = list(map(appengine_version_to_package_version, appengine_versions)) +def generate(): + # Get all release tags. + versions = get_release_tags() # Now sort, putting prerelease versions ahead of the corresponding release. versions.sort(key=version_key) @@ -158,5 +145,20 @@ def root(): version_metadata['nightly'] = version_to_metadata('nightly') version_metadata['nightly']['best'] = True - for i in version_metadata.values(): print(i) - return render_template('index.html', versions=version_metadata.values()) + # Debug: uncomment to see a list of version metadata objects. + #for i in version_metadata.values(): print(i) + + script_path = os.path.dirname(__file__) + template_path = os.path.join(script_path, 'templates', 'index.html') + output_path = os.path.join(script_path, 'static', 'index.html') + + os.makedirs(os.path.dirname(output_path), exist_ok=True) + + with open(template_path, 'r') as template_file: + with open(output_path, 'w') as output_file: + template = jinja2.Template(template_file.read()) + output = template.render(versions=version_metadata.values()) + output_file.write(output) + +if __name__ == '__main__': + generate() diff --git a/app-engine/demo-version-index/requirements.txt b/app-engine/demo-version-index/requirements.txt index f1baaf589..9675853bc 100644 --- a/app-engine/demo-version-index/requirements.txt +++ b/app-engine/demo-version-index/requirements.txt @@ -3,4 +3,3 @@ # SPDX-License-Identifier: Apache-2.0 Flask==2.0.3 -appengine-python-standard==0.3.1