Merge pull request #1335 from openziti/add-k8s-shortcut-to-getter-proxy

enable file shortcuts in the GitHub raw getter proxy

.github/workflows/release-quickstart.yml

@@ -126,7 +126,7 @@ jobs:
       - name: Configure Python
         shell: bash
         run: |
-          pip install --upgrade boto3 jinja2 requests
+          pip install --upgrade boto3 jinja2 requests pyyaml
           python --version
         
       - name: Deploy the CloudFront Function for get.openziti.io

dist/cloudfront/get.openziti.io/cloudfront-function-get-openziti-io.js.j2

@@ -4,11 +4,11 @@ function handler(event) {
     var uri = request.uri;
 
     switch (true) {
-      {% for name, data in routes.items() -%}
-      // GitHub raw shortcut route: /{{name}}/
-      case /{{data.re}}/i.test(uri):
-        var re = /{{data.re}}/;
-        request.uri = uri.replace(re, '{{data.uri}}');
+      {% for route in routes -%}
+      // GitHub raw routecut: {{route['get']}}
+      case /{{route['re']}}/i.test(uri):
+        var re = /{{route['re']}}/;
+        request.uri = uri.replace(re, '{{route['raw']}}');
         break;
       {%- endfor %}
     }

dist/cloudfront/get.openziti.io/deploy-cloudfront-function.py

@@ -3,9 +3,8 @@
 import json
 import logging
 import os
-import random
-import string
 import requests
+import yaml
 
 import jinja2
 
@@ -24,37 +23,45 @@ CF_HOST = 'get.openziti.io'
 CF_FUNCTION_NAME = 'github-raw-viewer-request-router'
 CF_FUNCTION_TEMPLATE = 'dist/cloudfront/get.openziti.io/cloudfront-function-get-openziti-io.js.j2'
 GITHUB_SHA = os.environ['GITHUB_SHA']
-CF_ROUTES = {
-    'qs': {
-        're': r'(^\/qs\/)(.*)',
-        'uri': f'/openziti/ziti/{GITHUB_SHA}/quickstart/$2',
-        'file': 'kubernetes/miniziti.bash',
-    },
-    'quick': {
-        're': r'(^\/quick\/)(.*)',
-        'uri': f'/openziti/ziti/{GITHUB_SHA}/quickstart/docker/image/$2',
-        'file': 'ziti-cli-functions.sh',
-    },
-    'dock': {
-        're': r'(^\/dock\/)(.*)',
-        'uri': f'/openziti/ziti/{GITHUB_SHA}/quickstart/docker/$2',
-        'file': 'docker-compose.yml',
-    },
-    'spec': {
-        're': r'(^\/spec\/)(.*)',
-        'uri': '/openziti/edge-api/main/$2',
-        'file': 'management.yml',
-    },
-    'pack': {
-        're': r'(^\/(tun|pack)\/)(.*)',
-        'uri': '/openziti/ziti-tunnel-sdk-c/main/$3',
-        'file': 'package-repos.gpg',
-    },
-}
-cf_function_template_bytes = open(CF_FUNCTION_TEMPLATE, 'rb').read()
+CF_ROUTES_FILE = 'dist/cloudfront/get.openziti.io/routes.yml'
+routes = yaml.safe_load(open(CF_ROUTES_FILE, 'r').read())
 jinja2_env = jinja2.Environment()
+
+unique_routes = dict()
+# validate the shape of routes and compute the regex and backreference for each
+for route in routes:
+    if bool(unique_routes.get(route['get'])):
+        raise ValueError(f"route 'get' path '{route['get']}' is not unique")
+    else:
+        unique_routes[route['get']] = True
+    # ensure the generated regex is valid for matching HTTP request paths from the viewer
+    if not route['get'].startswith('/'):
+        raise ValueError(f"route 'get' path '{route['get']}' must start with '/'")
+    # ensure the destination uri so we can append the backreference to compose a valid HTTP request path to the origin
+    elif not route['raw'].endswith('/'):
+        raise ValueError(f"GitHub raw path '{route['raw']}' must end with '/'")
+    # is a directory shortcut, so it must have a file to test the route
+    elif route['get'].endswith('/'):
+        if not bool(route.get('file')):
+            raise ValueError(f"route 'get' path '{route['get']}' ends with '/', but no test file is specified")
+        route['re'] = f"^\\{route['get'][0:-1]}\\/(.*)"
+    # is a file shortcut, so the file is the route
+    else:
+        if bool(route.get('file')):
+            raise ValueError(f"route 'get' '{route['get']}' does not end with '/', so no file may be specified because the 'get' path is the test file")
+        route['re'] = f"^\\/({route['get'][1:]})(\\?.*)?$"
+        route['file'] = route['get'][1:]
+    # always append backreference to the destination uri to compose a valid HTTP request path to the origin ending with
+    # the matching part of the request regex
+    route['raw'] = f"{route['raw']}$1"
+    # render the raw path as a jinja2 template to allow for dynamic values
+    route['raw'] = jinja2_env.from_string(route['raw']).render(GITHUB_SHA=GITHUB_SHA)
+
+cf_function_template_bytes = open(CF_FUNCTION_TEMPLATE, 'rb').read()
 cf_function_template = jinja2_env.from_string(cf_function_template_bytes.decode('utf-8'))
-cf_function_rendered = cf_function_template.render(routes=CF_ROUTES)
+cf_function_rendered = cf_function_template.render(routes=routes)
+
+# TODO: revert or comment after local testing
 # tmp = open('/tmp/rendered.js', 'w')
 # tmp.write(cf_function_rendered)
 # tmp.close()
@@ -81,11 +88,13 @@ logger.debug(f"got candidate etag: {candidate_function_etag}")
 
 
 # verify a random /path is handled correctly for each route by the candidate function
-def test_route(client: object, etag: str, requested: str, expected_path_prefix: str, expected_file: str = None):
-    if expected_file:
-        path_file = expected_file
+def test_route(client: object, etag: str, route: dict):
+    if route['get'].endswith('/'):
+        # the shortcut is a directory, so append the test file to the path
+        request_uri = f"{route['get']}{route['file']}"
     else:
-        path_file = f"{''.join(random.choices(string.ascii_uppercase+string.digits, k=4))}.html"
+        # the get shortcut is a file
+        request_uri = route['get']
 
     test_obj = {
         "version": "1.0",
@@ -97,7 +106,7 @@ def test_route(client: object, etag: str, requested: str, expected_path_prefix:
         },
         "request": {
             "method": "GET",
-            "uri": f"/{requested}/{path_file}",
+            "uri": request_uri,
             "headers": {
                 "host": {"value": CF_HOST}
             },
@@ -115,21 +124,19 @@ def test_route(client: object, etag: str, requested: str, expected_path_prefix:
     test_result = json.loads(test_response['TestResult']['FunctionOutput'])
     test_result_uri = test_result['request']['uri']
     logger.debug(f"got test result uri: {test_result_uri}")
-    full_expected_path = f'{os.path.dirname(expected_path_prefix)}/{path_file}'
+    full_expected_path = f"{os.path.dirname(route['raw'])}/{route['file']}"
     if test_result_uri == full_expected_path:
-        logger.debug(f"Test path '/{requested}/{path_file}' passed, got expected uri {full_expected_path}")
+        logger.debug(f"Test path '{request_uri}' passed, got expected uri {full_expected_path}")
     else:
-        logger.error(f"Test path '/{requested}/{path_file}' failed, got unexpected uri {test_result_uri}, expected {full_expected_path}")
-        exit(1)
+        raise ValueError(f"Test path '{request_uri}' failed, got unexpected uri {test_result_uri}, expected {full_expected_path}")
 
     # if a file is expected then independently verify it exists in Github
-    if expected_file:
-        file_result = requests.get(f'https://raw.githubusercontent.com{full_expected_path}')
-        file_result.raise_for_status()
+    file_result = requests.get(f'https://raw.githubusercontent.com{full_expected_path}')
+    file_result.raise_for_status()
 
 
-for name, data in CF_ROUTES.items():
-    test_route(client=client, etag=candidate_function_etag, requested=name, expected_path_prefix=data['uri'], expected_file=data['file'])
+for route in routes:
+    test_route(client=client, etag=candidate_function_etag, route=route)
 
 # promote candidate from DEVELOPMENT to LIVE
 publish_response = client.publish_function(

dist/cloudfront/get.openziti.io/routes.yml

@@ -0,0 +1,30 @@
+# this is a list of shortcut routes at get.openziti.io for raw.githubusercontent.com
+
+# these are file shortcuts, so the shortcut is the test file
+- get: /ziti-cli-functions.sh
+  raw: /openziti/ziti/{{GITHUB_SHA}}/quickstart/docker/image/
+
+- get: /miniziti.bash
+  raw: /openziti/ziti/{{GITHUB_SHA}}/quickstart/kubernetes/
+
+# these are directory shortcuts, so you must supply a test file
+- get: /quick/
+  raw: /openziti/ziti/{{GITHUB_SHA}}/quickstart/docker/image/
+  file: ziti-cli-functions.sh
+
+- get: /dock/
+  raw: /openziti/ziti/{{GITHUB_SHA}}/quickstart/docker/
+  file: docker-compose.yml
+
+- get: /spec/
+  raw: /openziti/edge-api/main/
+  file: management.yml
+
+- get: /tun/
+  raw: /openziti/ziti-tunnel-sdk-c/main/
+  file: scripts/install-ubuntu.bash
+  # file: docker/ziti-tun-daemonset.yaml
+
+- get: /pack/
+  raw: /openziti/ziti-tunnel-sdk-c/main/
+  file: package-repos.gpg