Make it possible to host update files on S3

README.md

@@ -106,4 +106,24 @@ export SMTP_FROM=server@email.com
 
 # Django SECRET_KEY:
 export SECRET_KEY=atLeast50RandomChars
-```
+
+DJANGO_SETTINGS_MODULE=core.settings
+
+################## Optional settings for storing files on S3 ###################
+DEFAULT_FILE_STORAGE=storages.backends.s3boto3.S3Boto3Storage
+AWS_STORAGE_BUCKET_NAME=mybucket
+AWS_ACCESS_KEY_ID=ABC....
+AWS_SECRET_ACCESS_KEY=XYZ...
+# Optionally set the access policy for uploaded files. The default is "private".
+# Even with "private", update clients receive access to the files via presigned
+# S3 URLs. Note however that the associated requests to S3 incur a performance
+# overhead. To avoid it, set AWS_S3_CUSTOM_DOMAIN below.
+AWS_DEFAULT_ACL=public-read
+# Optionally supply a CDN domain. You can set this to <bucket>.s3.amazonaws.com
+# to avoid the performance overhead of presigned URLs. This requires that the
+# files in the bucket are publicly readable. For example, via
+# AWS_DEFAULT_ACL=public-read. 
+AWS_S3_CUSTOM_DOMAIN=cdn.winget.pro
+# To use a non-AWS S3 endpoint, you can optionally set the following:
+AWS_S3_HOST=us-east-1.linodeobjects.com
+```

core/settings.py

@@ -1,7 +1,8 @@
-import os
+from django.conf import global_settings
+from core.util import get_bool_from_env
 from pathlib import Path
 
-from core.util import get_bool_from_env
+import os
 
 BASE_DIR = Path(__file__).resolve().parent.parent
 
@@ -105,6 +106,19 @@ STATIC_ROOT = BASE_DIR / 'static'
 MEDIA_ROOT = BASE_DIR / 'media'
 MEDIA_URL = 'media/'
 
+# Optional settings for serving files from S3 with django-storages:
+DEFAULT_FILE_STORAGE = os.getenv('DEFAULT_FILE_STORAGE') or \
+                       global_settings.DEFAULT_FILE_STORAGE
+
+if DEFAULT_FILE_STORAGE == 'storages.backends.s3boto3.S3Boto3Storage':
+    AWS_STORAGE_BUCKET_NAME = os.environ['AWS_STORAGE_BUCKET_NAME']
+    AWS_ACCESS_KEY_ID = os.environ['AWS_ACCESS_KEY_ID']
+    AWS_SECRET_ACCESS_KEY = os.environ['AWS_SECRET_ACCESS_KEY']
+    AWS_S3_HOST = os.getenv('AWS_S3_HOST') or 's3.amazonaws.com'
+    AWS_S3_ENDPOINT_URL = 'https://' + AWS_S3_HOST
+    AWS_DEFAULT_ACL = os.getenv('AWS_DEFAULT_ACL')
+    AWS_S3_CUSTOM_DOMAIN = os.getenv('AWS_S3_CUSTOM_DOMAIN')
+
 DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField'
 
 ADMINS = []

requirements/base-full.txt

@@ -1,10 +1,18 @@
 # Output of `pip freeze` after `pip install -Ur base.txt`.
 
 asgiref==3.7.2
+boto3==1.28.22
+botocore==1.31.24
 Django==3.2.20
+django-storages==1.13.2
 django-suit @ https://github.com/lumocodes/django-suit/archive/suit-0.2.28-for-django3.1.zip
 djangorestframework==3.14.0
 gunicorn==20.1.0
+jmespath==1.0.1
+python-dateutil==2.8.2
 pytz==2023.3
+s3transfer==0.6.1
+six==1.16.0
 sqlparse==0.4.4
 typing_extensions==4.7.1
+urllib3==1.26.16

requirements/base.txt

@@ -4,4 +4,6 @@ django==3.2.20
 # django-suit v0.2.28 fixed for Django 3.1+:
 https://github.com/lumocodes/django-suit/archive/suit-0.2.28-for-django3.1.zip#egg=django-suit
 gunicorn==20.1.0
-djangorestframework==3.14.0
+djangorestframework==3.14.0
+django-storages==1.13.2
+boto3==1.28.22

winget/migrations/0005_installer_upload_to.py

@@ -0,0 +1,19 @@
+# Generated by Django 3.2.20 on 2023-08-09 16:41
+
+from django.db import migrations, models
+import winget.models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('winget', '0004_improve_help_texts'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='installer',
+            name='file',
+            field=models.FileField(upload_to=winget.models.installer_upload_to),
+        ),
+    ]

winget/models.py

@@ -5,7 +5,7 @@ from django.db.models.signals import pre_save
 from django.dispatch import receiver
 from hashlib import sha256
 from tenants.models import Tenant
-from winget.util import CharFieldFromChoices
+from winget.util import CharFieldFromChoices, randomize_filename
 
 
 class Package(Model):
@@ -56,6 +56,13 @@ class Version(Model):
         return result
 
 
+def installer_upload_to(instance, filename):
+    # Randomize the upload path. This prevents users from guessing it and
+    # prevents clashes.
+    randomized_filename = randomize_filename(filename)
+    return f'{instance.version.package.tenant.uuid}/{randomized_filename}'
+
+
 class Installer(Model):
     version = ForeignKey(Version, on_delete=CASCADE)
     architecture = CharFieldFromChoices('x86', 'x64', 'arm', 'arm64')
@@ -68,7 +75,7 @@ class Installer(Model):
         help_text=
         "Is this a machine-wide installer, just for the current user, or both?"
     )
-    file = FileField()
+    file = FileField(upload_to=installer_upload_to)
     sha256 = CharField(
         max_length=64, validators=[RegexValidator('^[a-fA-F0-9]{64}$')]
     )

winget/tests/test_util.py

@@ -0,0 +1,9 @@
+from unittest import TestCase
+from winget.util import randomize_filename
+
+class RandomizeFilenameTest(TestCase):
+    def test_randomize_filename(self):
+        self.assertEqual(
+            randomize_filename('installer.exe', seed=1631548913),
+            'installer-v34EnYOkYJ.exe'
+        )

winget/util.py

@@ -1,5 +1,8 @@
 import json
+import string
 from functools import wraps
+from os.path import splitext
+from random import Random
 
 from django.db.models import CharField
 from django.http import JsonResponse
@@ -39,3 +42,11 @@ def return_jsonresponse(func):
         return JsonResponse({'Data': func(*args, **kwargs)})
 
     return inner
+
+
+def randomize_filename(filename, seed=None, prefix='-', suffix='', length=10):
+    basename, extension = splitext(filename)
+    letters = string.ascii_letters + string.digits
+    random = Random(seed)
+    random_suffix = ''.join(random.choice(letters) for _ in range(length))
+    return basename + prefix + random_suffix + suffix + extension