Add prod server scripts and config

2026-10-25 13:19:46 +00:00 · 2021-12-14 18:45:48 +01:00
parent 315e5355e6
commit 5688d8b710
18 changed files with 464 additions and 43 deletions
@@ -64,4 +64,42 @@ Then you can add the proxied source via:
 Now you can inspect the traffic in the output of `sslproxy` when you perform
 `winget` commands such as:

-    winget search "search term" -s ms
+    winget search "search term" -s ms
+
+## Deploying
+
+Upload [`install.sh`](install.sh) and a `.bashrc` file with the settings below
+to a Debian 11 server, then run `install.sh` as root.
+
+```bash
+# Email address of server administrator.
+export ADMIN_EMAIL=michael.herrmann@omaha-consulting.com
+
+# The domain under which this server runs
+export HOST_NAME=winget.sh
+
+# The server that hosts this repository:
+export GIT_SERVER=github.com
+
+# The path on GIT_SERVER where this repository lies:
+export GIT_REPO_NAME=omaha-consulting/winget-private-repository
+
+# The branch of this repository that should be checked out:
+export GIT_REPO_BRANCH=main
+
+# The SSH public key for fetching this repository from GIT_SERVER:
+export GIT_REPO_PUBKEY="ssh-rsa ... user@machine"
+
+# The SSH private key for fetching this repository from GIT_SERVER.
+# "\n"s separate lines:
+export GIT_REPO_PRIVKEY="-----BEGIN RSA PRIVATE KEY-----\n...\n-----END RSA PRIVATE KEY-----"
+
+# Emails to the admin will be sent via this account:
+export SMTP_HOST=email-smtp.eu-central-1.amazonaws.com
+export SMTP_USER=ABC123...
+export SMTP_PASSWORD=aBc1234...
+export SMTP_FROM=winget@servers.omaha-consulting.com
+
+# Django SECRET_KEY:
+export SECRET_KEY=atLeast50RandomChars
+```
@@ -0,0 +1,48 @@
+#!/bin/bash
+
+# Cronic v2 - cron job report wrapper
+# Copyright 2007 Chuck Houpt. No rights reserved, whatsoever.
+# Public Domain CC0: http://creativecommons.org/publicdomain/zero/1.0/
+
+set -eu
+
+OUT=/tmp/cronic.out.$$
+ERR=/tmp/cronic.err.$$
+TRACE=/tmp/cronic.trace.$$
+
+set +e
+"$@" >$OUT 2>$TRACE
+RESULT=$?
+set -e
+
+PATTERN="^${PS4:0:1}\\+${PS4:1}"
+if grep -aq "$PATTERN" $TRACE
+then
+    ! grep -av "$PATTERN" $TRACE > $ERR
+else
+    ERR=$TRACE
+fi
+
+if [ $RESULT -ne 0 ]
+    then
+    echo "Cronic detected failure for the command:"
+    echo "$@"
+    echo
+    echo "RESULT CODE: $RESULT"
+    echo
+    echo "ERROR OUTPUT:"
+    cat "$ERR"
+    echo
+    echo "STANDARD OUTPUT:"
+    cat "$OUT"
+    if [ $TRACE != $ERR ]
+    then
+        echo
+        echo "TRACE-ERROR OUTPUT:"
+        cat "$TRACE"
+    fi
+fi
+
+rm -f "$OUT"
+rm -f "$ERR"
+rm -f "$TRACE"
@@ -0,0 +1,31 @@
+#!/bin/bash
+
+NAME="gunicorn"
+DJANGODIR=/srv
+SOCKFILE=/var/run/gunicorn.sock
+GROUP=django
+NUM_WORKERS=1
+NUM_THREADS=5
+DJANGO_WSGI_MODULE=wpr.wsgi
+
+echo "Starting $NAME as `whoami`"
+
+# Activate the virtual environment
+cd $DJANGODIR
+export PYTHONPATH=$DJANGODIR:$PYTHONPATH
+
+# Create the run directory if it doesn't exist
+RUNDIR=$(dirname $SOCKFILE)
+test -d $RUNDIR || mkdir -p $RUNDIR
+
+# Start your Django Unicorn
+# Programs meant to be run under supervisor should not daemonize themselves
+# (do not use --daemon)
+exec gunicorn ${DJANGO_WSGI_MODULE}:application \
+  --name $NAME \
+  --workers $NUM_WORKERS \
+  --threads $NUM_THREADS \
+  --user=$USER --group=$GROUP \
+  --bind=unix:$SOCKFILE \
+  --log-level=warning \
+  --log-file=-
@@ -0,0 +1,6 @@
+#!/bin/bash
+
+set -e
+
+cd /srv
+./manage.py "$@"
@@ -0,0 +1,10 @@
+#!/bin/bash
+# Following an idea similar to https://web.archive.org/web/20130526045634/http:/
+# /sjsnyder.com/managing-virtualenv-apps-with-supervisor
+
+# Calculate the home directory from $(id -u). The motivation for this is that
+# Supervisor only sets the UID.
+export USER=$(getent passwd | cut -d : -f 1,3 | grep ":$(id -u)$" | cut -d : -f 1)
+export HOME=$(getent passwd $USER | cut -d : -f 6)
+source $HOME/.bashrc
+exec $@
@@ -0,0 +1,2 @@
+MAILTO=$ADMIN_EMAIL
+26 2 */7 * * cronic certbot renew
@@ -0,0 +1,3 @@
+export DEBUG=False
+export HOST_NAME=$HOST_NAME
+export PATH="/srv/venv/bin:$PATH"
@@ -0,0 +1,15 @@
+[program:gunicorn]
+command=/srv/bin/with-bashrc.sh /srv/bin/gunicorn.sh
+directory=/srv
+user=private
+stdout_logfile=/var/log/gunicorn.log
+stderr_logfile=/var/log/gunicorn.log
+autostart=true
+autorestart=true
+startsecs=10
+
+; Disable log file rotation - We use logrotate for this.
+stdout_logfile_maxbytes=0
+stderr_logfile_maxbytes=0
+stdout_logfile_backups=0
+stderr_logfile_backups=0
@@ -0,0 +1,29 @@
+/var/log/nginx-access.log
+/var/log/nginx-error.log {
+    su root django
+    create 0640 www-data adm
+    missingok
+    rotate 5
+    size 100M
+    compress
+    delaycompress
+    notifempty
+    sharedscripts
+    postrotate
+        nginx -s reopen
+    endscript
+}
+
+/var/log/gunicorn.log {
+    su root django
+    create 0640 root django
+    missingok
+    rotate 5
+    size 100M
+    compress
+    delaycompress
+    notifempty
+    postrotate
+        kill -USR2 `cat /run/supervisord.pid`
+    endscript
+}
@@ -0,0 +1,37 @@
+upstream django {
+  # fail_timeout=0 means we always retry an upstream even if it failed
+  # to return a good HTTP response (in case the Unicorn master nukes a
+  # single worker for timing out).
+
+  server unix:/var/run/gunicorn.sock fail_timeout=0;
+}
+
+server {
+    listen 80;
+    listen 443 ssl;
+    include includes/server-name;
+
+    client_max_body_size 500m;
+
+    access_log /var/log/nginx-access.log;
+    error_log /var/log/nginx-error.log;
+
+    location /static/ {
+        alias /srv/static/;
+    }
+
+    location /media/ {
+        alias /srv/media/;
+    }
+
+    location / {
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Protocol $scheme;
+        proxy_set_header Host $host;
+        proxy_redirect off;
+        if (!-f $request_filename) {
+            proxy_pass http://django;
+            break;
+        }
+    }
+}
@@ -0,0 +1 @@
+server_name $HOST_NAME;
@@ -0,0 +1,2 @@
+ssl_certificate /etc/letsencrypt/live/main/fullchain.pem;
+ssl_certificate_key /etc/letsencrypt/live/main/privkey.pem;
@@ -0,0 +1 @@
+@$HOST_NAME $SMTP_FROM
@@ -0,0 +1,48 @@
+# See /usr/share/postfix/main.cf.dist for a commented, more complete version
+
+
+# Debian specific:  Specifying a file name will cause the first
+# line of that file to be used as the name.  The Debian default
+# is /etc/mailname.
+#myorigin = /etc/mailname
+
+smtpd_banner = $myhostname ESMTP $mail_name (Ubuntu)
+biff = no
+
+# appending .domain is the MUA's job.
+append_dot_mydomain = no
+
+# Uncomment the next line to generate "delayed mail" warnings
+#delay_warning_time = 4h
+
+readme_directory = no
+
+# TLS parameters
+smtpd_tls_cert_file=/etc/ssl/certs/ssl-cert-snakeoil.pem
+smtpd_tls_key_file=/etc/ssl/private/ssl-cert-snakeoil.key
+smtpd_use_tls=yes
+smtpd_tls_session_cache_database = btree:${data_directory}/smtpd_scache
+smtp_tls_session_cache_database = btree:${data_directory}/smtp_scache
+
+# See /usr/share/doc/postfix/TLS_README.gz in the postfix-doc package for
+# information on enabling SSL in the smtp client.
+
+smtpd_relay_restrictions = permit_mynetworks permit_sasl_authenticated defer_unauth_destination
+myhostname = $HOST_NAME
+alias_maps = hash:/etc/aliases
+alias_database = hash:/etc/aliases
+myorigin = /etc/mailname
+mydestination = localhost.localdomain, localhost
+
+relayhost = [$SMTP_HOST]:587
+smtp_generic_maps = hash:/etc/postfix/generic
+smtp_sasl_auth_enable = yes
+smtp_sasl_password_maps = hash:/etc/postfix/sasl_passwd
+smtp_sasl_security_options = noanonymous
+#smtp_tls_CAfile = /etc/postfix/cacert.pem
+smtp_use_tls = yes
+
+mynetworks = 127.0.0.0/8 [::ffff:127.0.0.0]/104 [::1]/128
+mailbox_size_limit = 0
+recipient_delimiter = +
+inet_interfaces = 127.0.0.1
@@ -0,0 +1 @@
+[$SMTP_HOST]:587    $SMTP_USER:$SMTP_PASSWORD
@@ -0,0 +1,165 @@
+#!/bin/bash
+
+# Installs winget-private-repository and all dependencies on a fresh Debian 11.
+# To use, upload this file and .bashrc to /root, then execute this script as
+# root.
+
+# Error handling:
+set -e
+trap 'echo "Error on line $LINENO"' ERR
+
+################################################################################
+#                                  Settings                                    #
+################################################################################
+
+chmod 600 .bashrc
+source .bashrc
+
+################################################################################
+#                                 Settings end                                 #
+################################################################################
+
+log() {
+    echo -e "\n\033[0;32m`date +%H:%M:%S`\033[0m \033[0;93m$1\033[0m"
+}
+
+################################################################################
+#                                 Script start                                 #
+################################################################################
+
+log 'Setting hostname...'
+hostnamectl set-hostname "$HOST_NAME"
+
+log 'Updating system...'
+apt-get update > /dev/null
+apt-get upgrade -y > /dev/null
+
+log 'Installing iptables-persistent...'
+echo iptables-persistent iptables-persistent/autosave_v4 boolean true | debconf-set-selections
+echo iptables-persistent iptables-persistent/autosave_v6 boolean true | debconf-set-selections
+apt-get -y install iptables-persistent -y > /dev/null
+
+log 'Configuring iptables...'
+# Only allow local access to port 25:
+iptables -A INPUT -i lo -p tcp --dport 25 -j ACCEPT
+iptables -A INPUT -p tcp --dport 25 -j REJECT
+
+# Save iptables rules so they persist across restarts. At the time of this
+# writing, there is no open IPv6 port, so enough to save rules.v*4*.
+iptables-save > /etc/iptables/rules.v4
+
+log 'Installing git...'
+apt-get install git-core -y > /dev/null
+
+log 'Creating application users...'
+groupadd --system django
+useradd --system --gid django --shell /bin/bash django
+mkdir -p /home/django
+chown django /home/django
+chmod 700 /home/django
+
+log 'Setting up SSH keys. This is required so we can clone the Git repository without having to supply a password...'
+mkdir -p .ssh
+echo "$GIT_REPO_PUBKEY" > .ssh/id_rsa.pub
+chmod 644 .ssh/id_rsa.pub
+echo -e "$GIT_REPO_PRIVKEY" > .ssh/id_rsa
+chmod 600 .ssh/id_rsa
+
+log 'Adding git repository server to known hosts...'
+ssh-keyscan -H ${GIT_SERVER} >> .ssh/known_hosts 2>/dev/null; chmod 600 .ssh/known_hosts
+
+log 'Cloning the repository...'
+git clone -q -b ${GIT_REPO_BRANCH} git@${GIT_SERVER}:${GIT_REPO_NAME}.git /srv
+
+log 'Setting up bash profiles...'
+ln -sf /srv/conf/.bash_profile .
+ln -sf /srv/conf/.bash_profile /home/django
+PATH='$PATH' /usr/bin/envsubst < /srv/conf/django.bashrc > /home/django/.bashrc
+chown django:django /home/django/.bashrc
+
+log 'Installing Postfix...'
+debconf-set-selections <<< "postfix postfix/mailname string $HOST_NAME"
+debconf-set-selections <<< "postfix postfix/main_mailer_type string 'Internet Site'"
+apt-get install postfix mailutils libsasl2-2 ca-certificates libsasl2-modules -y > /dev/null
+
+log 'Configuring Postfix...'
+echo "$HOST_NAME" > /etc/mailname
+chown postfix /etc/mailname
+envsubst '$SMTP_HOST $HOST_NAME' < /srv/conf/postfix/main.cf > /etc/postfix/main.cf
+envsubst < /srv/conf/postfix/sasl_passwd > /etc/postfix/sasl_passwd
+postmap /etc/postfix/sasl_passwd
+chmod 400 /etc/postfix/sasl_passwd
+chown postfix /etc/postfix/sasl_passwd
+envsubst < /srv/conf/postfix/generic > /etc/postfix/generic
+postmap /etc/postfix/generic
+chown postfix /etc/postfix/generic
+/etc/init.d/postfix reload > /dev/null
+
+log 'Installing OS dependencies for our Python environment...'
+apt-get install python3-venv -y > /dev/null
+
+log 'Creating virtual environment...'
+python3 -m venv /srv/venv > /dev/null
+
+log 'Installing Python dependencies...'
+# N.B.: pip should be on PATH from sourcing .bashrc at the top.
+pip install -Ur /srv/requirements/base-full.txt > /dev/null
+
+log 'Migrating database...'
+su -c '/srv/manage.sh migrate | grep -v "INFO\|DEBUG"' - django
+
+log 'Collecting static files...'
+su -c '/srv/manage.sh collectstatic --noinput | grep -v "INFO\|DEBUG"' - django
+
+log 'Installing Supervisor...'
+apt-get install supervisor -y > /dev/null
+
+log 'Configuring Supervisor...'
+ln -s /srv/conf/supervisor/*.conf /etc/supervisor/conf.d
+
+log 'Starting services...'
+supervisorctl reread > /dev/null
+supervisorctl update > /dev/null
+
+log 'Installing Nginx...'
+apt-get install nginx -y > /dev/null
+
+log 'Creating Nginx includes directory...'
+mkdir /etc/nginx/includes
+
+log 'Creating Nginx config...'
+envsubst < /srv/conf/nginx/server-name > /etc/nginx/includes/server-name
+touch /etc/nginx/includes/ssl-cert
+
+log 'Applying Nginx config...'
+ln -s /srv/conf/nginx/nginx /etc/nginx/sites-available/wpr
+ln -s /etc/nginx/sites-available/wpr /etc/nginx/sites-enabled/wpr
+rm /etc/nginx/sites-enabled/default
+
+log 'Starting Nginx...'
+service nginx restart
+
+log 'Generating SSL certificates...'
+apt-get install certbot python3-certbot-nginx -y > /dev/null
+certbot register --quiet --email $ADMIN_EMAIL --agree-tos
+certbot certonly --nginx --quiet --cert-name main -d $HOST_NAME
+ln -sf /srv/conf/nginx/ssl-cert /etc/nginx/includes/ssl-cert
+service nginx restart
+
+log "The server is now serving requests at $HOST_NAME!"
+
+# Now do less important things that are not required for serving requests:
+
+log 'Setting up logrotate...'
+ln -s /srv/conf/logrotate /etc/logrotate.d/wpr
+
+log 'Setting up crontab...'
+ln -s /srv/bin/cronic /usr/bin/cronic
+sed -i "s/\$ADMIN_EMAIL/$ADMIN_EMAIL/g" /srv/conf/crontab
+crontab /srv/conf/crontab
+
+log 'Setting up automatic updates...'
+apt-get install unattended-upgrades -y > /dev/null
+printf 'APT::Periodic::Update-Package-Lists "1";\nAPT::Periodic::Unattended-Upgrade "1";\n' > /etc/apt/apt.conf.d/20auto-upgrades
+
+log 'Done.'
@@ -1,34 +1,21 @@
-"""
-Django settings for wpr project.
-
-Generated by 'django-admin startproject' using Django 3.2.10.
-
-For more information on this file, see
-https://docs.djangoproject.com/en/3.2/topics/settings/
-
-For the full list of settings and their values, see
-https://docs.djangoproject.com/en/3.2/ref/settings/
-"""
-
+import os
 from pathlib import Path

-# Build paths inside the project like this: BASE_DIR / 'subdir'.
+from wpr.util import get_bool_from_env
+
 BASE_DIR = Path(__file__).resolve().parent.parent

-
-# Quick-start development settings - unsuitable for production
-# See https://docs.djangoproject.com/en/3.2/howto/deployment/checklist/
-
 # SECURITY WARNING: keep the secret key used in production secret!
-SECRET_KEY = 'django-insecure-n$m*a%3#nqpl5=(sp7v+!6r98mld#o4*(24!dawoma2i5(%lh='
+SECRET_KEY = os.getenv('SECRET_KEY') or \
+             'django-insecure-n$m*a%3#nqpl5=(sp7v+!6r98mld#o4*(24!dawoma2i5(%lh'

 # SECURITY WARNING: don't run with debug turned on in production!
-DEBUG = True
+DEBUG = get_bool_from_env('DEBUG', True)

 ALLOWED_HOSTS = []

-
-# Application definition
+if os.getenv('HOST_NAME'):
+    ALLOWED_HOSTS.append(os.getenv('HOST_NAME'))

 INSTALLED_APPS = [
    'suit',
@@ -74,10 +61,6 @@ TEMPLATES = [

 WSGI_APPLICATION = 'wpr.wsgi.application'

-
-# Database
-# https://docs.djangoproject.com/en/3.2/ref/settings/#databases
-
 DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.sqlite3',
@@ -85,10 +68,6 @@ DATABASES = {
    }
 }

-
-# Password validation
-# https://docs.djangoproject.com/en/3.2/ref/settings/#auth-password-validators
-
 AUTH_PASSWORD_VALIDATORS = [
    {
        'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
@@ -104,10 +83,6 @@ AUTH_PASSWORD_VALIDATORS = [
    },
 ]

-
-# Internationalization
-# https://docs.djangoproject.com/en/3.2/topics/i18n/
-
 LANGUAGE_CODE = 'en-us'

 TIME_ZONE = 'UTC'
@@ -118,21 +93,13 @@ USE_L10N = True

 USE_TZ = True

-
-# Static files (CSS, JavaScript, Images)
-# https://docs.djangoproject.com/en/3.2/howto/static-files/
-
 STATIC_URL = '/static/'

 MEDIA_ROOT = BASE_DIR / 'media'
 MEDIA_URL = 'media/'

-# Default primary key field type
-# https://docs.djangoproject.com/en/3.2/ref/settings/#default-auto-field
-
 DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField'

-# Django Suit config
 SUIT_CONFIG = {
    'ADMIN_NAME': 'winget Admin',
    'MENU': (
@@ -141,4 +108,4 @@ SUIT_CONFIG = {
         'models': ('auth.user', 'tenants.tenant')}
    ),
    'CONFIRM_UNSAVED_CHANGES': False
-}
+}
@@ -0,0 +1,17 @@
+import os
+
+from django.core.exceptions import ImproperlyConfigured
+
+
+def get_bool_from_env(name, default):
+    result_str = os.getenv(name)
+    if not result_str:
+        return default
+    if result_str == 'True':
+        return True
+    elif result_str == 'False':
+        return False
+    raise ImproperlyConfigured(
+        'Environment variable %r has invalid value. Must be True or False.'
+        % name
+    )