diff --git a/README.md b/README.md index 8e3873f..0deded4 100644 --- a/README.md +++ b/README.md @@ -64,4 +64,42 @@ Then you can add the proxied source via: Now you can inspect the traffic in the output of `sslproxy` when you perform `winget` commands such as: - winget search "search term" -s ms \ No newline at end of file + winget search "search term" -s ms + +## Deploying + +Upload [`install.sh`](install.sh) and a `.bashrc` file with the settings below +to a Debian 11 server, then run `install.sh` as root. + +```bash +# Email address of server administrator. +export ADMIN_EMAIL=michael.herrmann@omaha-consulting.com + +# The domain under which this server runs +export HOST_NAME=winget.sh + +# The server that hosts this repository: +export GIT_SERVER=github.com + +# The path on GIT_SERVER where this repository lies: +export GIT_REPO_NAME=omaha-consulting/winget-private-repository + +# The branch of this repository that should be checked out: +export GIT_REPO_BRANCH=main + +# The SSH public key for fetching this repository from GIT_SERVER: +export GIT_REPO_PUBKEY="ssh-rsa ... user@machine" + +# The SSH private key for fetching this repository from GIT_SERVER. +# "\n"s separate lines: +export GIT_REPO_PRIVKEY="-----BEGIN RSA PRIVATE KEY-----\n...\n-----END RSA PRIVATE KEY-----" + +# Emails to the admin will be sent via this account: +export SMTP_HOST=email-smtp.eu-central-1.amazonaws.com +export SMTP_USER=ABC123... +export SMTP_PASSWORD=aBc1234... +export SMTP_FROM=winget@servers.omaha-consulting.com + +# Django SECRET_KEY: +export SECRET_KEY=atLeast50RandomChars +``` \ No newline at end of file diff --git a/bin/cronic b/bin/cronic new file mode 100755 index 0000000..9290f93 --- /dev/null +++ b/bin/cronic @@ -0,0 +1,48 @@ +#!/bin/bash + +# Cronic v2 - cron job report wrapper +# Copyright 2007 Chuck Houpt. No rights reserved, whatsoever. +# Public Domain CC0: http://creativecommons.org/publicdomain/zero/1.0/ + +set -eu + +OUT=/tmp/cronic.out.$$ +ERR=/tmp/cronic.err.$$ +TRACE=/tmp/cronic.trace.$$ + +set +e +"$@" >$OUT 2>$TRACE +RESULT=$? +set -e + +PATTERN="^${PS4:0:1}\\+${PS4:1}" +if grep -aq "$PATTERN" $TRACE +then + ! grep -av "$PATTERN" $TRACE > $ERR +else + ERR=$TRACE +fi + +if [ $RESULT -ne 0 ] + then + echo "Cronic detected failure for the command:" + echo "$@" + echo + echo "RESULT CODE: $RESULT" + echo + echo "ERROR OUTPUT:" + cat "$ERR" + echo + echo "STANDARD OUTPUT:" + cat "$OUT" + if [ $TRACE != $ERR ] + then + echo + echo "TRACE-ERROR OUTPUT:" + cat "$TRACE" + fi +fi + +rm -f "$OUT" +rm -f "$ERR" +rm -f "$TRACE" diff --git a/bin/gunicorn.sh b/bin/gunicorn.sh new file mode 100755 index 0000000..ddd585a --- /dev/null +++ b/bin/gunicorn.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +NAME="gunicorn" +DJANGODIR=/srv +SOCKFILE=/var/run/gunicorn.sock +GROUP=django +NUM_WORKERS=1 +NUM_THREADS=5 +DJANGO_WSGI_MODULE=wpr.wsgi + +echo "Starting $NAME as `whoami`" + +# Activate the virtual environment +cd $DJANGODIR +export PYTHONPATH=$DJANGODIR:$PYTHONPATH + +# Create the run directory if it doesn't exist +RUNDIR=$(dirname $SOCKFILE) +test -d $RUNDIR || mkdir -p $RUNDIR + +# Start your Django Unicorn +# Programs meant to be run under supervisor should not daemonize themselves +# (do not use --daemon) +exec gunicorn ${DJANGO_WSGI_MODULE}:application \ + --name $NAME \ + --workers $NUM_WORKERS \ + --threads $NUM_THREADS \ + --user=$USER --group=$GROUP \ + --bind=unix:$SOCKFILE \ + --log-level=warning \ + --log-file=- \ No newline at end of file diff --git a/bin/manage.sh b/bin/manage.sh new file mode 100755 index 0000000..f0ac2eb --- /dev/null +++ b/bin/manage.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +set -e + +cd /srv +./manage.py "$@" \ No newline at end of file diff --git a/bin/with-bashrc.sh b/bin/with-bashrc.sh new file mode 100755 index 0000000..5de4e55 --- /dev/null +++ b/bin/with-bashrc.sh @@ -0,0 +1,10 @@ +#!/bin/bash +# Following an idea similar to https://web.archive.org/web/20130526045634/http:/ +# /sjsnyder.com/managing-virtualenv-apps-with-supervisor + +# Calculate the home directory from $(id -u). The motivation for this is that +# Supervisor only sets the UID. +export USER=$(getent passwd | cut -d : -f 1,3 | grep ":$(id -u)$" | cut -d : -f 1) +export HOME=$(getent passwd $USER | cut -d : -f 6) +source $HOME/.bashrc +exec $@ \ No newline at end of file diff --git a/conf/crontab b/conf/crontab new file mode 100644 index 0000000..27871bf --- /dev/null +++ b/conf/crontab @@ -0,0 +1,2 @@ +MAILTO=$ADMIN_EMAIL +26 2 */7 * * cronic certbot renew diff --git a/conf/django.bashrc b/conf/django.bashrc new file mode 100755 index 0000000..8ae76b8 --- /dev/null +++ b/conf/django.bashrc @@ -0,0 +1,3 @@ +export DEBUG=False +export HOST_NAME=$HOST_NAME +export PATH="/srv/venv/bin:$PATH" \ No newline at end of file diff --git a/conf/gunicorn.conf b/conf/gunicorn.conf new file mode 100644 index 0000000..64db9df --- /dev/null +++ b/conf/gunicorn.conf @@ -0,0 +1,15 @@ +[program:gunicorn] +command=/srv/bin/with-bashrc.sh /srv/bin/gunicorn.sh +directory=/srv +user=private +stdout_logfile=/var/log/gunicorn.log +stderr_logfile=/var/log/gunicorn.log +autostart=true +autorestart=true +startsecs=10 + +; Disable log file rotation - We use logrotate for this. +stdout_logfile_maxbytes=0 +stderr_logfile_maxbytes=0 +stdout_logfile_backups=0 +stderr_logfile_backups=0 \ No newline at end of file diff --git a/conf/logrotate b/conf/logrotate new file mode 100644 index 0000000..660aeea --- /dev/null +++ b/conf/logrotate @@ -0,0 +1,29 @@ +/var/log/nginx-access.log +/var/log/nginx-error.log { + su root django + create 0640 www-data adm + missingok + rotate 5 + size 100M + compress + delaycompress + notifempty + sharedscripts + postrotate + nginx -s reopen + endscript +} + +/var/log/gunicorn.log { + su root django + create 0640 root django + missingok + rotate 5 + size 100M + compress + delaycompress + notifempty + postrotate + kill -USR2 `cat /run/supervisord.pid` + endscript +} \ No newline at end of file diff --git a/conf/nginx/nginx b/conf/nginx/nginx new file mode 100644 index 0000000..b994970 --- /dev/null +++ b/conf/nginx/nginx @@ -0,0 +1,37 @@ +upstream django { + # fail_timeout=0 means we always retry an upstream even if it failed + # to return a good HTTP response (in case the Unicorn master nukes a + # single worker for timing out). + + server unix:/var/run/gunicorn.sock fail_timeout=0; +} + +server { + listen 80; + listen 443 ssl; + include includes/server-name; + + client_max_body_size 500m; + + access_log /var/log/nginx-access.log; + error_log /var/log/nginx-error.log; + + location /static/ { + alias /srv/static/; + } + + location /media/ { + alias /srv/media/; + } + + location / { + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Protocol $scheme; + proxy_set_header Host $host; + proxy_redirect off; + if (!-f $request_filename) { + proxy_pass http://django; + break; + } + } +} \ No newline at end of file diff --git a/conf/nginx/server-name b/conf/nginx/server-name new file mode 100644 index 0000000..dcc87a2 --- /dev/null +++ b/conf/nginx/server-name @@ -0,0 +1 @@ +server_name $HOST_NAME; \ No newline at end of file diff --git a/conf/nginx/ssl-cert b/conf/nginx/ssl-cert new file mode 100644 index 0000000..2ce66ec --- /dev/null +++ b/conf/nginx/ssl-cert @@ -0,0 +1,2 @@ +ssl_certificate /etc/letsencrypt/live/main/fullchain.pem; +ssl_certificate_key /etc/letsencrypt/live/main/privkey.pem; \ No newline at end of file diff --git a/conf/postfix/generic b/conf/postfix/generic new file mode 100644 index 0000000..f91ffd6 --- /dev/null +++ b/conf/postfix/generic @@ -0,0 +1 @@ +@$HOST_NAME $SMTP_FROM \ No newline at end of file diff --git a/conf/postfix/main.cf b/conf/postfix/main.cf new file mode 100644 index 0000000..1c5a884 --- /dev/null +++ b/conf/postfix/main.cf @@ -0,0 +1,48 @@ +# See /usr/share/postfix/main.cf.dist for a commented, more complete version + + +# Debian specific: Specifying a file name will cause the first +# line of that file to be used as the name. The Debian default +# is /etc/mailname. +#myorigin = /etc/mailname + +smtpd_banner = $myhostname ESMTP $mail_name (Ubuntu) +biff = no + +# appending .domain is the MUA's job. +append_dot_mydomain = no + +# Uncomment the next line to generate "delayed mail" warnings +#delay_warning_time = 4h + +readme_directory = no + +# TLS parameters +smtpd_tls_cert_file=/etc/ssl/certs/ssl-cert-snakeoil.pem +smtpd_tls_key_file=/etc/ssl/private/ssl-cert-snakeoil.key +smtpd_use_tls=yes +smtpd_tls_session_cache_database = btree:${data_directory}/smtpd_scache +smtp_tls_session_cache_database = btree:${data_directory}/smtp_scache + +# See /usr/share/doc/postfix/TLS_README.gz in the postfix-doc package for +# information on enabling SSL in the smtp client. + +smtpd_relay_restrictions = permit_mynetworks permit_sasl_authenticated defer_unauth_destination +myhostname = $HOST_NAME +alias_maps = hash:/etc/aliases +alias_database = hash:/etc/aliases +myorigin = /etc/mailname +mydestination = localhost.localdomain, localhost + +relayhost = [$SMTP_HOST]:587 +smtp_generic_maps = hash:/etc/postfix/generic +smtp_sasl_auth_enable = yes +smtp_sasl_password_maps = hash:/etc/postfix/sasl_passwd +smtp_sasl_security_options = noanonymous +#smtp_tls_CAfile = /etc/postfix/cacert.pem +smtp_use_tls = yes + +mynetworks = 127.0.0.0/8 [::ffff:127.0.0.0]/104 [::1]/128 +mailbox_size_limit = 0 +recipient_delimiter = + +inet_interfaces = 127.0.0.1 diff --git a/conf/postfix/sasl_passwd b/conf/postfix/sasl_passwd new file mode 100644 index 0000000..d4c5a41 --- /dev/null +++ b/conf/postfix/sasl_passwd @@ -0,0 +1 @@ +[$SMTP_HOST]:587 $SMTP_USER:$SMTP_PASSWORD \ No newline at end of file diff --git a/install.sh b/install.sh new file mode 100644 index 0000000..5c9cfcb --- /dev/null +++ b/install.sh @@ -0,0 +1,165 @@ +#!/bin/bash + +# Installs winget-private-repository and all dependencies on a fresh Debian 11. +# To use, upload this file and .bashrc to /root, then execute this script as +# root. + +# Error handling: +set -e +trap 'echo "Error on line $LINENO"' ERR + +################################################################################ +# Settings # +################################################################################ + +chmod 600 .bashrc +source .bashrc + +################################################################################ +# Settings end # +################################################################################ + +log() { + echo -e "\n\033[0;32m`date +%H:%M:%S`\033[0m \033[0;93m$1\033[0m" +} + +################################################################################ +# Script start # +################################################################################ + +log 'Setting hostname...' +hostnamectl set-hostname "$HOST_NAME" + +log 'Updating system...' +apt-get update > /dev/null +apt-get upgrade -y > /dev/null + +log 'Installing iptables-persistent...' +echo iptables-persistent iptables-persistent/autosave_v4 boolean true | debconf-set-selections +echo iptables-persistent iptables-persistent/autosave_v6 boolean true | debconf-set-selections +apt-get -y install iptables-persistent -y > /dev/null + +log 'Configuring iptables...' +# Only allow local access to port 25: +iptables -A INPUT -i lo -p tcp --dport 25 -j ACCEPT +iptables -A INPUT -p tcp --dport 25 -j REJECT + +# Save iptables rules so they persist across restarts. At the time of this +# writing, there is no open IPv6 port, so enough to save rules.v*4*. +iptables-save > /etc/iptables/rules.v4 + +log 'Installing git...' +apt-get install git-core -y > /dev/null + +log 'Creating application users...' +groupadd --system django +useradd --system --gid django --shell /bin/bash django +mkdir -p /home/django +chown django /home/django +chmod 700 /home/django + +log 'Setting up SSH keys. This is required so we can clone the Git repository without having to supply a password...' +mkdir -p .ssh +echo "$GIT_REPO_PUBKEY" > .ssh/id_rsa.pub +chmod 644 .ssh/id_rsa.pub +echo -e "$GIT_REPO_PRIVKEY" > .ssh/id_rsa +chmod 600 .ssh/id_rsa + +log 'Adding git repository server to known hosts...' +ssh-keyscan -H ${GIT_SERVER} >> .ssh/known_hosts 2>/dev/null; chmod 600 .ssh/known_hosts + +log 'Cloning the repository...' +git clone -q -b ${GIT_REPO_BRANCH} git@${GIT_SERVER}:${GIT_REPO_NAME}.git /srv + +log 'Setting up bash profiles...' +ln -sf /srv/conf/.bash_profile . +ln -sf /srv/conf/.bash_profile /home/django +PATH='$PATH' /usr/bin/envsubst < /srv/conf/django.bashrc > /home/django/.bashrc +chown django:django /home/django/.bashrc + +log 'Installing Postfix...' +debconf-set-selections <<< "postfix postfix/mailname string $HOST_NAME" +debconf-set-selections <<< "postfix postfix/main_mailer_type string 'Internet Site'" +apt-get install postfix mailutils libsasl2-2 ca-certificates libsasl2-modules -y > /dev/null + +log 'Configuring Postfix...' +echo "$HOST_NAME" > /etc/mailname +chown postfix /etc/mailname +envsubst '$SMTP_HOST $HOST_NAME' < /srv/conf/postfix/main.cf > /etc/postfix/main.cf +envsubst < /srv/conf/postfix/sasl_passwd > /etc/postfix/sasl_passwd +postmap /etc/postfix/sasl_passwd +chmod 400 /etc/postfix/sasl_passwd +chown postfix /etc/postfix/sasl_passwd +envsubst < /srv/conf/postfix/generic > /etc/postfix/generic +postmap /etc/postfix/generic +chown postfix /etc/postfix/generic +/etc/init.d/postfix reload > /dev/null + +log 'Installing OS dependencies for our Python environment...' +apt-get install python3-venv -y > /dev/null + +log 'Creating virtual environment...' +python3 -m venv /srv/venv > /dev/null + +log 'Installing Python dependencies...' +# N.B.: pip should be on PATH from sourcing .bashrc at the top. +pip install -Ur /srv/requirements/base-full.txt > /dev/null + +log 'Migrating database...' +su -c '/srv/manage.sh migrate | grep -v "INFO\|DEBUG"' - django + +log 'Collecting static files...' +su -c '/srv/manage.sh collectstatic --noinput | grep -v "INFO\|DEBUG"' - django + +log 'Installing Supervisor...' +apt-get install supervisor -y > /dev/null + +log 'Configuring Supervisor...' +ln -s /srv/conf/supervisor/*.conf /etc/supervisor/conf.d + +log 'Starting services...' +supervisorctl reread > /dev/null +supervisorctl update > /dev/null + +log 'Installing Nginx...' +apt-get install nginx -y > /dev/null + +log 'Creating Nginx includes directory...' +mkdir /etc/nginx/includes + +log 'Creating Nginx config...' +envsubst < /srv/conf/nginx/server-name > /etc/nginx/includes/server-name +touch /etc/nginx/includes/ssl-cert + +log 'Applying Nginx config...' +ln -s /srv/conf/nginx/nginx /etc/nginx/sites-available/wpr +ln -s /etc/nginx/sites-available/wpr /etc/nginx/sites-enabled/wpr +rm /etc/nginx/sites-enabled/default + +log 'Starting Nginx...' +service nginx restart + +log 'Generating SSL certificates...' +apt-get install certbot python3-certbot-nginx -y > /dev/null +certbot register --quiet --email $ADMIN_EMAIL --agree-tos +certbot certonly --nginx --quiet --cert-name main -d $HOST_NAME +ln -sf /srv/conf/nginx/ssl-cert /etc/nginx/includes/ssl-cert +service nginx restart + +log "The server is now serving requests at $HOST_NAME!" + +# Now do less important things that are not required for serving requests: + +log 'Setting up logrotate...' +ln -s /srv/conf/logrotate /etc/logrotate.d/wpr + +log 'Setting up crontab...' +ln -s /srv/bin/cronic /usr/bin/cronic +sed -i "s/\$ADMIN_EMAIL/$ADMIN_EMAIL/g" /srv/conf/crontab +crontab /srv/conf/crontab + +log 'Setting up automatic updates...' +apt-get install unattended-upgrades -y > /dev/null +printf 'APT::Periodic::Update-Package-Lists "1";\nAPT::Periodic::Unattended-Upgrade "1";\n' > /etc/apt/apt.conf.d/20auto-upgrades + +log 'Done.' \ No newline at end of file diff --git a/wpr/settings.py b/wpr/settings.py index eba5404..6a11fcf 100644 --- a/wpr/settings.py +++ b/wpr/settings.py @@ -1,34 +1,21 @@ -""" -Django settings for wpr project. - -Generated by 'django-admin startproject' using Django 3.2.10. - -For more information on this file, see -https://docs.djangoproject.com/en/3.2/topics/settings/ - -For the full list of settings and their values, see -https://docs.djangoproject.com/en/3.2/ref/settings/ -""" - +import os from pathlib import Path -# Build paths inside the project like this: BASE_DIR / 'subdir'. +from wpr.util import get_bool_from_env + BASE_DIR = Path(__file__).resolve().parent.parent - -# Quick-start development settings - unsuitable for production -# See https://docs.djangoproject.com/en/3.2/howto/deployment/checklist/ - # SECURITY WARNING: keep the secret key used in production secret! -SECRET_KEY = 'django-insecure-n$m*a%3#nqpl5=(sp7v+!6r98mld#o4*(24!dawoma2i5(%lh=' +SECRET_KEY = os.getenv('SECRET_KEY') or \ + 'django-insecure-n$m*a%3#nqpl5=(sp7v+!6r98mld#o4*(24!dawoma2i5(%lh' # SECURITY WARNING: don't run with debug turned on in production! -DEBUG = True +DEBUG = get_bool_from_env('DEBUG', True) ALLOWED_HOSTS = [] - -# Application definition +if os.getenv('HOST_NAME'): + ALLOWED_HOSTS.append(os.getenv('HOST_NAME')) INSTALLED_APPS = [ 'suit', @@ -74,10 +61,6 @@ TEMPLATES = [ WSGI_APPLICATION = 'wpr.wsgi.application' - -# Database -# https://docs.djangoproject.com/en/3.2/ref/settings/#databases - DATABASES = { 'default': { 'ENGINE': 'django.db.backends.sqlite3', @@ -85,10 +68,6 @@ DATABASES = { } } - -# Password validation -# https://docs.djangoproject.com/en/3.2/ref/settings/#auth-password-validators - AUTH_PASSWORD_VALIDATORS = [ { 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', @@ -104,10 +83,6 @@ AUTH_PASSWORD_VALIDATORS = [ }, ] - -# Internationalization -# https://docs.djangoproject.com/en/3.2/topics/i18n/ - LANGUAGE_CODE = 'en-us' TIME_ZONE = 'UTC' @@ -118,21 +93,13 @@ USE_L10N = True USE_TZ = True - -# Static files (CSS, JavaScript, Images) -# https://docs.djangoproject.com/en/3.2/howto/static-files/ - STATIC_URL = '/static/' MEDIA_ROOT = BASE_DIR / 'media' MEDIA_URL = 'media/' -# Default primary key field type -# https://docs.djangoproject.com/en/3.2/ref/settings/#default-auto-field - DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField' -# Django Suit config SUIT_CONFIG = { 'ADMIN_NAME': 'winget Admin', 'MENU': ( @@ -141,4 +108,4 @@ SUIT_CONFIG = { 'models': ('auth.user', 'tenants.tenant')} ), 'CONFIRM_UNSAVED_CHANGES': False -} \ No newline at end of file +} diff --git a/wpr/util.py b/wpr/util.py new file mode 100644 index 0000000..07866f1 --- /dev/null +++ b/wpr/util.py @@ -0,0 +1,17 @@ +import os + +from django.core.exceptions import ImproperlyConfigured + + +def get_bool_from_env(name, default): + result_str = os.getenv(name) + if not result_str: + return default + if result_str == 'True': + return True + elif result_str == 'False': + return False + raise ImproperlyConfigured( + 'Environment variable %r has invalid value. Must be True or False.' + % name + )