steid

steid/deploy.sh

2b64cf6feat: deploy.sh upgrades a live instance, and rolls back if it fails22h	1	#!/usr/bin/env bash
	2	# Deploy a build of Steid to a running instance.
	3	#
	4	# ./deploy.sh # build, upload, upgrade, verify
	5	# ./deploy.sh --skip-build # use whatever is already in dist/
	6	# ./deploy.sh --host steid --domain jpgill.dev
	7	#
	8	# This is the operator's script, not the stranger's — `install.sh` is what someone
	9	# else runs to install Steid for the first time. This one assumes an instance
	10	# already exists and you want it running a newer build.
	11	#
	12	# WHY IT DOES MORE THAN scp-AND-RUN. An upgrade replaces the binary of a live
	13	# service. Three things can go wrong and each is silent: the new build might not
	14	# boot, the state might not survive, and a mistake leaves nothing to go back to.
	15	# So it snapshots the database and the current binary first, upgrades, then
	16	# checks the site actually works — and restores if it does not.
	17
	18	set -euo pipefail
	19
	20	HOST="steid" # an ssh alias or user@address
	21	DOMAIN="jpgill.dev"
	22	TARGET="x86_64-unknown-linux-gnu"
	23	SKIP_BUILD=0
	24	VERSION=""
	25
	26	die() { echo "deploy.sh: $*" >&2; exit 1; }
	27	say() { echo "==> $*"; }
	28
	29	while [ $# -gt 0 ]; do
	30	case "$1" in
	31	--host) HOST="${2:?--host needs a value}"; shift 2 ;;
	32	--domain) DOMAIN="${2:?--domain needs a value}"; shift 2 ;;
	33	--target) TARGET="${2:?--target needs a triple}"; shift 2 ;;
	34	--version) VERSION="${2:?--version needs a value}"; shift 2 ;;
	35	--skip-build) SKIP_BUILD=1; shift ;;
	36	-h\|--help) sed -n '2,17p' "$0" \| sed 's/^#\{1,2\} \{0,1\}//'; exit 0 ;;
	37	*) die "unknown argument: $1" ;;
	38	esac
	39	done
	40
	41	cd "$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd)"
	42
	43	[ -n "$VERSION" ] \|\| VERSION="$(grep -m1 '^version = "' Cargo.toml \| sed 's/."$.$".*/\1/')"
	44	NAME="steid-${VERSION}-${TARGET}"
	45	TARBALL="dist/${NAME}.tar.gz"
	46
	47	# Refuse to deploy a working tree that is not what is committed. The instance
	48	# should always be running a commit you can point at, or debugging it later means
	49	# guessing what was on the machine that built it.
	50	if [ -n "$(git status --porcelain)" ]; then
	51	die "working tree is dirty — commit first, so the deployed build has a commit to name"
	52	fi
	53	COMMIT="$(git rev-parse --short HEAD)"
	54
	55	if [ "$SKIP_BUILD" = 0 ]; then
	56	say "building ${NAME} from ${COMMIT}"
	57	./release.sh --target "$TARGET" --version "$VERSION" >/dev/null
	58	else
	59	say "skipping build (--skip-build)"
	60	fi
	61	[ -f "$TARBALL" ] \|\| die "no ${TARBALL} — build it, or drop --skip-build"
	62
	63	say "checking the instance is reachable"
	64	ssh -o BatchMode=yes "$HOST" true \|\| die "cannot ssh to ${HOST}"
	65
	66	# --- snapshot, so there is something to go back to --------------------------
	67
	68	STAMP="$(date +%Y%m%d-%H%M%S)"
	69	say "snapshotting state and the current binary (${STAMP})"
	70	ssh -o BatchMode=yes "$HOST" "
	71	set -e
	72	sudo mkdir -p /var/backups/steid
	73	sudo tar czf /var/backups/steid/state-${STAMP}.tar.gz -C /var/lib steid
	74	sudo tar czf /var/backups/steid/binary-${STAMP}.tar.gz -C /opt steid
	75	sudo find /var/backups/steid -name '*.tar.gz' -mtime +14 -delete
	76	"
	77
	78	# --- upgrade ----------------------------------------------------------------
	79
	80	say "uploading"
	81	scp -q -o BatchMode=yes "$TARBALL" "${TARBALL}.sha256" install.sh "${HOST}:"
	82
	83	say "upgrading"
	84	ssh -o BatchMode=yes "$HOST" "sudo sh ./install.sh --domain '${DOMAIN}' --tarball './${NAME}.tar.gz'" \
	85	\| grep -E '^==>\|is running' \|\| true
	86
	87	# --- verify, and roll back if it is not actually serving --------------------
	88
	89	say "verifying https://${DOMAIN}"
	90	OK=0
	91	for _ in $(seq 1 15); do
	92	sleep 2
	93	if [ "$(curl -s -o /dev/null -m 10 -w '%{http_code}' "https://${DOMAIN}/healthz")" = "200" ]; then
	94	OK=1; break
	95	fi
	96	done
	97
	98	if [ "$OK" = 1 ]; then
	99	say "healthy — ${VERSION} (${COMMIT}) is live at https://${DOMAIN}"
	100	ssh -o BatchMode=yes "$HOST" "systemctl is-active steid caddy \| paste -sd' ' -" \| sed 's/^/ steid caddy: /'
	101	else
	102	echo "deploy.sh: the new build is NOT serving. Rolling back." >&2
	103	ssh -o BatchMode=yes "$HOST" "
	104	set -e
	105	sudo systemctl stop steid
	106	sudo rm -rf /opt/steid
	107	sudo tar xzf /var/backups/steid/binary-${STAMP}.tar.gz -C /opt
	108	sudo systemctl start steid
	109	"
	110	sleep 4
	111	ROLLED="$(curl -s -o /dev/null -m 10 -w '%{http_code}' "https://${DOMAIN}/healthz")"
	112	echo "deploy.sh: rolled back to the previous binary (healthz ${ROLLED})." >&2
	113	echo "deploy.sh: state was NOT restored — the database is untouched by an upgrade," >&2
	114	echo " and /var/backups/steid/state-${STAMP}.tar.gz holds it if you need it." >&2
	115	exit 1
	116	fi