| 1 | #!/usr/bin/env bash |
| 2 | # Deploy a build of Steid to a running instance. |
| 3 | # |
| 4 | # ./deploy.sh # build, upload, upgrade, verify |
| 5 | # ./deploy.sh --skip-build # use whatever is already in dist/ |
| 6 | # ./deploy.sh --host steid --domain jpgill.dev |
| 7 | # |
| 8 | # This is the operator's script, not the stranger's — `install.sh` is what someone |
| 9 | # else runs to install Steid for the first time. This one assumes an instance |
| 10 | # already exists and you want it running a newer build. |
| 11 | # |
| 12 | # WHY IT DOES MORE THAN scp-AND-RUN. An upgrade replaces the binary of a live |
| 13 | # service. Three things can go wrong and each is silent: the new build might not |
| 14 | # boot, the state might not survive, and a mistake leaves nothing to go back to. |
| 15 | # So it snapshots the database and the current binary first, upgrades, then |
| 16 | # *checks the site actually works* — and restores if it does not. |
| 17 | |
| 18 | set -euo pipefail |
| 19 | |
| 20 | HOST="steid" # an ssh alias or user@address |
| 21 | DOMAIN="jpgill.dev" |
| 22 | TARGET="x86_64-unknown-linux-gnu" |
| 23 | SKIP_BUILD=0 |
| 24 | VERSION="" |
| 25 | |
| 26 | die() { echo "deploy.sh: $*" >&2; exit 1; } |
| 27 | say() { echo "==> $*"; } |
| 28 | |
| 29 | while [ $# -gt 0 ]; do |
| 30 | case "$1" in |
| 31 | --host) HOST="${2:?--host needs a value}"; shift 2 ;; |
| 32 | --domain) DOMAIN="${2:?--domain needs a value}"; shift 2 ;; |
| 33 | --target) TARGET="${2:?--target needs a triple}"; shift 2 ;; |
| 34 | --version) VERSION="${2:?--version needs a value}"; shift 2 ;; |
| 35 | --skip-build) SKIP_BUILD=1; shift ;; |
| 36 | -h|--help) sed -n '2,17p' "$0" | sed 's/^#\{1,2\} \{0,1\}//'; exit 0 ;; |
| 37 | *) die "unknown argument: $1" ;; |
| 38 | esac |
| 39 | done |
| 40 | |
| 41 | cd "$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd)" |
| 42 | |
| 43 | [ -n "$VERSION" ] || VERSION="$(grep -m1 '^version *= *"' Cargo.toml | sed 's/.*"\(.*\)".*/\1/')" |
| 44 | NAME="steid-${VERSION}-${TARGET}" |
| 45 | TARBALL="dist/${NAME}.tar.gz" |
| 46 | |
| 47 | # Refuse to deploy a working tree that is not what is committed. The instance |
| 48 | # should always be running a commit you can point at, or debugging it later means |
| 49 | # guessing what was on the machine that built it. |
| 50 | if [ -n "$(git status --porcelain)" ]; then |
| 51 | die "working tree is dirty — commit first, so the deployed build has a commit to name" |
| 52 | fi |
| 53 | COMMIT="$(git rev-parse --short HEAD)" |
| 54 | |
| 55 | if [ "$SKIP_BUILD" = 0 ]; then |
| 56 | say "building ${NAME} from ${COMMIT}" |
| 57 | ./release.sh --target "$TARGET" --version "$VERSION" >/dev/null |
| 58 | else |
| 59 | say "skipping build (--skip-build)" |
| 60 | fi |
| 61 | [ -f "$TARBALL" ] || die "no ${TARBALL} — build it, or drop --skip-build" |
| 62 | |
| 63 | say "checking the instance is reachable" |
| 64 | ssh -o BatchMode=yes "$HOST" true || die "cannot ssh to ${HOST}" |
| 65 | |
| 66 | # --- snapshot, so there is something to go back to -------------------------- |
| 67 | |
| 68 | STAMP="$(date +%Y%m%d-%H%M%S)" |
| 69 | say "snapshotting state and the current binary (${STAMP})" |
| 70 | ssh -o BatchMode=yes "$HOST" " |
| 71 | set -e |
| 72 | sudo mkdir -p /var/backups/steid |
| 73 | sudo tar czf /var/backups/steid/state-${STAMP}.tar.gz -C /var/lib steid |
| 74 | sudo tar czf /var/backups/steid/binary-${STAMP}.tar.gz -C /opt steid |
| 75 | sudo find /var/backups/steid -name '*.tar.gz' -mtime +14 -delete |
| 76 | " |
| 77 | |
| 78 | # --- upgrade ---------------------------------------------------------------- |
| 79 | |
| 80 | say "uploading" |
| 81 | scp -q -o BatchMode=yes "$TARBALL" "${TARBALL}.sha256" install.sh "${HOST}:" |
| 82 | |
| 83 | say "upgrading" |
| 84 | ssh -o BatchMode=yes "$HOST" "sudo sh ./install.sh --domain '${DOMAIN}' --tarball './${NAME}.tar.gz'" \ |
| 85 | | grep -E '^==>|is running' || true |
| 86 | |
| 87 | # --- verify, and roll back if it is not actually serving -------------------- |
| 88 | |
| 89 | say "verifying https://${DOMAIN}" |
| 90 | OK=0 |
| 91 | for _ in $(seq 1 15); do |
| 92 | sleep 2 |
| 93 | if [ "$(curl -s -o /dev/null -m 10 -w '%{http_code}' "https://${DOMAIN}/healthz")" = "200" ]; then |
| 94 | OK=1; break |
| 95 | fi |
| 96 | done |
| 97 | |
| 98 | if [ "$OK" = 1 ]; then |
| 99 | say "healthy — ${VERSION} (${COMMIT}) is live at https://${DOMAIN}" |
| 100 | ssh -o BatchMode=yes "$HOST" "systemctl is-active steid caddy | paste -sd' ' -" | sed 's/^/ steid caddy: /' |
| 101 | else |
| 102 | echo "deploy.sh: the new build is NOT serving. Rolling back." >&2 |
| 103 | ssh -o BatchMode=yes "$HOST" " |
| 104 | set -e |
| 105 | sudo systemctl stop steid |
| 106 | sudo rm -rf /opt/steid |
| 107 | sudo tar xzf /var/backups/steid/binary-${STAMP}.tar.gz -C /opt |
| 108 | sudo systemctl start steid |
| 109 | " |
| 110 | sleep 4 |
| 111 | ROLLED="$(curl -s -o /dev/null -m 10 -w '%{http_code}' "https://${DOMAIN}/healthz")" |
| 112 | echo "deploy.sh: rolled back to the previous binary (healthz ${ROLLED})." >&2 |
| 113 | echo "deploy.sh: state was NOT restored — the database is untouched by an upgrade," >&2 |
| 114 | echo " and /var/backups/steid/state-${STAMP}.tar.gz holds it if you need it." >&2 |
| 115 | exit 1 |
| 116 | fi |