From bcccba92f7687946c97b9fa3573ca031c7897bd1 Mon Sep 17 00:00:00 2001
From: Michael Czechowski <mail@dailysh.it>
Date: Tue, 28 Apr 2026 22:49:40 +0200
Subject: [PATCH] Add deploy workflow and backup tooling

CI deployment (.gitea/workflows/deploy.yml):
- Two jobs (build, deploy) gated on the repo variable
  DEPLOY_ENABLED=true so the workflow exists but does nothing
  until secrets and host are configured.
- Build pushes two image tags per run: rolling :main + the short
  SHA on main, or vX.Y.Z + :latest on tag pushes. Immutable per
  commit/tag tags make rollback trivial.
- Deploy SSHes to DEPLOY_HOST, runs docker compose pull && up -d
  in DEPLOY_PATH, then polls HEALTH_URL for up to a minute. A
  failed health check fails the workflow, which is the alert.
- Required secrets and the rollback procedure are documented in
  docs/operations.md.

Backup tooling (scripts/):
- backup.sh: SQLite online .backup snapshot + tarball of the
  per-tenant data dir + info.txt header, all wrapped into a
  single librenotes-YYYYMMDD-HHMMSS.tar.gz. Optional BACKUP_REMOTE
  triggers an rclone copy for off-site storage.
- backup-prune.sh: enforces retention "30 daily + 12 monthly".
  Sorts archives by filename (date is in the name so lex order
  matches chronological) and keeps the newest 30 plus the newest
  archive for each of the most recent 12 months.
- backup-restore-test.sh: extracts the most recent archive into
  a tmpdir, runs sqlite3 .schema (proves DB readability), and
  asserts the notes tar has at least one entry. Failure is the
  alert. Wired into a separate weekly timer.
- librenotes-backup.{service,timer}: systemd units for the daily
  03:17 UTC run with 5min jitter; ProtectSystem=strict, only
  /var/backups/librenotes is writable.
- librenotes-backup-verify.{service,timer}: weekly Monday
  04:00 UTC restore test.

Closes #26 and #27.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
---
 .gitea/workflows/deploy.yml              |  99 +++++++++++++++++++
 docs/operations.md                       | 117 +++++++++++++++++++++++
 scripts/backup-prune.sh                  |  57 +++++++++++
 scripts/backup-restore-test.sh           |  51 ++++++++++
 scripts/backup.sh                        |  60 ++++++++++++
 scripts/librenotes-backup-verify.service |  12 +++
 scripts/librenotes-backup-verify.timer   |  10 ++
 scripts/librenotes-backup.service        |  19 ++++
 scripts/librenotes-backup.timer          |  13 +++
 9 files changed, 438 insertions(+)
 create mode 100644 .gitea/workflows/deploy.yml
 create mode 100644 docs/operations.md
 create mode 100755 scripts/backup-prune.sh
 create mode 100755 scripts/backup-restore-test.sh
 create mode 100755 scripts/backup.sh
 create mode 100644 scripts/librenotes-backup-verify.service
 create mode 100644 scripts/librenotes-backup-verify.timer
 create mode 100644 scripts/librenotes-backup.service
 create mode 100644 scripts/librenotes-backup.timer

diff --git a/.gitea/workflows/deploy.yml b/.gitea/workflows/deploy.yml
new file mode 100644
index 0000000..c2655de
--- /dev/null
+++ b/.gitea/workflows/deploy.yml
@@ -0,0 +1,99 @@
+name: Deploy
+
+on:
+  push:
+    branches: [main]
+    tags: ["v*"]
+
+# Required repository secrets:
+#   REGISTRY            registry hostname, e.g. registry.librete.ch
+#   REGISTRY_USER       robot account
+#   REGISTRY_PASS       robot token
+#   DEPLOY_HOST         deployment SSH target, e.g. root@librenot.es
+#   DEPLOY_KEY          private SSH key (PEM, no passphrase)
+#   DEPLOY_PATH         remote directory containing the compose stack
+#   HEALTH_URL          public URL to verify post-deploy, e.g.
+#                       https://librenot.es/healthz
+#
+# Tag pushes deploy the tag (vX.Y.Z); main-branch pushes deploy
+# the rolling :main image. Set image to immutable tag so rollback
+# is just `docker compose -f ... up -d` with the previous tag.
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    if: ${{ vars.DEPLOY_ENABLED == 'true' }}
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: docker/setup-buildx-action@v3
+
+      - name: Log in to registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ secrets.REGISTRY }}
+          username: ${{ secrets.REGISTRY_USER }}
+          password: ${{ secrets.REGISTRY_PASS }}
+
+      - name: Compute tags
+        id: tags
+        run: |
+          BASE="${{ secrets.REGISTRY }}/librenotes"
+          if [[ "${GITHUB_REF}" == refs/tags/* ]]; then
+            TAG="${GITHUB_REF##refs/tags/}"
+            echo "tags=${BASE}:${TAG},${BASE}:latest" >> "$GITHUB_OUTPUT"
+            echo "version=${TAG}" >> "$GITHUB_OUTPUT"
+          else
+            echo "tags=${BASE}:main,${BASE}:${GITHUB_SHA::7}" >> "$GITHUB_OUTPUT"
+            echo "version=${GITHUB_SHA::7}" >> "$GITHUB_OUTPUT"
+          fi
+
+      - uses: docker/build-push-action@v6
+        with:
+          context: .
+          push: true
+          tags: ${{ steps.tags.outputs.tags }}
+          build-args: |
+            VERSION=${{ steps.tags.outputs.version }}
+            BUILDTIME=${{ github.event.head_commit.timestamp }}
+
+  deploy:
+    runs-on: ubuntu-latest
+    needs: build
+    timeout-minutes: 10
+    if: ${{ vars.DEPLOY_ENABLED == 'true' }}
+    steps:
+      - name: Configure SSH
+        run: |
+          mkdir -p ~/.ssh
+          echo "${{ secrets.DEPLOY_KEY }}" > ~/.ssh/id_deploy
+          chmod 600 ~/.ssh/id_deploy
+          ssh-keyscan -H "${{ secrets.DEPLOY_HOST#*@ }}" >> ~/.ssh/known_hosts || true
+
+      - name: Pull and restart on deploy host
+        env:
+          DEPLOY_HOST: ${{ secrets.DEPLOY_HOST }}
+          DEPLOY_PATH: ${{ secrets.DEPLOY_PATH }}
+        run: |
+          ssh -i ~/.ssh/id_deploy "$DEPLOY_HOST" \
+            "cd $DEPLOY_PATH && \
+             docker compose -f docker-compose.yml -f docker-compose.prod.yml pull && \
+             docker compose -f docker-compose.yml -f docker-compose.prod.yml up -d --remove-orphans"
+
+      - name: Verify health
+        env:
+          HEALTH_URL: ${{ secrets.HEALTH_URL }}
+        run: |
+          # Give the new container ~30s to come up, then poll for
+          # a 200 from /healthz. Failure aborts the workflow which
+          # is the alert.
+          for i in $(seq 1 12); do
+            if curl -fsS "$HEALTH_URL" >/dev/null; then
+              echo "deploy verified"
+              exit 0
+            fi
+            sleep 5
+          done
+          echo "deploy verification failed"
+          exit 1
diff --git a/docs/operations.md b/docs/operations.md
new file mode 100644
index 0000000..996a4a3
--- /dev/null
+++ b/docs/operations.md
@@ -0,0 +1,117 @@
+# Operations
+
+This document covers deployment, backups, and rollback for the
+librenot.es production environment. For developer / contributor
+docs see the top-level [README](../README.md).
+
+## Deployment
+
+The repository ships a CI workflow at `.gitea/workflows/deploy.yml`
+that builds and pushes a Docker image on every push to `main` and
+on every `vX.Y.Z` tag, then SSHes to the deployment host and runs
+`docker compose pull && up -d`.
+
+The workflow is gated on the repository variable
+`DEPLOY_ENABLED=true`. Deployment is opt-in; flipping the variable
+disables the workflow without removing the file.
+
+### Required secrets
+
+The workflow expects these secrets and variables on the repo:
+
+| Name | Type | Purpose |
+| ---- | ---- | ------- |
+| `REGISTRY` | secret | hostname of the OCI registry |
+| `REGISTRY_USER` | secret | robot account |
+| `REGISTRY_PASS` | secret | robot token |
+| `DEPLOY_HOST` | secret | `user@host` SSH target |
+| `DEPLOY_KEY` | secret | passphrase-less private key |
+| `DEPLOY_PATH` | secret | absolute path on host with `docker-compose.*.yml` |
+| `HEALTH_URL` | secret | e.g. `https://librenot.es/healthz` |
+| `DEPLOY_ENABLED` | variable | `true` to enable the workflow |
+
+### Production compose stack
+
+On the deployment host, place `docker-compose.yml` and
+`docker-compose.prod.yml` from this repo at `$DEPLOY_PATH`,
+together with an `.env` file containing the runtime configuration
+(JWT secret, SMTP credentials, public base URL, image tag).
+
+Bring it up with:
+
+```sh
+docker compose -f docker-compose.yml -f docker-compose.prod.yml up -d
+```
+
+### Rollback
+
+Image tags are immutable per commit / version. To roll back, set
+`LIBRENOTES_IMAGE` to the previous tag in `.env` and run the same
+`up -d` command. The deployment workflow does not auto-rollback
+on health-check failure — failed health alerts the operator via
+the workflow itself, who can then redeploy the prior tag manually.
+
+```sh
+# Example: roll back to v0.1.2
+sed -i 's/^LIBRENOTES_IMAGE=.*/LIBRENOTES_IMAGE=registry.librete.ch\/librenotes:v0.1.2/' .env
+docker compose -f docker-compose.yml -f docker-compose.prod.yml up -d
+```
+
+## Backups
+
+`scripts/backup.sh` is a self-contained backup driver suitable for
+running from cron, a systemd timer, or the supplied
+`scripts/librenotes-backup.{service,timer}` units.
+
+Required tools on the backup host: `sqlite3`, `tar`, `gzip`, and
+optionally `rclone` for off-site copy. On Debian/Ubuntu:
+`apt install sqlite3 rclone`.
+
+### What is backed up
+
+- The SQLite database (`$LIBRENOTES_DB`) via the SQLite
+  `.backup` command, which produces a consistent online snapshot
+  without needing to stop the application.
+- The per-tenant note directory (`$LIBRENOTES_DATA_DIR`) as a
+  gzipped tar.
+- An `info.txt` in each archive recording the backup timestamp,
+  hostname, and version.
+
+### Off-site copy
+
+Set `BACKUP_REMOTE` to an `rclone` destination (e.g.
+`s3:librenotes-backups`). When set, the script invokes
+`rclone copy` to upload the archive after creation. Without it,
+backups stay on the local disk only.
+
+### Retention
+
+`scripts/backup-prune.sh` enforces the policy "keep 30 daily
++ 12 monthly". Run it from the same timer as the backup. Files
+keep the form `librenotes-YYYYMMDD-HHMMSS.tar.gz` so the prune
+script can sort and select by name alone.
+
+### Restore test
+
+`scripts/backup-restore-test.sh` picks the most recent archive,
+extracts it into a scratch directory, runs `sqlite3 .schema` on
+the database to confirm it is readable, and verifies that the
+note tar lists at least one entry. It is wired into a separate
+weekly timer so a silent backup-corruption regression cannot hide
+indefinitely.
+
+### Systemd
+
+Drop `scripts/librenotes-backup.{service,timer}` into
+`/etc/systemd/system/`, then:
+
+```sh
+systemctl daemon-reload
+systemctl enable --now librenotes-backup.timer
+```
+
+Confirm with:
+
+```sh
+systemctl list-timers librenotes-backup.timer
+```
diff --git a/scripts/backup-prune.sh b/scripts/backup-prune.sh
new file mode 100755
index 0000000..0c9a222
--- /dev/null
+++ b/scripts/backup-prune.sh
@@ -0,0 +1,57 @@
+#!/usr/bin/env bash
+# backup-prune.sh — enforce retention "keep 30 daily + 12 monthly".
+#
+# Walks $BACKUP_DIR for librenotes-YYYYMMDD-HHMMSS.tar.gz and
+# deletes archives outside the retention policy.
+#   - keep the most recent N daily archives (default 30)
+#   - additionally keep the most recent archive of each of the
+#     last M distinct months (default 12)
+# Anything else is removed.
+
+set -euo pipefail
+
+BACKUP_DIR="${BACKUP_DIR:-/var/backups/librenotes}"
+DAILY_KEEP="${DAILY_KEEP:-30}"
+MONTHLY_KEEP="${MONTHLY_KEEP:-12}"
+
+if [ ! -d "$BACKUP_DIR" ]; then
+  echo "backup dir $BACKUP_DIR does not exist; nothing to prune"
+  exit 0
+fi
+
+cd "$BACKUP_DIR"
+
+# Sort archives by name (date is in the filename, lexicographic
+# order == chronological order).
+mapfile -t archives < <(ls -1 librenotes-*.tar.gz 2>/dev/null | sort)
+
+declare -A keep
+# Keep the newest DAILY_KEEP outright.
+for a in "${archives[@]: -$DAILY_KEEP}"; do
+  keep["$a"]=1
+done
+
+# Walk archives newest-first, recording one per month until we
+# have MONTHLY_KEEP distinct months.
+declare -A month_seen
+months_kept=0
+for ((i=${#archives[@]}-1; i>=0; i--)); do
+  a="${archives[$i]}"
+  # filename: librenotes-YYYYMMDD-HHMMSS.tar.gz -> YYYYMM
+  ym="${a:11:6}"
+  if [ -z "${month_seen[$ym]:-}" ] && [ "$months_kept" -lt "$MONTHLY_KEEP" ]; then
+    keep["$a"]=1
+    month_seen[$ym]=1
+    months_kept=$((months_kept + 1))
+  fi
+done
+
+removed=0
+for a in "${archives[@]}"; do
+  if [ -z "${keep[$a]:-}" ]; then
+    rm -f "$a"
+    removed=$((removed + 1))
+  fi
+done
+
+echo "kept ${#keep[@]} archives, removed $removed"
diff --git a/scripts/backup-restore-test.sh b/scripts/backup-restore-test.sh
new file mode 100755
index 0000000..b460ec8
--- /dev/null
+++ b/scripts/backup-restore-test.sh
@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+# backup-restore-test.sh — verify the most recent backup is usable.
+#
+# Failure of this script is an alert. It does NOT touch the live
+# database or data dir.
+#
+# Steps:
+#   1. Pick the newest archive in $BACKUP_DIR.
+#   2. Extract into a scratch tmpdir.
+#   3. Run sqlite3 ".schema" against the snapshotted DB; non-empty
+#      output means the file is readable as SQLite.
+#   4. Run tar -tzf on notes.tar.gz; we expect at least one entry.
+#   5. Print the info.txt header so logs include backup metadata.
+
+set -euo pipefail
+
+BACKUP_DIR="${BACKUP_DIR:-/var/backups/librenotes}"
+
+archive="$(ls -1t "$BACKUP_DIR"/librenotes-*.tar.gz 2>/dev/null | head -n 1 || true)"
+if [ -z "$archive" ]; then
+  echo "no archives in $BACKUP_DIR" >&2
+  exit 1
+fi
+
+work="$(mktemp -d)"
+trap 'rm -rf "$work"' EXIT
+
+tar -C "$work" -xzf "$archive"
+
+if [ ! -f "$work/librenotes.db" ]; then
+  echo "archive missing librenotes.db: $archive" >&2
+  exit 1
+fi
+schema="$(sqlite3 "$work/librenotes.db" ".schema" || true)"
+if [ -z "$schema" ]; then
+  echo "sqlite .schema returned empty for $archive" >&2
+  exit 1
+fi
+
+if [ ! -f "$work/notes.tar.gz" ]; then
+  echo "archive missing notes.tar.gz: $archive" >&2
+  exit 1
+fi
+n="$(tar -tzf "$work/notes.tar.gz" | wc -l)"
+if [ "$n" -lt 1 ]; then
+  echo "notes.tar.gz contains no entries: $archive" >&2
+  exit 1
+fi
+
+echo "verified $archive"
+[ -f "$work/info.txt" ] && cat "$work/info.txt"
diff --git a/scripts/backup.sh b/scripts/backup.sh
new file mode 100755
index 0000000..09bd70e
--- /dev/null
+++ b/scripts/backup.sh
@@ -0,0 +1,60 @@
+#!/usr/bin/env bash
+# backup.sh — daily backup of librenotes state.
+#
+# Produces $BACKUP_DIR/librenotes-YYYYMMDD-HHMMSS.tar.gz containing:
+#   - librenotes.db        (consistent SQLite .backup snapshot)
+#   - notes.tar.gz         (per-tenant note files)
+#   - info.txt             (timestamp, hostname, version)
+#
+# Required env:
+#   LIBRENOTES_DB        path to the SQLite database
+#   LIBRENOTES_DATA_DIR  path to the per-tenant note directory
+#
+# Optional env:
+#   BACKUP_DIR           where to write archives (default /var/backups/librenotes)
+#   BACKUP_REMOTE        rclone target for off-site copy (e.g. s3:bucket/path)
+#   BACKUP_VERSION       version string written into info.txt
+#
+# The script is intentionally a single self-contained file so it
+# can run on a minimal host with only sqlite3, tar, gzip, and
+# (optionally) rclone.
+
+set -euo pipefail
+
+: "${LIBRENOTES_DB:?LIBRENOTES_DB is required}"
+: "${LIBRENOTES_DATA_DIR:?LIBRENOTES_DATA_DIR is required}"
+BACKUP_DIR="${BACKUP_DIR:-/var/backups/librenotes}"
+BACKUP_VERSION="${BACKUP_VERSION:-unknown}"
+
+mkdir -p "$BACKUP_DIR"
+
+ts="$(date -u +%Y%m%d-%H%M%S)"
+work="$(mktemp -d)"
+trap 'rm -rf "$work"' EXIT
+
+# Online SQLite snapshot. .backup is atomic from the application's
+# perspective even while writes are happening.
+sqlite3 "$LIBRENOTES_DB" ".backup '$work/librenotes.db'"
+
+# Notes archive. Use --warning=no-file-changed because per-user
+# files may be touched concurrently; we still get a consistent
+# point-in-time view per file thanks to tar's read semantics.
+tar --warning=no-file-changed -C "$LIBRENOTES_DATA_DIR" -czf "$work/notes.tar.gz" .
+
+cat > "$work/info.txt" <<EOF
+backup_at: $ts UTC
+host:      $(hostname)
+version:   $BACKUP_VERSION
+db_size:   $(stat -c%s "$work/librenotes.db" 2>/dev/null || stat -f%z "$work/librenotes.db")
+notes_size:$(stat -c%s "$work/notes.tar.gz" 2>/dev/null || stat -f%z "$work/notes.tar.gz")
+EOF
+
+archive="$BACKUP_DIR/librenotes-$ts.tar.gz"
+tar -C "$work" -czf "$archive" librenotes.db notes.tar.gz info.txt
+
+echo "wrote $archive ($(stat -c%s "$archive" 2>/dev/null || stat -f%z "$archive") bytes)"
+
+if [ -n "${BACKUP_REMOTE:-}" ]; then
+  rclone copy "$archive" "$BACKUP_REMOTE" --quiet
+  echo "uploaded to $BACKUP_REMOTE"
+fi
diff --git a/scripts/librenotes-backup-verify.service b/scripts/librenotes-backup-verify.service
new file mode 100644
index 0000000..cfbc0e8
--- /dev/null
+++ b/scripts/librenotes-backup-verify.service
@@ -0,0 +1,12 @@
+[Unit]
+Description=librenotes weekly backup-restore test
+
+[Service]
+Type=oneshot
+EnvironmentFile=-/etc/librenotes/backup.env
+ExecStart=/usr/local/bin/librenotes-backup-restore-test.sh
+TimeoutStartSec=10min
+ProtectSystem=strict
+ProtectHome=true
+PrivateTmp=true
+NoNewPrivileges=true
diff --git a/scripts/librenotes-backup-verify.timer b/scripts/librenotes-backup-verify.timer
new file mode 100644
index 0000000..ab837f0
--- /dev/null
+++ b/scripts/librenotes-backup-verify.timer
@@ -0,0 +1,10 @@
+[Unit]
+Description=Weekly verification of latest librenotes backup
+
+[Timer]
+OnCalendar=Mon *-*-* 04:00:00 UTC
+Persistent=true
+Unit=librenotes-backup-verify.service
+
+[Install]
+WantedBy=timers.target
diff --git a/scripts/librenotes-backup.service b/scripts/librenotes-backup.service
new file mode 100644
index 0000000..e212075
--- /dev/null
+++ b/scripts/librenotes-backup.service
@@ -0,0 +1,19 @@
+[Unit]
+Description=librenotes daily backup
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+Type=oneshot
+EnvironmentFile=-/etc/librenotes/backup.env
+ExecStart=/usr/local/bin/librenotes-backup.sh
+ExecStartPost=/usr/local/bin/librenotes-backup-prune.sh
+SuccessExitStatus=0
+TimeoutStartSec=30min
+# Backup driver only reads from the live data dir; never writes
+# back. Sandbox accordingly.
+ProtectSystem=strict
+ProtectHome=true
+PrivateTmp=true
+ReadWritePaths=/var/backups/librenotes
+NoNewPrivileges=true
diff --git a/scripts/librenotes-backup.timer b/scripts/librenotes-backup.timer
new file mode 100644
index 0000000..2150a38
--- /dev/null
+++ b/scripts/librenotes-backup.timer
@@ -0,0 +1,13 @@
+[Unit]
+Description=Run librenotes backup nightly
+
+[Timer]
+# 03:17 UTC nightly with up to 5min jitter so multiple machines on
+# the same schedule don't all hit the off-site target at once.
+OnCalendar=*-*-* 03:17:00 UTC
+RandomizedDelaySec=5min
+Persistent=true
+Unit=librenotes-backup.service
+
+[Install]
+WantedBy=timers.target