From dc74380fb7afb513e1f4b0ce377a1687c9aaf016 Mon Sep 17 00:00:00 2001 From: Michael Czechowski Date: Fri, 9 Oct 2026 01:11:32 +0200 Subject: [PATCH] wip: uncommitted work on kraftwerk, saved 9 Oct 2026 (netcup issue 86) --- .gitea/workflows/blog-ci.yml | 26 + .gitea/workflows/deploy-blog.yml | 161 +++++ .gitea/workflows/publish-blog.yml | 96 +++ .gitignore | 7 + README.md | 47 ++ delivery/.delivery/config.json | 1 + delivery/build.sh | 28 + .../scripts/__init__.py | 0 delivery/scripts/delivery_gate.py | 48 ++ delivery/scripts/deploy_contract.py | 246 +++++++ delivery/scripts/fetch_hugo.py | 63 ++ delivery/scripts/generic_package.py | 266 ++++++++ delivery/scripts/static_release.py | 646 ++++++++++++++++++ delivery/site.json | 1 + delivery/tests/test_blog_delivery.py | 104 +++ public/categories/index.xml | 11 - public/index.xml | 11 - public/sitemap.xml | 11 - public/tags/index.xml | 11 - 19 files changed, 1740 insertions(+), 44 deletions(-) create mode 100644 .gitea/workflows/blog-ci.yml create mode 100644 .gitea/workflows/deploy-blog.yml create mode 100644 .gitea/workflows/publish-blog.yml create mode 100644 .gitignore create mode 100644 README.md create mode 100644 delivery/.delivery/config.json create mode 100755 delivery/build.sh rename .hugo_build.lock => delivery/scripts/__init__.py (100%) create mode 100755 delivery/scripts/delivery_gate.py create mode 100755 delivery/scripts/deploy_contract.py create mode 100644 delivery/scripts/fetch_hugo.py create mode 100755 delivery/scripts/generic_package.py create mode 100755 delivery/scripts/static_release.py create mode 100644 delivery/site.json create mode 100644 delivery/tests/test_blog_delivery.py delete mode 100644 public/categories/index.xml delete mode 100644 public/index.xml delete mode 100644 public/sitemap.xml delete mode 100644 public/tags/index.xml diff --git a/.gitea/workflows/blog-ci.yml b/.gitea/workflows/blog-ci.yml new file mode 100644 index 0000000..a65884e --- /dev/null +++ b/.gitea/workflows/blog-ci.yml @@ -0,0 +1,26 @@ +name: Blog release + +on: + pull_request: + branches: [main] + +permissions: + contents: read + +jobs: + release: + runs-on: ubuntu-latest + container: + image: git.librete.ch/public/runner-image:v0.2.0@sha256:f60c587d3c0b0aac04a572db5349e27672bf76baec2ce547a3dcc28cebcf1b7e + timeout-minutes: 15 + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 + with: + fetch-depth: 0 + submodules: true + + - name: Test the release contract + run: cd delivery && python3 -m unittest discover -s tests -v + + - name: Build, package and inspect the immutable release + run: delivery/build.sh diff --git a/.gitea/workflows/deploy-blog.yml b/.gitea/workflows/deploy-blog.yml new file mode 100644 index 0000000..be5c4c7 --- /dev/null +++ b/.gitea/workflows/deploy-blog.yml @@ -0,0 +1,161 @@ +name: Deploy blog immutable package + +on: + workflow_dispatch: + inputs: + revision: + description: Full source commit + required: true + artifact_url: + description: Immutable Gitea generic-package URL + required: true + artifact_digest: + description: sha256 digest of site.tar.gz + required: true + target: + description: Provisioned site-scoped release root + required: true + verify_url: + description: Public release marker URL + required: true + desired_commit: + description: gitops-sandbox commit that selected this release + required: true + desired_generation: + description: Ancestor count of desired_commit; the receiver refuses older selections + required: true + +permissions: + contents: read + +# Best-effort only. Ordering is enforced by the receiver, which applies a +# selection only if its generation is not older than the one already live, so +# a stale dispatch or a re-run of an old run cannot replace a newer release. +concurrency: + group: blog-deploy + cancel-in-progress: false + +jobs: + contract: + runs-on: ubuntu-latest + container: + image: git.librete.ch/public/runner-image:v0.2.0@sha256:f60c587d3c0b0aac04a572db5349e27672bf76baec2ce547a3dcc28cebcf1b7e + timeout-minutes: 5 + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 + + - name: Validate dispatch contract without network access + env: + REVISION: ${{ inputs.revision }} + ARTIFACT_URL: ${{ inputs.artifact_url }} + ARTIFACT_DIGEST: ${{ inputs.artifact_digest }} + TARGET: ${{ inputs.target }} + VERIFY_URL: ${{ inputs.verify_url }} + DESIRED_COMMIT: ${{ inputs.desired_commit }} + DESIRED_GENERATION: ${{ inputs.desired_generation }} + run: | + set -eu + python3 delivery/scripts/deploy_contract.py validate \ + --revision "$REVISION" \ + --artifact-url "$ARTIFACT_URL" \ + --artifact-digest "$ARTIFACT_DIGEST" \ + --target "$TARGET" \ + --verify-url "$VERIFY_URL" \ + --site-config delivery/site.json \ + --desired-commit "$DESIRED_COMMIT" \ + --desired-generation "$DESIRED_GENERATION" + + deploy: + needs: contract + if: ${{ vars.BLOG_STATIC_DEPLOY_ENABLED == 'true' }} + runs-on: ubuntu-latest + container: + image: git.librete.ch/public/runner-image:v0.2.0@sha256:f60c587d3c0b0aac04a572db5349e27672bf76baec2ce547a3dcc28cebcf1b7e + timeout-minutes: 30 + env: + ACTIVATION: ${{ vars.BLOG_STATIC_DEPLOY_ENABLED }} + REVISION: ${{ inputs.revision }} + ARTIFACT_URL: ${{ inputs.artifact_url }} + ARTIFACT_DIGEST: ${{ inputs.artifact_digest }} + TARGET: ${{ inputs.target }} + VERIFY_URL: ${{ inputs.verify_url }} + DESIRED_COMMIT: ${{ inputs.desired_commit }} + DESIRED_GENERATION: ${{ inputs.desired_generation }} + STATIC_PACKAGE_READ_USER: ${{ secrets.BLOG_PACKAGE_READ_USER }} + STATIC_PACKAGE_READ_TOKEN: ${{ secrets.BLOG_PACKAGE_READ_TOKEN }} + STATIC_DEPLOY_HOST: ${{ secrets.BLOG_STATIC_DEPLOY_HOST }} + STATIC_DEPLOY_USER: ${{ secrets.BLOG_STATIC_DEPLOY_USER }} + STATIC_DEPLOY_SSH_KEY: ${{ secrets.BLOG_STATIC_DEPLOY_SSH_KEY }} + STATIC_DEPLOY_KNOWN_HOSTS: ${{ secrets.BLOG_STATIC_DEPLOY_KNOWN_HOSTS }} + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 + + - name: Enforce reviewed deployment gate + run: | + set -eu + test "$ACTIVATION" = true + python3 delivery/scripts/delivery_gate.py static_deploy \ + --config delivery/.delivery/config.json + test -n "$STATIC_DEPLOY_HOST" + test -n "$STATIC_DEPLOY_USER" + test -n "$STATIC_DEPLOY_SSH_KEY" + test -n "$STATIC_DEPLOY_KNOWN_HOSTS" + case "$STATIC_DEPLOY_HOST" in *[!A-Za-z0-9.-]*|'') exit 1;; esac + case "$STATIC_DEPLOY_USER" in *[!A-Za-z0-9_-]*|'') exit 1;; esac + + - name: Download and inspect the declared immutable artifact + run: | + set -eu + python3 delivery/scripts/deploy_contract.py prepare \ + --revision "$REVISION" \ + --artifact-url "$ARTIFACT_URL" \ + --artifact-digest "$ARTIFACT_DIGEST" \ + --target "$TARGET" \ + --verify-url "$VERIFY_URL" \ + --site-config delivery/site.json \ + --desired-commit "$DESIRED_COMMIT" \ + --desired-generation "$DESIRED_GENERATION" \ + --archive delivery/.build/site.tar.gz \ + --checksum delivery/.build/site.tar.gz.sha256 \ + --request-token delivery/.build/deploy-request.token + + - name: Send the artifact to the site-scoped forced command + id: receive + run: | + set -eu + umask 077 + mkdir -p "$HOME/.ssh" + printf '%s\n' "$STATIC_DEPLOY_SSH_KEY" > "$HOME/.ssh/id_static_deploy" + printf '%s\n' "$STATIC_DEPLOY_KNOWN_HOSTS" > "$HOME/.ssh/known_hosts" + cat > "$HOME/.ssh/config" <> "$GITHUB_OUTPUT" + exit 0 + fi + test "$status" -eq 0 + printf 'superseded=false\n' >> "$GITHUB_OUTPUT" + + - name: Confirm public revision and digest + if: ${{ steps.receive.outputs.superseded == 'false' }} + run: | + set -eu + python3 delivery/scripts/static_release.py verify \ + --url "$VERIFY_URL" \ + --revision "$REVISION" \ + --digest "$ARTIFACT_DIGEST" diff --git a/.gitea/workflows/publish-blog.yml b/.gitea/workflows/publish-blog.yml new file mode 100644 index 0000000..97f5c4a --- /dev/null +++ b/.gitea/workflows/publish-blog.yml @@ -0,0 +1,96 @@ +name: Publish blog immutable package + +# Every main push builds and tests the release. Publication additionally needs +# the activation variable, the reviewed repository gate and the scoped package +# secrets; without them nothing is published. Publication never deploys: +# selecting the release is a reviewed change to gitops-sandbox/stacks.yml. + +on: + push: + branches: [main] + workflow_dispatch: + inputs: + revision: + description: Full public/blog commit to publish + required: true + +permissions: + contents: read + +concurrency: + group: blog-publish + cancel-in-progress: false + +jobs: + validate: + runs-on: ubuntu-latest + container: + image: git.librete.ch/public/runner-image:v0.2.0@sha256:f60c587d3c0b0aac04a572db5349e27672bf76baec2ce547a3dcc28cebcf1b7e + timeout-minutes: 15 + env: + REQUESTED_REVISION: ${{ inputs.revision || github.sha }} + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 + with: + ref: ${{ inputs.revision || github.sha }} + fetch-depth: 0 + submodules: true + + - name: Validate requested source, tests and build + run: | + set -eu + test "$(git rev-parse HEAD)" = "$REQUESTED_REVISION" + (cd delivery && python3 -m unittest discover -s tests -v) + delivery/build.sh + + publish: + needs: validate + if: ${{ vars.BLOG_PACKAGE_PUBLISH_ENABLED == 'true' }} + runs-on: ubuntu-latest + container: + image: git.librete.ch/public/runner-image:v0.2.0@sha256:f60c587d3c0b0aac04a572db5349e27672bf76baec2ce547a3dcc28cebcf1b7e + timeout-minutes: 15 + env: + ACTIVATION: ${{ vars.BLOG_PACKAGE_PUBLISH_ENABLED }} + REQUESTED_REVISION: ${{ inputs.revision || github.sha }} + STATIC_PACKAGE_USER: ${{ secrets.BLOG_PACKAGE_USER }} + STATIC_PACKAGE_TOKEN: ${{ secrets.BLOG_PACKAGE_TOKEN }} + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 + with: + ref: ${{ inputs.revision || github.sha }} + fetch-depth: 0 + submodules: true + + - name: Enforce reviewed publication gate + run: | + set -eu + test "$ACTIVATION" = true + python3 delivery/scripts/delivery_gate.py package_publish \ + --config delivery/.delivery/config.json + test -n "$STATIC_PACKAGE_USER" + test -n "$STATIC_PACKAGE_TOKEN" + + - name: Build and publish immutable package files + run: | + set -eu + test "$(git rev-parse HEAD)" = "$REQUESTED_REVISION" + delivery/build.sh + python3 delivery/scripts/generic_package.py \ + --registry "${{ gitea.server_url }}" \ + --owner public \ + --package blog-site \ + --revision "$REQUESTED_REVISION" \ + --archive delivery/.build/site.tar.gz \ + --checksum delivery/.build/site.tar.gz.sha256 \ + --coordinate delivery/.build/release-coordinate.json \ + > delivery/.build/published.json + cat delivery/.build/published.json + echo "Select this release with a reviewed gitops-sandbox stacks.yml change:" + python3 - delivery/.build/published.json <<'PY' + import json, sys + c = json.load(open(sys.argv[1])) + print(" blog:") + for key in ("revision", "artifact_url", "artifact_digest"): + print(f" {key}: {c[key]}") + PY diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..4512e36 --- /dev/null +++ b/.gitignore @@ -0,0 +1,7 @@ +# Generated output and caches. Releases are immutable packages built in CI; +# nothing here is published from a checkout. +/public/ +/resources/_gen/ +.hugo_build.lock +/delivery/.build/ +__pycache__/ diff --git a/README.md b/README.md new file mode 100644 index 0000000..a3c7531 --- /dev/null +++ b/README.md @@ -0,0 +1,47 @@ +# blog + +Hugo source for the LibreTECH blog with the +[Pickles](https://github.com/mismith0227/hugo_theme_pickles) theme as a pinned +submodule. + +```sh +git clone --recurse-submodules https://git.librete.ch/public/blog.git +hugo server # local preview +``` + +## Releases + +The blog is published as an immutable release, never from a checkout. The +served tree contains only the built site and its release marker: no source, +`.git`, credentials or links outside the release. + +1. `delivery/build.sh` builds the checked-out commit with the pinned Hugo + release (verified by SHA-256) and packages it deterministically as + `delivery/.build/site.tar.gz`. A rebuild of the same commit has the same + digest. +2. **Publish blog immutable package** (`publish-blog.yml`) runs on every `main` + push. With `BLOG_PACKAGE_PUBLISH_ENABLED=true`, the reviewed + `package_publish` gate and the `BLOG_PACKAGE_USER`/`BLOG_PACKAGE_TOKEN` + secrets, it publishes `public/blog-site//` to the Gitea package + registry and prints the `stacks.yml` coordinate. +3. A reviewed pull request in + [`libretech/gitops-sandbox`](https://git.librete.ch/libretech/gitops-sandbox) + puts that coordinate in the `blog` record. Merging it dispatches + **Deploy blog immutable package** (`deploy-blog.yml`). +4. The deploy workflow re-downloads and inspects the package, then streams it + to the site-scoped receiver on Netcup, which activates + `/srv/libretech-static/blog/current` atomically and verifies + `https://blog.static.librete.ch/.well-known/release.json`. A stale or + re-run dispatch older than the live selection ends as *superseded* and + changes nothing. + +Rollback is a reviewed `stacks.yml` change back to an earlier complete +coordinate. See the +[operations guide](https://git.librete.ch/libretech/gitops-sandbox/src/branch/main/OPERATIONS.md). + +`delivery/scripts/` is a copy of the receiver modules from +`libretech/librete.ch` (`donatella/scripts/`); update them together. + +`blog.librete.ch` itself is still served from the earlier Uberspace in-place +build (`publishDir` in `hugo.toml`) until its DNS cut-over; the release +workflows override `publishDir` and never write there. diff --git a/delivery/.delivery/config.json b/delivery/.delivery/config.json new file mode 100644 index 0000000..4b79319 --- /dev/null +++ b/delivery/.delivery/config.json @@ -0,0 +1 @@ +{"schema":1,"package_publish":true,"static_deploy":true} diff --git a/delivery/build.sh b/delivery/build.sh new file mode 100755 index 0000000..bdea5ca --- /dev/null +++ b/delivery/build.sh @@ -0,0 +1,28 @@ +#!/bin/sh +# Build the checked-out commit into an immutable release archive: +# delivery/.build/site.tar.gz and its .sha256. Needs full Git history +# (enableGitInfo) and the theme submodule. +set -eu +cd "$(dirname "$0")/.." +revision="$(git rev-parse HEAD)" +test -f themes/hugo_theme_pickles/theme.toml +build="$PWD/delivery/.build" +rm -rf "$build" +mkdir -p "$build" +# CI uses the pinned release; HUGO may name a local binary for previews only. +hugo="${HUGO:-$(python3 delivery/scripts/fetch_hugo.py --destination "$build/bin")}" +"$hugo" version +# --destination and --cacheDir override the legacy in-place publishDir. +"$hugo" --source . --environment production \ + --destination "$build/hugo" --cacheDir "$build/cache" --cleanDestinationDir +python3 delivery/scripts/static_release.py build \ + --source "$build/hugo" --output "$build/dist" --revision "$revision" +python3 delivery/scripts/static_release.py package \ + --directory "$build/dist" \ + --archive "$build/site.tar.gz" \ + --checksum "$build/site.tar.gz.sha256" +python3 delivery/scripts/static_release.py inspect \ + --archive "$build/site.tar.gz" \ + --checksum "$build/site.tar.gz.sha256" \ + --revision "$revision" +cat "$build/site.tar.gz.sha256" diff --git a/.hugo_build.lock b/delivery/scripts/__init__.py similarity index 100% rename from .hugo_build.lock rename to delivery/scripts/__init__.py diff --git a/delivery/scripts/delivery_gate.py b/delivery/scripts/delivery_gate.py new file mode 100755 index 0000000..0d82f09 --- /dev/null +++ b/delivery/scripts/delivery_gate.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 +"""Fail closed unless a delivery capability is enabled in reviewed config.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path +import sys + + +CAPABILITIES = {"package_publish", "static_deploy"} + + +class GateError(RuntimeError): + pass + + +def check(config_path: Path, capability: str) -> None: + if capability not in CAPABILITIES: + raise GateError(f"unsupported delivery capability: {capability}") + try: + config = json.loads(config_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise GateError(f"invalid delivery configuration: {exc}") from exc + if set(config) != {"schema", *CAPABILITIES} or config["schema"] != 1: + raise GateError("delivery configuration has an unsupported shape or schema") + if any(type(config[name]) is not bool for name in CAPABILITIES): + raise GateError("delivery capability values must be booleans") + if not config[capability]: + raise GateError(f"{capability} is disabled in reviewed repository configuration") + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("capability", choices=sorted(CAPABILITIES)) + parser.add_argument("--config", type=Path, default=Path(".delivery/config.json")) + args = parser.parse_args() + try: + check(args.config, args.capability) + except GateError as exc: + print(f"error: {exc}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/delivery/scripts/deploy_contract.py b/delivery/scripts/deploy_contract.py new file mode 100755 index 0000000..e0c1495 --- /dev/null +++ b/delivery/scripts/deploy_contract.py @@ -0,0 +1,246 @@ +#!/usr/bin/env python3 +"""Validate and prepare the project-owned static deployment request.""" + +from __future__ import annotations + +import argparse +import base64 +from dataclasses import dataclass +import json +import os +from pathlib import Path +import re +import sys + +try: + from .generic_package import PackageError, package_file_url, request + from .static_release import ( + ReleaseError, + inspect_archive, + validate_digest, + validate_order, + validate_revision, + ) +except ImportError: # Direct script execution adds scripts/ to sys.path. + from generic_package import PackageError, package_file_url, request + from static_release import ( + ReleaseError, + inspect_archive, + validate_digest, + validate_order, + validate_revision, + ) + + +REGISTRY = "https://git.librete.ch" +OWNER = "libretech" +PACKAGE = "donatella-site" +PUBLIC_HOST = "donatella.static.librete.ch" +TARGET = "/srv/libretech-static/donatella" +SERVE_ROOT = f"{TARGET}/current" +VERIFY_URL = f"https://{PUBLIC_HOST}/.well-known/release.json" +RELEASE_FIELDS = ("revision", "artifact_url", "artifact_digest", "target", "verify_url") +ORDER_FIELDS = ("desired_commit", "desired_generation") +SITE_NAME_RE = re.compile(r"^[a-z0-9][a-z0-9-]{0,62}$") + + +class ContractError(RuntimeError): + pass + + +@dataclass(frozen=True) +class Site: + """One enrolled static site: its package, release root and public marker.""" + + name: str + owner: str + package: str + target: str + verify_url: str + registry: str = REGISTRY + + +DONATELLA = Site("donatella", OWNER, PACKAGE, TARGET, VERIFY_URL) + + +def load_site(path: Path | None) -> Site: + """Read a reviewed site.json; without one, the receiver serves Donatella.""" + if path is None: + return DONATELLA + try: + document = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise ContractError(f"invalid site configuration: {exc}") from exc + fields = {"schema", "name", "owner", "package", "target", "verify_url"} + if not isinstance(document, dict) or set(document) != fields or document["schema"] != 1: + raise ContractError("site configuration has an unsupported shape or schema") + site = Site(**{key: document[key] for key in fields - {"schema"}}) + if not all(isinstance(value, str) for value in vars(site).values()): + raise ContractError("site configuration values must be strings") + if not SITE_NAME_RE.fullmatch(site.name) or not SITE_NAME_RE.fullmatch(site.owner): + raise ContractError("site name and owner must be lowercase identifiers") + if not SITE_NAME_RE.fullmatch(site.package): + raise ContractError("package must be a lowercase identifier") + if site.target != f"/srv/libretech-static/{site.name}": + raise ContractError("site target must be /srv/libretech-static/") + if not site.verify_url.startswith("https://") or not site.verify_url.endswith( + "/.well-known/release.json", + ): + raise ContractError("site verify_url must be an HTTPS release marker") + return site + + +def validate_contract( + *, + revision: str, + artifact_url: str, + artifact_digest: str, + target: str, + verify_url: str, + registry: str | None = None, + allow_http: bool = False, + site: Site = DONATELLA, +) -> dict[str, str]: + values = { + "revision": revision, + "artifact_url": artifact_url, + "artifact_digest": artifact_digest, + "target": target, + "verify_url": verify_url, + } + if any(not isinstance(value, str) or not value for value in values.values()): + raise ContractError("all deployment contract fields must be non-empty strings") + try: + validate_revision(revision) + validate_digest(artifact_digest) + expected_url = package_file_url( + registry or site.registry, + site.owner, + site.package, + revision, + "site.tar.gz", + allow_http=allow_http, + ) + except (ReleaseError, PackageError) as exc: + raise ContractError(str(exc)) from exc + if artifact_url != expected_url: + raise ContractError("artifact URL is not the immutable project package for this revision") + if target != site.target: + raise ContractError(f"target must be exactly {site.target}") + if verify_url != site.verify_url: + raise ContractError(f"verification URL must be exactly {site.verify_url}") + return values + + +def validate_request(*, site: Site = DONATELLA, **fields) -> dict[str, str]: + """Validate a release coordinate plus the GitOps selection that chose it.""" + if set(fields) != set(RELEASE_FIELDS) | set(ORDER_FIELDS): + raise ContractError("deployment request must contain exactly the seven declared fields") + contract = validate_contract(site=site, **{key: fields[key] for key in RELEASE_FIELDS}) + order = {key: fields[key] for key in ORDER_FIELDS} + try: + validate_order(order) + except ReleaseError as exc: + raise ContractError(str(exc)) from exc + return {**contract, **order} + + +def order_of(request: dict[str, str]) -> dict[str, str]: + return {key: request[key] for key in ORDER_FIELDS} + + +def request_token(contract: dict[str, str]) -> str: + encoded = base64.urlsafe_b64encode( + json.dumps(contract, separators=(",", ":")).encode("utf-8"), + ).decode("ascii") + return encoded.rstrip("=") + + +def prepare( + contract: dict[str, str], + *, + archive: Path, + checksum: Path, + token_output: Path, + username: str | None = None, + package_token: str | None = None, +) -> None: + status, content = request( + "GET", + contract["artifact_url"], + username=username, + token=package_token, + ) + if status != 200: + raise ContractError(f"artifact download returned HTTP {status}") + archive.parent.mkdir(parents=True, exist_ok=True) + checksum.parent.mkdir(parents=True, exist_ok=True) + token_output.parent.mkdir(parents=True, exist_ok=True) + archive.write_bytes(content) + checksum.write_text( + f"{contract['artifact_digest']} {archive.name}\n", + encoding="ascii", + ) + try: + inspect_archive(archive, checksum, contract["revision"]) + except ReleaseError as exc: + raise ContractError(str(exc)) from exc + token_output.write_text(request_token(contract) + "\n", encoding="ascii") + + +def add_contract_arguments(command: argparse.ArgumentParser) -> None: + command.add_argument("--revision", required=True) + command.add_argument("--artifact-url", required=True) + command.add_argument("--artifact-digest", required=True) + command.add_argument("--target", required=True) + command.add_argument("--verify-url", required=True) + command.add_argument("--desired-commit", required=True) + command.add_argument("--desired-generation", required=True) + command.add_argument("--site-config", type=Path) + + +def parser() -> argparse.ArgumentParser: + command = argparse.ArgumentParser(description=__doc__) + sub = command.add_subparsers(dest="command", required=True) + validate_cmd = sub.add_parser("validate") + add_contract_arguments(validate_cmd) + prepare_cmd = sub.add_parser("prepare") + add_contract_arguments(prepare_cmd) + prepare_cmd.add_argument("--archive", type=Path, required=True) + prepare_cmd.add_argument("--checksum", type=Path, required=True) + prepare_cmd.add_argument("--request-token", type=Path, required=True) + return command + + +def main() -> int: + args = parser().parse_args() + try: + contract = validate_request( + site=load_site(args.site_config), + revision=args.revision, + artifact_url=args.artifact_url, + artifact_digest=args.artifact_digest, + target=args.target, + verify_url=args.verify_url, + desired_commit=args.desired_commit, + desired_generation=args.desired_generation, + ) + if args.command == "prepare": + prepare( + contract, + archive=args.archive, + checksum=args.checksum, + token_output=args.request_token, + username=os.environ.get("STATIC_PACKAGE_READ_USER") or None, + package_token=os.environ.get("STATIC_PACKAGE_READ_TOKEN") or None, + ) + else: + print(json.dumps(contract, separators=(",", ":"))) + except (ContractError, PackageError) as exc: + print(f"error: {exc}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/delivery/scripts/fetch_hugo.py b/delivery/scripts/fetch_hugo.py new file mode 100644 index 0000000..966468a --- /dev/null +++ b/delivery/scripts/fetch_hugo.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 +"""Install the pinned Hugo release after verifying its published SHA-256.""" + +from __future__ import annotations + +import argparse +import hashlib +import io +from pathlib import Path +import sys +import tarfile +import urllib.request + + +VERSION = "0.150.1" +URL = ( + f"https://github.com/gohugoio/hugo/releases/download/v{VERSION}/" + f"hugo_extended_{VERSION}_linux-amd64.tar.gz" +) +SHA256 = "e1248fa077d99232794e38df5ec533494993aafafca1ae3e331a4ed629079ed6" +MAX_BYTES = 128 * 1024 * 1024 + + +class FetchError(RuntimeError): + pass + + +def extract_hugo(archive: bytes, expected: str, destination: Path) -> Path: + actual = hashlib.sha256(archive).hexdigest() + if actual != expected: + raise FetchError(f"Hugo archive digest mismatch: {actual}") + with tarfile.open(fileobj=io.BytesIO(archive), mode="r:gz") as tar: + member = tar.getmember("hugo") + if not member.isfile(): + raise FetchError("Hugo archive member is not a regular file") + source = tar.extractfile(member) + if source is None: + raise FetchError("cannot read the Hugo binary") + destination.mkdir(parents=True, exist_ok=True) + binary = destination / "hugo" + binary.write_bytes(source.read()) + binary.chmod(0o755) + return binary + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--destination", type=Path, required=True) + args = parser.parse_args() + try: + with urllib.request.urlopen(URL, timeout=60) as response: + archive = response.read(MAX_BYTES + 1) + if len(archive) > MAX_BYTES: + raise FetchError("Hugo archive exceeds the size limit") + print(extract_hugo(archive, SHA256, args.destination)) + except (OSError, KeyError, tarfile.TarError, FetchError) as exc: + print(f"error: {exc}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/delivery/scripts/generic_package.py b/delivery/scripts/generic_package.py new file mode 100755 index 0000000..d8024fd --- /dev/null +++ b/delivery/scripts/generic_package.py @@ -0,0 +1,266 @@ +#!/usr/bin/env python3 +"""Publish a static release as immutable files in Gitea's generic registry.""" + +from __future__ import annotations + +import argparse +import base64 +import json +import os +from pathlib import Path +import re +import sys +import urllib.error +import urllib.parse +import urllib.request + +try: + from .static_release import ReleaseError, inspect_archive, validate_revision +except ImportError: # Direct script execution adds scripts/ to sys.path. + from static_release import ReleaseError, inspect_archive, validate_revision + + +COMPONENT_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._+-]*$") +ARCHIVE_NAME = "site.tar.gz" +CHECKSUM_NAME = "site.tar.gz.sha256" +COORDINATE_NAME = "release-coordinate.json" +MAX_RESPONSE_BYTES = 200 * 1024 * 1024 + + +class PackageError(RuntimeError): + pass + + +def validate_component(value: str, label: str) -> None: + if not COMPONENT_RE.fullmatch(value): + raise PackageError(f"{label} contains unsupported characters") + + +def validate_registry(registry: str, *, allow_http: bool = False) -> str: + parsed = urllib.parse.urlsplit(registry) + allowed_schemes = {"https"} | ({"http"} if allow_http else set()) + if ( + parsed.scheme not in allowed_schemes + or not parsed.netloc + or parsed.query + or parsed.fragment + or parsed.username + or parsed.password + ): + raise PackageError("registry must be an HTTPS origin without credentials, query or fragment") + return registry.rstrip("/") + + +def package_file_url( + registry: str, + owner: str, + package: str, + version: str, + filename: str, + *, + allow_http: bool = False, +) -> str: + registry = validate_registry(registry, allow_http=allow_http) + for value, label in ((owner, "owner"), (package, "package"), (filename, "filename")): + validate_component(value, label) + validate_revision(version) + components = [owner, package, version, filename] + owner_part, package_part, version_part, filename_part = ( + urllib.parse.quote(value, safe="") for value in components + ) + return ( + f"{registry}/api/packages/{owner_part}/generic/" + f"{package_part}/{version_part}/{filename_part}" + ) + + +def authorization(username: str | None, token: str | None) -> str | None: + if bool(username) != bool(token): + raise PackageError("package username and token must be supplied together") + if not username: + return None + encoded = base64.b64encode(f"{username}:{token}".encode()).decode("ascii") + return f"Basic {encoded}" + + +def request( + method: str, + url: str, + *, + username: str | None, + token: str | None, + data: bytes | None = None, + timeout: float = 300.0, +) -> tuple[int, bytes]: + headers = {"User-Agent": "donatella-static-publisher/1"} + auth = authorization(username, token) + if auth: + headers["Authorization"] = auth + if data is not None: + headers["Content-Type"] = "application/octet-stream" + outgoing = urllib.request.Request(url, data=data, headers=headers, method=method) + try: + with urllib.request.urlopen(outgoing, timeout=timeout) as response: + content = response.read(MAX_RESPONSE_BYTES + 1) + if len(content) > MAX_RESPONSE_BYTES: + raise PackageError("package registry response exceeds the size limit") + return response.status, content + except urllib.error.HTTPError as exc: + try: + content = exc.read(MAX_RESPONSE_BYTES + 1) + if len(content) > MAX_RESPONSE_BYTES: + raise PackageError("package registry error response exceeds the size limit") + return exc.code, content + finally: + exc.close() + except (OSError, urllib.error.URLError) as exc: + raise PackageError(f"package registry request failed: {exc}") from exc + + +def put_immutable( + url: str, + content: bytes, + *, + username: str, + token: str, + timeout: float = 300.0, +) -> None: + status, existing = request("GET", url, username=username, token=token, timeout=timeout) + if status == 200: + if existing != content: + raise PackageError(f"immutable package file already exists with different content: {url}") + return + if status != 404: + raise PackageError(f"package preflight returned HTTP {status}: {url}") + + status, _ = request( + "PUT", + url, + username=username, + token=token, + data=content, + timeout=timeout, + ) + if status not in {201, 409}: + raise PackageError(f"package upload returned HTTP {status}: {url}") + + status, published = request("GET", url, username=username, token=token, timeout=timeout) + if status != 200 or published != content: + raise PackageError(f"published package file did not verify byte-for-byte: {url}") + + +def coordinate_bytes(revision: str, artifact_url: str, digest: str) -> bytes: + return ( + json.dumps( + { + "revision": revision, + "artifact_url": artifact_url, + "artifact_digest": digest, + }, + separators=(",", ":"), + ) + + "\n" + ).encode("utf-8") + + +def publish_release( + *, + registry: str, + owner: str, + package: str, + revision: str, + archive: Path, + checksum: Path, + coordinate: Path, + username: str, + token: str, + allow_http: bool = False, +) -> dict[str, str]: + if not username or not token: + raise PackageError("package username and token are required") + if archive.name != ARCHIVE_NAME or checksum.name != CHECKSUM_NAME: + raise PackageError(f"release files must be named {ARCHIVE_NAME} and {CHECKSUM_NAME}") + try: + _, _, digest = inspect_archive(archive, checksum, revision) + except ReleaseError as exc: + raise PackageError(str(exc)) from exc + + artifact_url = package_file_url( + registry, + owner, + package, + revision, + ARCHIVE_NAME, + allow_http=allow_http, + ) + checksum_url = package_file_url( + registry, + owner, + package, + revision, + CHECKSUM_NAME, + allow_http=allow_http, + ) + coordinate_url = package_file_url( + registry, + owner, + package, + revision, + COORDINATE_NAME, + allow_http=allow_http, + ) + payload = coordinate_bytes(revision, artifact_url, digest) + if coordinate.resolve() in {archive.resolve(), checksum.resolve()}: + raise PackageError("coordinate output must be separate from the archive and checksum") + coordinate.parent.mkdir(parents=True, exist_ok=True) + coordinate.write_bytes(payload) + + for url, content in ( + (artifact_url, archive.read_bytes()), + (checksum_url, checksum.read_bytes()), + (coordinate_url, payload), + ): + put_immutable(url, content, username=username, token=token) + + return { + "revision": revision, + "artifact_url": artifact_url, + "artifact_digest": digest, + } + + +def parser() -> argparse.ArgumentParser: + command = argparse.ArgumentParser(description=__doc__) + command.add_argument("--registry", required=True) + command.add_argument("--owner", required=True) + command.add_argument("--package", required=True) + command.add_argument("--revision", required=True) + command.add_argument("--archive", type=Path, required=True) + command.add_argument("--checksum", type=Path, required=True) + command.add_argument("--coordinate", type=Path, required=True) + return command + + +def main() -> int: + args = parser().parse_args() + try: + result = publish_release( + registry=args.registry, + owner=args.owner, + package=args.package, + revision=args.revision, + archive=args.archive, + checksum=args.checksum, + coordinate=args.coordinate, + username=os.environ.get("STATIC_PACKAGE_USER", ""), + token=os.environ.get("STATIC_PACKAGE_TOKEN", ""), + ) + except PackageError as exc: + print(f"error: {exc}", file=sys.stderr) + return 1 + print(json.dumps(result, separators=(",", ":"))) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/delivery/scripts/static_release.py b/delivery/scripts/static_release.py new file mode 100755 index 0000000..e4134ba --- /dev/null +++ b/delivery/scripts/static_release.py @@ -0,0 +1,646 @@ +#!/usr/bin/env python3 +"""Build, inspect, publish, verify and roll back trusted static releases.""" + +from __future__ import annotations + +import argparse +import contextlib +import datetime as dt +import fcntl +import gzip +import hashlib +import io +import json +import os +from pathlib import Path, PurePosixPath +import re +import shutil +import sys +import tarfile +import tempfile +import urllib.error +import urllib.parse +import urllib.request + + +REVISION_RE = re.compile(r"^[0-9a-f]{40}$") +GENERATION_RE = re.compile(r"^[1-9][0-9]{0,11}$") +DIGEST_RE = re.compile(r"^sha256:[0-9a-f]{64}$") +MAX_FILES = 10_000 +# The Sarah Weisman portfolio contains 164,651,446 bytes of site assets. +# Keep one bounded limit for the archive, receiver and extracted release. +MAX_BYTES = 200 * 1024 * 1024 +SOURCE_MARKER = Path(".release-source.json") +PUBLIC_MARKER = Path(".well-known/release.json") +# Last desired-state selection this target applied, outside the served tree. +DESIRED_STATE = Path(".desired-state.json") +FORBIDDEN_PARTS = {".git", ".hg", ".svn"} + + +class ReleaseError(RuntimeError): + pass + + +class SupersededError(ReleaseError): + """A newer desired-state generation has already been applied here.""" + + +def validate_revision(revision: str) -> None: + if not REVISION_RE.fullmatch(revision): + raise ReleaseError("revision must be a full 40-character lowercase Git SHA") + + +def validate_digest(digest: str) -> None: + if not DIGEST_RE.fullmatch(digest): + raise ReleaseError("digest must have the form sha256:<64 lowercase hex characters>") + + +def validate_order(order: dict) -> dict: + """Validate the GitOps selection that requested a release. + + ``desired_generation`` is the ancestor count of ``desired_commit`` on the + GitOps main branch. Main only moves to descendants, so a later selection + always has a larger generation than an earlier one. + """ + if not isinstance(order, dict) or set(order) != {"desired_commit", "desired_generation"}: + raise ReleaseError("order must contain exactly desired_commit and desired_generation") + if not isinstance(order["desired_commit"], str) or not REVISION_RE.fullmatch(order["desired_commit"]): + raise ReleaseError("desired_commit must be a full 40-character lowercase Git SHA") + if not isinstance(order["desired_generation"], str) or not GENERATION_RE.fullmatch(order["desired_generation"]): + raise ReleaseError("desired_generation must be a positive decimal integer without leading zeros") + return order + + +def load_desired_state(root: Path) -> dict | None: + path = root / DESIRED_STATE + if not path.exists() and not path.is_symlink(): + return None + if path.is_symlink() or not path.is_file(): + raise ReleaseError("desired-state record must be a regular file") + try: + state = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise ReleaseError(f"invalid desired-state record: {exc}") from exc + if not isinstance(state, dict) or set(state) != { + "desired_commit", "desired_generation", "revision", "artifact_digest", + }: + raise ReleaseError("desired-state record has an unsupported shape") + validate_order({key: state[key] for key in ("desired_commit", "desired_generation")}) + validate_revision(state["revision"]) + validate_digest(state["artifact_digest"]) + return state + + +def check_order(root: Path, revision: str, digest: str, order: dict) -> None: + """Refuse a request whose selection is older than the one applied here. + + An identical request (same generation, commit and release) is a retry of + the accepted selection and may be applied again. + """ + state = load_desired_state(root) + if state is None: + return + requested = int(order["desired_generation"]) + applied = int(state["desired_generation"]) + if requested < applied: + raise SupersededError( + f"superseded: generation {requested} ({order['desired_commit']}) is older than " + f"applied generation {applied} ({state['desired_commit']})" + ) + if requested == applied and ( + order["desired_commit"] != state["desired_commit"] + or revision != state["revision"] + or digest != state["artifact_digest"] + ): + raise ReleaseError( + f"generation {requested} was already applied with a different selection" + ) + + +def write_desired_state(root: Path, revision: str, digest: str, order: dict) -> None: + state = {**order, "revision": revision, "artifact_digest": digest} + temporary = root / f".desired-state.next-{os.getpid()}" + temporary.write_text(json.dumps(state, sort_keys=True) + "\n", encoding="utf-8") + os.replace(temporary, root / DESIRED_STATE) + + +def validate_site_path(path: PurePosixPath) -> None: + if any(part in FORBIDDEN_PARTS for part in path.parts): + raise ReleaseError(f"version-control metadata is forbidden: {path}") + filename = path.name.lower() + if filename == ".env" or filename.startswith(".env.") or filename in {"id_rsa", "id_ed25519"}: + raise ReleaseError(f"common secret filename is forbidden: {path}") + + +def iter_files(root: Path): + for path in sorted(root.rglob("*")): + relative = PurePosixPath(path.relative_to(root).as_posix()) + validate_site_path(relative) + if path.is_symlink(): + raise ReleaseError(f"symlinks are forbidden: {relative}") + if path.is_file(): + yield path + elif not path.is_dir(): + raise ReleaseError(f"unsupported filesystem entry: {relative}") + + +def validate_tree(root: Path) -> None: + count = 0 + size = 0 + for path in iter_files(root): + count += 1 + size += path.stat().st_size + if count > MAX_FILES or size > MAX_BYTES: + raise ReleaseError("site exceeds the file-count or uncompressed-size limit") + if not (root / "index.html").is_file(): + raise ReleaseError("release has no index.html") + + +def load_source_marker(root: Path) -> dict: + try: + marker = json.loads((root / SOURCE_MARKER).read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise ReleaseError(f"invalid or missing source marker: {exc}") from exc + if set(marker) != {"revision"}: + raise ReleaseError("source marker must contain only the revision") + validate_revision(marker["revision"]) + validate_tree(root) + return marker + + +def load_public_marker(root: Path) -> dict: + try: + marker = json.loads((root / PUBLIC_MARKER).read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise ReleaseError(f"invalid or missing public release marker: {exc}") from exc + if set(marker) != {"revision", "artifact_digest"}: + raise ReleaseError("public marker must contain revision and artifact_digest") + validate_revision(marker["revision"]) + validate_digest(marker["artifact_digest"]) + return marker + + +def build(source: Path, output: Path, revision: str) -> None: + validate_revision(revision) + if not source.is_dir(): + raise ReleaseError(f"source directory does not exist: {source}") + if (source / SOURCE_MARKER).exists() or (source / PUBLIC_MARKER).exists(): + raise ReleaseError("source must not provide reserved release markers") + validate_tree(source) + + source_root = source.resolve() + output_root = output.resolve() + if ( + output_root == source_root + or output_root.is_relative_to(source_root) + or source_root.is_relative_to(output_root) + ): + raise ReleaseError("build output and source must be separate directory trees") + + if output.exists(): + if output.is_symlink() or not output.is_dir(): + raise ReleaseError(f"build output must be a directory: {output}") + shutil.rmtree(output) + output.mkdir(parents=True) + for path in iter_files(source): + target = output / path.relative_to(source) + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(path, target) + target.chmod(0o644) + (output / SOURCE_MARKER).write_text( + json.dumps({"revision": revision}, separators=(",", ":")) + "\n", + encoding="utf-8", + ) + load_source_marker(output) + + +def package(directory: Path, archive: Path, checksum: Path) -> str: + load_source_marker(directory) + release_root = directory.resolve() + if archive.resolve().is_relative_to(release_root) or checksum.resolve().is_relative_to(release_root): + raise ReleaseError("archive and checksum must be written outside the release directory") + if archive.resolve() == checksum.resolve(): + raise ReleaseError("archive and checksum paths must differ") + archive.parent.mkdir(parents=True, exist_ok=True) + with archive.open("wb") as raw: + with gzip.GzipFile(filename="", mode="wb", fileobj=raw, mtime=0) as compressed: + with tarfile.open(fileobj=compressed, mode="w", format=tarfile.PAX_FORMAT) as tar: + for path in iter_files(directory): + relative = path.relative_to(directory).as_posix() + info = tar.gettarinfo(str(path), arcname=relative) + info.uid = info.gid = 0 + info.uname = info.gname = "" + info.mode = 0o644 + info.mtime = 0 + with path.open("rb") as source: + tar.addfile(info, source) + digest = "sha256:" + hashlib.sha256(archive.read_bytes()).hexdigest() + checksum.write_text(f"{digest} {archive.name}\n", encoding="ascii") + return digest + + +def expected_archive_digest(checksum: Path, archive: Path) -> str: + try: + fields = checksum.read_text(encoding="ascii").strip().split() + except OSError as exc: + raise ReleaseError(f"cannot read checksum: {exc}") from exc + if len(fields) != 2 or fields[1] != archive.name: + raise ReleaseError("checksum file must contain 'sha256: '") + validate_digest(fields[0]) + return fields[0] + + +def inspect_archive( + archive: Path, + checksum: Path, + revision: str, +) -> tuple[bytes, list[tarfile.TarInfo], str]: + validate_revision(revision) + expected = expected_archive_digest(checksum, archive) + try: + if archive.stat().st_size > MAX_BYTES: + raise ReleaseError("compressed archive exceeds the size limit") + archive_data = archive.read_bytes() + except OSError as exc: + raise ReleaseError(f"cannot read archive: {exc}") from exc + actual = "sha256:" + hashlib.sha256(archive_data).hexdigest() + if actual != expected: + raise ReleaseError("archive digest mismatch") + + members: list[tarfile.TarInfo] = [] + member_names: set[str] = set() + total = 0 + try: + with tarfile.open(fileobj=io.BytesIO(archive_data), mode="r:gz") as tar: + for member in tar.getmembers(): + path = PurePosixPath(member.name) + if path.is_absolute() or not path.parts or any(part in ("", ".", "..") for part in path.parts): + raise ReleaseError(f"unsafe archive path: {member.name!r}") + validate_site_path(path) + if not member.isfile(): + raise ReleaseError(f"archive may contain regular files only: {member.name!r}") + if member.name in member_names: + raise ReleaseError(f"duplicate archive path: {member.name!r}") + total += member.size + members.append(member) + member_names.add(member.name) + if len(members) > MAX_FILES or total > MAX_BYTES: + raise ReleaseError("archive exceeds the file-count or uncompressed-size limit") + except (OSError, tarfile.TarError) as exc: + raise ReleaseError(f"invalid archive: {exc}") from exc + + names = {member.name for member in members} + if "index.html" not in names or SOURCE_MARKER.as_posix() not in names: + raise ReleaseError("archive must contain index.html and .release-source.json") + if PUBLIC_MARKER.as_posix() in names: + raise ReleaseError("artifact must not provide the publisher-owned public marker") + with tempfile.TemporaryDirectory() as temp: + destination = Path(temp) + extract_members(archive_data, members, destination) + marker = load_source_marker(destination) + if marker["revision"] != revision: + raise ReleaseError("archive revision does not match the requested revision") + return archive_data, members, actual + + +def extract_members(archive_data: bytes, members: list[tarfile.TarInfo], destination: Path) -> None: + with tarfile.open(fileobj=io.BytesIO(archive_data), mode="r:gz") as tar: + by_name = {member.name: member for member in tar.getmembers()} + for inspected in members: + member = by_name.get(inspected.name) + if member is None or not member.isfile() or member.size != inspected.size: + raise ReleaseError("archive changed while it was being inspected") + target = destination.joinpath(*PurePosixPath(member.name).parts) + target.parent.mkdir(parents=True, exist_ok=True) + source = tar.extractfile(member) + if source is None: + raise ReleaseError(f"cannot extract archive member: {member.name}") + with source, target.open("wb") as output: + shutil.copyfileobj(source, output) + target.chmod(0o644) + + +def validate_verify_url(url: str) -> None: + parsed = urllib.parse.urlsplit(url) + if parsed.scheme not in {"http", "https"} or not parsed.netloc or parsed.query or parsed.fragment: + raise ReleaseError("verify URL must be an HTTP(S) marker URL without query or fragment") + if not parsed.path.endswith("/.well-known/release.json"): + raise ReleaseError("verify URL must end with /.well-known/release.json") + + +def read_url_json(url: str, timeout: float) -> dict: + request = urllib.request.Request(url, headers={"Cache-Control": "no-cache"}) + try: + with urllib.request.urlopen(request, timeout=timeout) as response: + if response.status != 200: + raise ReleaseError(f"public marker returned HTTP {response.status}") + return json.loads(response.read().decode("utf-8")) + except (OSError, urllib.error.URLError, json.JSONDecodeError) as exc: + raise ReleaseError(f"public marker verification failed: {exc}") from exc + + +def verify_public(url: str, revision: str, digest: str, timeout: float = 10.0) -> dict: + validate_revision(revision) + validate_digest(digest) + validate_verify_url(url) + marker = read_url_json(url, timeout) + if marker.get("revision") != revision or marker.get("artifact_digest") != digest: + raise ReleaseError("public marker does not identify the expected release") + + index_url = url[: -len(PUBLIC_MARKER.as_posix())] + "index.html" + try: + with urllib.request.urlopen( + urllib.request.Request(index_url, headers={"Cache-Control": "no-cache"}), + timeout=timeout, + ) as response: + if response.status != 200 or not response.read(1): + raise ReleaseError("public index is missing or empty") + except (OSError, urllib.error.URLError) as exc: + raise ReleaseError(f"public index verification failed: {exc}") from exc + return marker + + +@contextlib.contextmanager +def release_lock(root: Path): + if ( + not root.is_absolute() + or root == Path("/") + or root.resolve(strict=False) != root + or not root.is_dir() + or root.is_symlink() + ): + raise ReleaseError(f"release root must be a provisioned directory: {root}") + with (root / ".deploy.lock").open("a+") as handle: + fcntl.flock(handle, fcntl.LOCK_EX) + yield + + +def current_release(root: Path) -> dict | None: + pointer = root / "current" + if not pointer.exists() and not pointer.is_symlink(): + return None + if not pointer.is_symlink(): + raise ReleaseError(f"current target must be a managed symlink: {pointer}") + resolved = pointer.resolve(strict=True) + releases = (root / "releases").resolve() + try: + relative = resolved.relative_to(releases) + except ValueError as exc: + raise ReleaseError("current pointer escapes the managed releases directory") from exc + if len(relative.parts) != 1 or not re.fullmatch(r"[0-9a-f]{64}", relative.name): + raise ReleaseError("current pointer has an invalid release target") + marker = load_public_marker(resolved) + if marker["artifact_digest"] != f"sha256:{relative.name}": + raise ReleaseError("current pointer and public marker digest disagree") + return marker + + +def switch(root: Path, digest: str) -> None: + digest_hex = digest.removeprefix("sha256:") + temporary = root / f".current.next-{os.getpid()}" + if temporary.exists() or temporary.is_symlink(): + temporary.unlink() + temporary.symlink_to(Path("releases") / digest_hex) + os.replace(temporary, root / "current") + + +def record( + root: Path, + action: str, + revision: str, + digest: str, + previous: dict | None, + outcome: str, + actor: str | None, + order: dict | None = None, +) -> None: + entry = { + "at": dt.datetime.now(dt.timezone.utc).isoformat().replace("+00:00", "Z"), + "action": action, + "actor": actor or os.environ.get("GITHUB_ACTOR") or os.environ.get("USER") or "unknown", + "artifact_digest": digest, + "revision": revision, + "previous_revision": previous["revision"] if previous else None, + "previous_artifact_digest": previous["artifact_digest"] if previous else None, + "target": str(root / "current"), + "outcome": outcome, + "desired_commit": order["desired_commit"] if order else None, + "desired_generation": order["desired_generation"] if order else None, + } + with (root / ".deployments.jsonl").open("a", encoding="utf-8") as handle: + handle.write(json.dumps(entry, sort_keys=True) + "\n") + + +def promote( + root: Path, + revision: str, + digest: str, + *, + verify_url: str | None, + action: str, + actor: str | None, + order: dict | None = None, +) -> None: + validate_revision(revision) + validate_digest(digest) + release = root / "releases" / digest.removeprefix("sha256:") + if not release.is_dir(): + raise ReleaseError(f"release is not staged: {digest}") + source = load_source_marker(release) + marker = load_public_marker(release) + if source["revision"] != revision or marker != {"revision": revision, "artifact_digest": digest}: + raise ReleaseError("staged release identity mismatch") + + previous = current_release(root) + switch(root, digest) + try: + if verify_url: + verify_public(verify_url, revision, digest) + record(root, action, revision, digest, previous, "verified", actor, order) + except BaseException: + if previous is None: + (root / "current").unlink(missing_ok=True) + else: + switch(root, previous["artifact_digest"]) + record(root, action, revision, digest, previous, "rolled-back", actor, order) + raise + if order is not None: + write_desired_state(root, revision, digest, order) + + +def publish( + archive: Path, + checksum: Path, + root: Path, + revision: str, + verify_url: str | None, + actor: str | None = None, + order: dict | None = None, +) -> None: + if order is not None: + validate_order(order) + archive_data, members, digest = inspect_archive(archive, checksum, revision) + with release_lock(root): + if order is not None: + try: + check_order(root, revision, digest, order) + except SupersededError: + record(root, "publish", revision, digest, current_release(root), "superseded", actor, order) + raise + releases = root / "releases" + releases.mkdir(exist_ok=True) + destination = releases / digest.removeprefix("sha256:") + if destination.exists(): + marker = load_public_marker(destination) + if marker != {"revision": revision, "artifact_digest": digest}: + raise ReleaseError("existing immutable release has the wrong identity") + else: + with tempfile.TemporaryDirectory(prefix=".incoming-", dir=releases) as temp: + incoming = Path(temp) + extract_members(archive_data, members, incoming) + source = load_source_marker(incoming) + if source["revision"] != revision: + raise ReleaseError("extracted release revision mismatch") + marker_path = incoming / PUBLIC_MARKER + marker_path.parent.mkdir(parents=True, exist_ok=True) + marker_path.write_text( + json.dumps({"revision": revision, "artifact_digest": digest}, separators=(",", ":")) + "\n", + encoding="utf-8", + ) + load_public_marker(incoming) + os.replace(incoming, destination) + promote( + root, + revision, + digest, + verify_url=verify_url, + action="publish", + actor=actor, + order=order, + ) + + +def rollback( + root: Path, + revision: str, + digest: str, + verify_url: str | None, + actor: str | None = None, +) -> None: + with release_lock(root): + promote(root, revision, digest, verify_url=verify_url, action="rollback", actor=actor) + + +def retention(root: Path, keep: int) -> dict: + """Report releases outside the retention set. Never deletes anything. + + Kept: the current release, the last applied desired selection, and the + ``keep`` most recent distinct verified releases from the deployment log. + """ + if keep < 1: + raise ReleaseError("keep must be at least 1") + releases = root / "releases" + present = sorted( + path.name for path in releases.iterdir() + if path.is_dir() and re.fullmatch(r"[0-9a-f]{64}", path.name) + ) if releases.is_dir() else [] + reasons: dict[str, list[str]] = {} + + def keep_digest(digest: str | None, reason: str) -> None: + if digest: + reasons.setdefault(digest.removeprefix("sha256:"), []).append(reason) + + current = current_release(root) + keep_digest(current and current["artifact_digest"], "current") + state = load_desired_state(root) + keep_digest(state and state["artifact_digest"], "desired-state") + verified: list[str] = [] + log = root / ".deployments.jsonl" + if log.is_file(): + for line in reversed(log.read_text(encoding="utf-8").splitlines()): + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + digest = entry.get("artifact_digest") + if entry.get("outcome") == "verified" and digest not in verified: + verified.append(digest) + for digest in verified[:keep]: + keep_digest(digest, "recent-verified") + return { + "root": str(root), + "keep": {digest: reasons[digest] for digest in present if digest in reasons}, + "candidates": [digest for digest in present if digest not in reasons], + } + + +def parser() -> argparse.ArgumentParser: + command = argparse.ArgumentParser(description=__doc__) + sub = command.add_subparsers(dest="command", required=True) + + build_cmd = sub.add_parser("build") + build_cmd.add_argument("--source", type=Path, required=True) + build_cmd.add_argument("--output", type=Path, required=True) + build_cmd.add_argument("--revision", required=True) + + package_cmd = sub.add_parser("package") + package_cmd.add_argument("--directory", type=Path, required=True) + package_cmd.add_argument("--archive", type=Path, required=True) + package_cmd.add_argument("--checksum", type=Path, required=True) + + inspect_cmd = sub.add_parser("inspect") + inspect_cmd.add_argument("--archive", type=Path, required=True) + inspect_cmd.add_argument("--checksum", type=Path, required=True) + inspect_cmd.add_argument("--revision", required=True) + + publish_cmd = sub.add_parser("publish") + publish_cmd.add_argument("--archive", type=Path, required=True) + publish_cmd.add_argument("--checksum", type=Path, required=True) + publish_cmd.add_argument("--root", type=Path, required=True) + publish_cmd.add_argument("--revision", required=True) + publish_cmd.add_argument("--verify-url") + publish_cmd.add_argument("--actor") + + verify_cmd = sub.add_parser("verify") + verify_cmd.add_argument("--url", required=True) + verify_cmd.add_argument("--revision", required=True) + verify_cmd.add_argument("--digest", required=True) + + rollback_cmd = sub.add_parser("rollback") + rollback_cmd.add_argument("--root", type=Path, required=True) + rollback_cmd.add_argument("--revision", required=True) + rollback_cmd.add_argument("--digest", required=True) + rollback_cmd.add_argument("--verify-url") + rollback_cmd.add_argument("--actor") + + retention_cmd = sub.add_parser("retention", help="report prune candidates; never deletes") + retention_cmd.add_argument("--root", type=Path, required=True) + retention_cmd.add_argument("--keep", type=int, default=5) + return command + + +def main() -> int: + args = parser().parse_args() + try: + if args.command == "build": + build(args.source, args.output, args.revision) + elif args.command == "package": + package(args.directory, args.archive, args.checksum) + elif args.command == "inspect": + inspect_archive(args.archive, args.checksum, args.revision) + elif args.command == "publish": + publish(args.archive, args.checksum, args.root, args.revision, args.verify_url, args.actor) + elif args.command == "verify": + verify_public(args.url, args.revision, args.digest) + elif args.command == "rollback": + rollback(args.root, args.revision, args.digest, args.verify_url, args.actor) + elif args.command == "retention": + print(json.dumps(retention(args.root, args.keep), indent=2, sort_keys=True)) + except ReleaseError as exc: + print(f"error: {exc}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/delivery/site.json b/delivery/site.json new file mode 100644 index 0000000..4620a8e --- /dev/null +++ b/delivery/site.json @@ -0,0 +1 @@ +{"schema":1,"name":"blog","owner":"public","package":"blog-site","target":"/srv/libretech-static/blog","verify_url":"https://blog.static.librete.ch/.well-known/release.json"} diff --git a/delivery/tests/test_blog_delivery.py b/delivery/tests/test_blog_delivery.py new file mode 100644 index 0000000..6d006c6 --- /dev/null +++ b/delivery/tests/test_blog_delivery.py @@ -0,0 +1,104 @@ +from __future__ import annotations + +import hashlib +import io +import json +from pathlib import Path +import tarfile +import tempfile +import unittest + +from scripts import deploy_contract +from scripts import delivery_gate +from scripts import fetch_hugo +from scripts import generic_package + + +DELIVERY = Path(__file__).resolve().parents[1] +REPO_ROOT = DELIVERY.parent +WORKFLOWS = REPO_ROOT / ".gitea/workflows" +REVISION = "a" * 40 + + +class BlogSiteTests(unittest.TestCase): + def test_site_is_the_netcup_release_root_not_a_checkout(self): + site = deploy_contract.load_site(DELIVERY / "site.json") + self.assertEqual(site.target, "/srv/libretech-static/blog") + self.assertEqual(site.verify_url, "https://blog.static.librete.ch/.well-known/release.json") + url = generic_package.package_file_url( + deploy_contract.REGISTRY, "public", "blog-site", REVISION, generic_package.ARCHIVE_NAME, + ) + request = deploy_contract.validate_request( + site=site, + revision=REVISION, + artifact_url=url, + artifact_digest="sha256:" + "b" * 64, + target=site.target, + verify_url=site.verify_url, + desired_commit="c" * 40, + desired_generation="1", + ) + self.assertEqual(request["artifact_url"], url) + with self.assertRaisesRegex(deploy_contract.ContractError, "immutable project package"): + deploy_contract.validate_request(**{**request, "artifact_url": url.replace("public", "libretech")}, site=site) + + def test_reviewed_gate_enables_publication_and_deployment(self): + config = DELIVERY / ".delivery/config.json" + for capability in delivery_gate.CAPABILITIES: + delivery_gate.check(config, capability) + + def test_hugo_is_pinned_and_verified(self): + self.assertRegex(fetch_hugo.SHA256, r"^[0-9a-f]{64}$") + self.assertIn(f"/v{fetch_hugo.VERSION}/", fetch_hugo.URL) + with tempfile.TemporaryDirectory() as temp: + buffer = io.BytesIO() + with tarfile.open(fileobj=buffer, mode="w:gz") as tar: + info = tarfile.TarInfo("hugo") + info.size = 4 + tar.addfile(info, io.BytesIO(b"\x7fELF")) + archive = buffer.getvalue() + binary = fetch_hugo.extract_hugo(archive, hashlib.sha256(archive).hexdigest(), Path(temp)) + self.assertEqual(binary.read_bytes(), b"\x7fELF") + with self.assertRaisesRegex(fetch_hugo.FetchError, "digest mismatch"): + fetch_hugo.extract_hugo(archive, "0" * 64, Path(temp)) + + +class BlogWorkflowTests(unittest.TestCase): + def read(self, name): + return (WORKFLOWS / name).read_text(encoding="utf-8") + + def test_no_workflow_publishes_from_a_checkout(self): + for workflow in sorted(WORKFLOWS.glob("*.yml")): + text = workflow.read_text(encoding="utf-8") + with self.subTest(workflow=workflow.name): + for forbidden in ("rsync", "scp ", "publishDir", "/var/www"): + self.assertNotIn(forbidden, text) + + def test_publication_is_gated_before_any_package_write(self): + workflow = self.read("publish-blog.yml") + _, _, publish = workflow.partition("\n publish:\n") + self.assertIn("vars.BLOG_PACKAGE_PUBLISH_ENABLED == 'true'", publish) + self.assertLess(publish.index("delivery_gate.py package_publish"), publish.index("generic_package.py")) + _, _, validate = workflow.partition("\n validate:\n") + self.assertNotIn("secrets.", validate.partition("\n publish:\n")[0]) + + def test_deployment_is_gated_ordered_and_site_scoped(self): + workflow = self.read("deploy-blog.yml") + self.assertIn("workflow_dispatch:", workflow) + self.assertNotIn("\n push:", workflow) + self.assertIn("vars.BLOG_STATIC_DEPLOY_ENABLED == 'true'", workflow) + self.assertNotIn("DONATELLA", workflow) + self.assertEqual(workflow.count("--site-config delivery/site.json"), 2) + for field in ("revision", "artifact_url", "artifact_digest", "target", "verify_url", + "desired_commit", "desired_generation"): + self.assertIn(f" {field}:\n", workflow) + self.assertIn("StrictHostKeyChecking yes", workflow) + self.assertIn("steps.receive.outputs.superseded == 'false'", workflow) + self.assertLess( + workflow.index("delivery_gate.py static_deploy"), + workflow.index("ssh static-release-target"), + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/public/categories/index.xml b/public/categories/index.xml deleted file mode 100644 index c7abb16..0000000 --- a/public/categories/index.xml +++ /dev/null @@ -1,11 +0,0 @@ - - - - Categories on LibreTECH - https://blog.librete.ch/categories/ - Recent content in Categories on LibreTECH - Hugo - en-us - - - diff --git a/public/index.xml b/public/index.xml deleted file mode 100644 index 6aa8293..0000000 --- a/public/index.xml +++ /dev/null @@ -1,11 +0,0 @@ - - - - LibreTECH - https://blog.librete.ch/ - Recent content on LibreTECH - Hugo - en-us - - - diff --git a/public/sitemap.xml b/public/sitemap.xml deleted file mode 100644 index 149be8a..0000000 --- a/public/sitemap.xml +++ /dev/null @@ -1,11 +0,0 @@ - - - - https://blog.librete.ch/categories/ - - https://blog.librete.ch/ - - https://blog.librete.ch/tags/ - - diff --git a/public/tags/index.xml b/public/tags/index.xml deleted file mode 100644 index 3ac3d12..0000000 --- a/public/tags/index.xml +++ /dev/null @@ -1,11 +0,0 @@ - - - - Tags on LibreTECH - https://blog.librete.ch/tags/ - Recent content in Tags on LibreTECH - Hugo - en-us - - -