public/blog
2
0
Fork 0
Code Pull Requests Actions Wiki Activity

wip: uncommitted work on kraftwerk, saved 9 Oct 2026 (netcup issue 86)

Browse Source
This commit is contained in:
libretech
2026-10-09 01:11:32 +02:00
parent 753701ca64
commit dc74380fb7
19 changed files with 1740 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitea/workflows/blog-ci.yml
+26
View File
@@ -0,0 +1,26 @@
name: Blog release
on:
pull_request:
branches: [main]
permissions:
contents: read
jobs:
release:
runs-on: ubuntu-latest
container:
image: git.librete.ch/public/runner-image:v0.2.0@sha256:f60c587d3c0b0aac04a572db5349e27672bf76baec2ce547a3dcc28cebcf1b7e
timeout-minutes: 15
steps:
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262
with:
fetch-depth: 0
submodules: true
- name: Test the release contract
run: cd delivery && python3 -m unittest discover -s tests -v
- name: Build, package and inspect the immutable release
run: delivery/build.sh
.gitea/workflows/deploy-blog.yml
+161
View File
@@ -0,0 +1,161 @@
name: Deploy blog immutable package
on:
workflow_dispatch:
inputs:
revision:
description: Full source commit
required: true
artifact_url:
description: Immutable Gitea generic-package URL
required: true
artifact_digest:
description: sha256 digest of site.tar.gz
required: true
target:
description: Provisioned site-scoped release root
required: true
verify_url:
description: Public release marker URL
required: true
desired_commit:
description: gitops-sandbox commit that selected this release
required: true
desired_generation:
description: Ancestor count of desired_commit; the receiver refuses older selections
required: true
permissions:
contents: read
# Best-effort only. Ordering is enforced by the receiver, which applies a
# selection only if its generation is not older than the one already live, so
# a stale dispatch or a re-run of an old run cannot replace a newer release.
concurrency:
group: blog-deploy
cancel-in-progress: false
jobs:
contract:
runs-on: ubuntu-latest
container:
image: git.librete.ch/public/runner-image:v0.2.0@sha256:f60c587d3c0b0aac04a572db5349e27672bf76baec2ce547a3dcc28cebcf1b7e
timeout-minutes: 5
steps:
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262
- name: Validate dispatch contract without network access
env:
REVISION: ${{ inputs.revision }}
ARTIFACT_URL: ${{ inputs.artifact_url }}
ARTIFACT_DIGEST: ${{ inputs.artifact_digest }}
TARGET: ${{ inputs.target }}
VERIFY_URL: ${{ inputs.verify_url }}
DESIRED_COMMIT: ${{ inputs.desired_commit }}
DESIRED_GENERATION: ${{ inputs.desired_generation }}
run: |
set -eu
python3 delivery/scripts/deploy_contract.py validate \
--revision "$REVISION" \
--artifact-url "$ARTIFACT_URL" \
--artifact-digest "$ARTIFACT_DIGEST" \
--target "$TARGET" \
--verify-url "$VERIFY_URL" \
--site-config delivery/site.json \
--desired-commit "$DESIRED_COMMIT" \
--desired-generation "$DESIRED_GENERATION"
deploy:
needs: contract
if: ${{ vars.BLOG_STATIC_DEPLOY_ENABLED == 'true' }}
runs-on: ubuntu-latest
container:
image: git.librete.ch/public/runner-image:v0.2.0@sha256:f60c587d3c0b0aac04a572db5349e27672bf76baec2ce547a3dcc28cebcf1b7e
timeout-minutes: 30
env:
ACTIVATION: ${{ vars.BLOG_STATIC_DEPLOY_ENABLED }}
REVISION: ${{ inputs.revision }}
ARTIFACT_URL: ${{ inputs.artifact_url }}
ARTIFACT_DIGEST: ${{ inputs.artifact_digest }}
TARGET: ${{ inputs.target }}
VERIFY_URL: ${{ inputs.verify_url }}
DESIRED_COMMIT: ${{ inputs.desired_commit }}
DESIRED_GENERATION: ${{ inputs.desired_generation }}
STATIC_PACKAGE_READ_USER: ${{ secrets.BLOG_PACKAGE_READ_USER }}
STATIC_PACKAGE_READ_TOKEN: ${{ secrets.BLOG_PACKAGE_READ_TOKEN }}
STATIC_DEPLOY_HOST: ${{ secrets.BLOG_STATIC_DEPLOY_HOST }}
STATIC_DEPLOY_USER: ${{ secrets.BLOG_STATIC_DEPLOY_USER }}
STATIC_DEPLOY_SSH_KEY: ${{ secrets.BLOG_STATIC_DEPLOY_SSH_KEY }}
STATIC_DEPLOY_KNOWN_HOSTS: ${{ secrets.BLOG_STATIC_DEPLOY_KNOWN_HOSTS }}
steps:
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262
- name: Enforce reviewed deployment gate
run: |
set -eu
test "$ACTIVATION" = true
python3 delivery/scripts/delivery_gate.py static_deploy \
--config delivery/.delivery/config.json
test -n "$STATIC_DEPLOY_HOST"
test -n "$STATIC_DEPLOY_USER"
test -n "$STATIC_DEPLOY_SSH_KEY"
test -n "$STATIC_DEPLOY_KNOWN_HOSTS"
case "$STATIC_DEPLOY_HOST" in *[!A-Za-z0-9.-]*|'') exit 1;; esac
case "$STATIC_DEPLOY_USER" in *[!A-Za-z0-9_-]*|'') exit 1;; esac
- name: Download and inspect the declared immutable artifact
run: |
set -eu
python3 delivery/scripts/deploy_contract.py prepare \
--revision "$REVISION" \
--artifact-url "$ARTIFACT_URL" \
--artifact-digest "$ARTIFACT_DIGEST" \
--target "$TARGET" \
--verify-url "$VERIFY_URL" \
--site-config delivery/site.json \
--desired-commit "$DESIRED_COMMIT" \
--desired-generation "$DESIRED_GENERATION" \
--archive delivery/.build/site.tar.gz \
--checksum delivery/.build/site.tar.gz.sha256 \
--request-token delivery/.build/deploy-request.token
- name: Send the artifact to the site-scoped forced command
id: receive
run: |
set -eu
umask 077
mkdir -p "$HOME/.ssh"
printf '%s\n' "$STATIC_DEPLOY_SSH_KEY" > "$HOME/.ssh/id_static_deploy"
printf '%s\n' "$STATIC_DEPLOY_KNOWN_HOSTS" > "$HOME/.ssh/known_hosts"
cat > "$HOME/.ssh/config" <<CONFIG
Host static-release-target
HostName $STATIC_DEPLOY_HOST
User $STATIC_DEPLOY_USER
IdentityFile $HOME/.ssh/id_static_deploy
UserKnownHostsFile $HOME/.ssh/known_hosts
IdentitiesOnly yes
BatchMode yes
StrictHostKeyChecking yes
ConnectTimeout 10
CONFIG
request="$(cat delivery/.build/deploy-request.token)"
status=0
ssh static-release-target "static-release-receive $request" \
< delivery/.build/site.tar.gz || status=$?
# 3: the receiver already applied a newer selection; nothing changed.
if [ "$status" -eq 3 ]; then
echo "Superseded: generation $DESIRED_GENERATION ($DESIRED_COMMIT) is older than the live selection; nothing deployed."
printf 'superseded=true\n' >> "$GITHUB_OUTPUT"
exit 0
fi
test "$status" -eq 0
printf 'superseded=false\n' >> "$GITHUB_OUTPUT"
- name: Confirm public revision and digest
if: ${{ steps.receive.outputs.superseded == 'false' }}
run: |
set -eu
python3 delivery/scripts/static_release.py verify \
--url "$VERIFY_URL" \
--revision "$REVISION" \
--digest "$ARTIFACT_DIGEST"
.gitea/workflows/publish-blog.yml
+96
View File
@@ -0,0 +1,96 @@
name: Publish blog immutable package
# Every main push builds and tests the release. Publication additionally needs
# the activation variable, the reviewed repository gate and the scoped package
# secrets; without them nothing is published. Publication never deploys:
# selecting the release is a reviewed change to gitops-sandbox/stacks.yml.
on:
push:
branches: [main]
workflow_dispatch:
inputs:
revision:
description: Full public/blog commit to publish
required: true
permissions:
contents: read
concurrency:
group: blog-publish
cancel-in-progress: false
jobs:
validate:
runs-on: ubuntu-latest
container:
image: git.librete.ch/public/runner-image:v0.2.0@sha256:f60c587d3c0b0aac04a572db5349e27672bf76baec2ce547a3dcc28cebcf1b7e
timeout-minutes: 15
env:
REQUESTED_REVISION: ${{ inputs.revision || github.sha }}
steps:
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262
with:
ref: ${{ inputs.revision || github.sha }}
fetch-depth: 0
submodules: true
- name: Validate requested source, tests and build
run: |
set -eu
test "$(git rev-parse HEAD)" = "$REQUESTED_REVISION"
(cd delivery && python3 -m unittest discover -s tests -v)
delivery/build.sh
publish:
needs: validate
if: ${{ vars.BLOG_PACKAGE_PUBLISH_ENABLED == 'true' }}
runs-on: ubuntu-latest
container:
image: git.librete.ch/public/runner-image:v0.2.0@sha256:f60c587d3c0b0aac04a572db5349e27672bf76baec2ce547a3dcc28cebcf1b7e
timeout-minutes: 15
env:
ACTIVATION: ${{ vars.BLOG_PACKAGE_PUBLISH_ENABLED }}
REQUESTED_REVISION: ${{ inputs.revision || github.sha }}
STATIC_PACKAGE_USER: ${{ secrets.BLOG_PACKAGE_USER }}
STATIC_PACKAGE_TOKEN: ${{ secrets.BLOG_PACKAGE_TOKEN }}
steps:
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262
with:
ref: ${{ inputs.revision || github.sha }}
fetch-depth: 0
submodules: true
- name: Enforce reviewed publication gate
run: |
set -eu
test "$ACTIVATION" = true
python3 delivery/scripts/delivery_gate.py package_publish \
--config delivery/.delivery/config.json
test -n "$STATIC_PACKAGE_USER"
test -n "$STATIC_PACKAGE_TOKEN"
- name: Build and publish immutable package files
run: |
set -eu
test "$(git rev-parse HEAD)" = "$REQUESTED_REVISION"
delivery/build.sh
python3 delivery/scripts/generic_package.py \
--registry "${{ gitea.server_url }}" \
--owner public \
--package blog-site \
--revision "$REQUESTED_REVISION" \
--archive delivery/.build/site.tar.gz \
--checksum delivery/.build/site.tar.gz.sha256 \
--coordinate delivery/.build/release-coordinate.json \
> delivery/.build/published.json
cat delivery/.build/published.json
echo "Select this release with a reviewed gitops-sandbox stacks.yml change:"
python3 - delivery/.build/published.json <<'PY'
import json, sys
c = json.load(open(sys.argv[1]))
print(" blog:")
for key in ("revision", "artifact_url", "artifact_digest"):
print(f" {key}: {c[key]}")
PY
.gitignore
+7
View File
@@ -0,0 +1,7 @@
# Generated output and caches. Releases are immutable packages built in CI;
# nothing here is published from a checkout.
/public/
/resources/_gen/
.hugo_build.lock
/delivery/.build/
__pycache__/
README.md
+47
View File
@@ -0,0 +1,47 @@
# blog
Hugo source for the LibreTECH blog with the
[Pickles](https://github.com/mismith0227/hugo_theme_pickles) theme as a pinned
submodule.
```sh
git clone --recurse-submodules https://git.librete.ch/public/blog.git
hugo server # local preview
```
## Releases
The blog is published as an immutable release, never from a checkout. The
served tree contains only the built site and its release marker: no source,
`.git`, credentials or links outside the release.
1. `delivery/build.sh` builds the checked-out commit with the pinned Hugo
release (verified by SHA-256) and packages it deterministically as
`delivery/.build/site.tar.gz`. A rebuild of the same commit has the same
digest.
2. **Publish blog immutable package** (`publish-blog.yml`) runs on every `main`
push. With `BLOG_PACKAGE_PUBLISH_ENABLED=true`, the reviewed
`package_publish` gate and the `BLOG_PACKAGE_USER`/`BLOG_PACKAGE_TOKEN`
secrets, it publishes `public/blog-site/<revision>/` to the Gitea package
registry and prints the `stacks.yml` coordinate.
3. A reviewed pull request in
[`libretech/gitops-sandbox`](https://git.librete.ch/libretech/gitops-sandbox)
puts that coordinate in the `blog` record. Merging it dispatches
**Deploy blog immutable package** (`deploy-blog.yml`).
4. The deploy workflow re-downloads and inspects the package, then streams it
to the site-scoped receiver on Netcup, which activates
`/srv/libretech-static/blog/current` atomically and verifies
`https://blog.static.librete.ch/.well-known/release.json`. A stale or
re-run dispatch older than the live selection ends as *superseded* and
changes nothing.
Rollback is a reviewed `stacks.yml` change back to an earlier complete
coordinate. See the
[operations guide](https://git.librete.ch/libretech/gitops-sandbox/src/branch/main/OPERATIONS.md).
`delivery/scripts/` is a copy of the receiver modules from
`libretech/librete.ch` (`donatella/scripts/`); update them together.
`blog.librete.ch` itself is still served from the earlier Uberspace in-place
build (`publishDir` in `hugo.toml`) until its DNS cut-over; the release
workflows override `publishDir` and never write there.
delivery/.delivery/config.json
+1
View File
@@ -0,0 +1 @@
{"schema":1,"package_publish":true,"static_deploy":true}
delivery/build.sh
Executable
+28
View File
@@ -0,0 +1,28 @@
#!/bin/sh
# Build the checked-out commit into an immutable release archive:
# delivery/.build/site.tar.gz and its .sha256. Needs full Git history
# (enableGitInfo) and the theme submodule.
set -eu
cd "$(dirname "$0")/.."
revision="$(git rev-parse HEAD)"
test -f themes/hugo_theme_pickles/theme.toml
build="$PWD/delivery/.build"
rm -rf "$build"
mkdir -p "$build"
# CI uses the pinned release; HUGO may name a local binary for previews only.
hugo="${HUGO:-$(python3 delivery/scripts/fetch_hugo.py --destination "$build/bin")}"
"$hugo" version
# --destination and --cacheDir override the legacy in-place publishDir.
"$hugo" --source . --environment production \
--destination "$build/hugo" --cacheDir "$build/cache" --cleanDestinationDir
python3 delivery/scripts/static_release.py build \
--source "$build/hugo" --output "$build/dist" --revision "$revision"
python3 delivery/scripts/static_release.py package \
--directory "$build/dist" \
--archive "$build/site.tar.gz" \
--checksum "$build/site.tar.gz.sha256"
python3 delivery/scripts/static_release.py inspect \
--archive "$build/site.tar.gz" \
--checksum "$build/site.tar.gz.sha256" \
--revision "$revision"
cat "$build/site.tar.gz.sha256"
.hugo_build.lock → delivery/scripts/__init__.py
View File
delivery/scripts/delivery_gate.py
Executable
+48
View File
@@ -0,0 +1,48 @@
#!/usr/bin/env python3
"""Fail closed unless a delivery capability is enabled in reviewed config."""
from __future__ import annotations
import argparse
import json
from pathlib import Path
import sys
CAPABILITIES = {"package_publish", "static_deploy"}
class GateError(RuntimeError):
pass
def check(config_path: Path, capability: str) -> None:
if capability not in CAPABILITIES:
raise GateError(f"unsupported delivery capability: {capability}")
try:
config = json.loads(config_path.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError) as exc:
raise GateError(f"invalid delivery configuration: {exc}") from exc
if set(config) != {"schema", *CAPABILITIES} or config["schema"] != 1:
raise GateError("delivery configuration has an unsupported shape or schema")
if any(type(config[name]) is not bool for name in CAPABILITIES):
raise GateError("delivery capability values must be booleans")
if not config[capability]:
raise GateError(f"{capability} is disabled in reviewed repository configuration")
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("capability", choices=sorted(CAPABILITIES))
parser.add_argument("--config", type=Path, default=Path(".delivery/config.json"))
args = parser.parse_args()
try:
check(args.config, args.capability)
except GateError as exc:
print(f"error: {exc}", file=sys.stderr)
return 1
return 0
if __name__ == "__main__":
raise SystemExit(main())
delivery/scripts/deploy_contract.py
Executable
+246
View File
@@ -0,0 +1,246 @@
#!/usr/bin/env python3
"""Validate and prepare the project-owned static deployment request."""
from __future__ import annotations
import argparse
import base64
from dataclasses import dataclass
import json
import os
from pathlib import Path
import re
import sys
try:
from .generic_package import PackageError, package_file_url, request
from .static_release import (
ReleaseError,
inspect_archive,
validate_digest,
validate_order,
validate_revision,
)
except ImportError: # Direct script execution adds scripts/ to sys.path.
from generic_package import PackageError, package_file_url, request
from static_release import (
ReleaseError,
inspect_archive,
validate_digest,
validate_order,
validate_revision,
)
REGISTRY = "https://git.librete.ch"
OWNER = "libretech"
PACKAGE = "donatella-site"
PUBLIC_HOST = "donatella.static.librete.ch"
TARGET = "/srv/libretech-static/donatella"
SERVE_ROOT = f"{TARGET}/current"
VERIFY_URL = f"https://{PUBLIC_HOST}/.well-known/release.json"
RELEASE_FIELDS = ("revision", "artifact_url", "artifact_digest", "target", "verify_url")
ORDER_FIELDS = ("desired_commit", "desired_generation")
SITE_NAME_RE = re.compile(r"^[a-z0-9][a-z0-9-]{0,62}$")
class ContractError(RuntimeError):
pass
@dataclass(frozen=True)
class Site:
"""One enrolled static site: its package, release root and public marker."""
name: str
owner: str
package: str
target: str
verify_url: str
registry: str = REGISTRY
DONATELLA = Site("donatella", OWNER, PACKAGE, TARGET, VERIFY_URL)
def load_site(path: Path | None) -> Site:
"""Read a reviewed site.json; without one, the receiver serves Donatella."""
if path is None:
return DONATELLA
try:
document = json.loads(path.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError) as exc:
raise ContractError(f"invalid site configuration: {exc}") from exc
fields = {"schema", "name", "owner", "package", "target", "verify_url"}
if not isinstance(document, dict) or set(document) != fields or document["schema"] != 1:
raise ContractError("site configuration has an unsupported shape or schema")
site = Site(**{key: document[key] for key in fields - {"schema"}})
if not all(isinstance(value, str) for value in vars(site).values()):
raise ContractError("site configuration values must be strings")
if not SITE_NAME_RE.fullmatch(site.name) or not SITE_NAME_RE.fullmatch(site.owner):
raise ContractError("site name and owner must be lowercase identifiers")
if not SITE_NAME_RE.fullmatch(site.package):
raise ContractError("package must be a lowercase identifier")
if site.target != f"/srv/libretech-static/{site.name}":
raise ContractError("site target must be /srv/libretech-static/<name>")
if not site.verify_url.startswith("https://") or not site.verify_url.endswith(
"/.well-known/release.json",
):
raise ContractError("site verify_url must be an HTTPS release marker")
return site
def validate_contract(
*,
revision: str,
artifact_url: str,
artifact_digest: str,
target: str,
verify_url: str,
registry: str | None = None,
allow_http: bool = False,
site: Site = DONATELLA,
) -> dict[str, str]:
values = {
"revision": revision,
"artifact_url": artifact_url,
"artifact_digest": artifact_digest,
"target": target,
"verify_url": verify_url,
}
if any(not isinstance(value, str) or not value for value in values.values()):
raise ContractError("all deployment contract fields must be non-empty strings")
try:
validate_revision(revision)
validate_digest(artifact_digest)
expected_url = package_file_url(
registry or site.registry,
site.owner,
site.package,
revision,
"site.tar.gz",
allow_http=allow_http,
)
except (ReleaseError, PackageError) as exc:
raise ContractError(str(exc)) from exc
if artifact_url != expected_url:
raise ContractError("artifact URL is not the immutable project package for this revision")
if target != site.target:
raise ContractError(f"target must be exactly {site.target}")
if verify_url != site.verify_url:
raise ContractError(f"verification URL must be exactly {site.verify_url}")
return values
def validate_request(*, site: Site = DONATELLA, **fields) -> dict[str, str]:
"""Validate a release coordinate plus the GitOps selection that chose it."""
if set(fields) != set(RELEASE_FIELDS) | set(ORDER_FIELDS):
raise ContractError("deployment request must contain exactly the seven declared fields")
contract = validate_contract(site=site, **{key: fields[key] for key in RELEASE_FIELDS})
order = {key: fields[key] for key in ORDER_FIELDS}
try:
validate_order(order)
except ReleaseError as exc:
raise ContractError(str(exc)) from exc
return {**contract, **order}
def order_of(request: dict[str, str]) -> dict[str, str]:
return {key: request[key] for key in ORDER_FIELDS}
def request_token(contract: dict[str, str]) -> str:
encoded = base64.urlsafe_b64encode(
json.dumps(contract, separators=(",", ":")).encode("utf-8"),
).decode("ascii")
return encoded.rstrip("=")
def prepare(
contract: dict[str, str],
*,
archive: Path,
checksum: Path,
token_output: Path,
username: str | None = None,
package_token: str | None = None,
) -> None:
status, content = request(
"GET",
contract["artifact_url"],
username=username,
token=package_token,
)
if status != 200:
raise ContractError(f"artifact download returned HTTP {status}")
archive.parent.mkdir(parents=True, exist_ok=True)
checksum.parent.mkdir(parents=True, exist_ok=True)
token_output.parent.mkdir(parents=True, exist_ok=True)
archive.write_bytes(content)
checksum.write_text(
f"{contract['artifact_digest']} {archive.name}\n",
encoding="ascii",
)
try:
inspect_archive(archive, checksum, contract["revision"])
except ReleaseError as exc:
raise ContractError(str(exc)) from exc
token_output.write_text(request_token(contract) + "\n", encoding="ascii")
def add_contract_arguments(command: argparse.ArgumentParser) -> None:
command.add_argument("--revision", required=True)
command.add_argument("--artifact-url", required=True)
command.add_argument("--artifact-digest", required=True)
command.add_argument("--target", required=True)
command.add_argument("--verify-url", required=True)
command.add_argument("--desired-commit", required=True)
command.add_argument("--desired-generation", required=True)
command.add_argument("--site-config", type=Path)
def parser() -> argparse.ArgumentParser:
command = argparse.ArgumentParser(description=__doc__)
sub = command.add_subparsers(dest="command", required=True)
validate_cmd = sub.add_parser("validate")
add_contract_arguments(validate_cmd)
prepare_cmd = sub.add_parser("prepare")
add_contract_arguments(prepare_cmd)
prepare_cmd.add_argument("--archive", type=Path, required=True)
prepare_cmd.add_argument("--checksum", type=Path, required=True)
prepare_cmd.add_argument("--request-token", type=Path, required=True)
return command
def main() -> int:
args = parser().parse_args()
try:
contract = validate_request(
site=load_site(args.site_config),
revision=args.revision,
artifact_url=args.artifact_url,
artifact_digest=args.artifact_digest,
target=args.target,
verify_url=args.verify_url,
desired_commit=args.desired_commit,
desired_generation=args.desired_generation,
)
if args.command == "prepare":
prepare(
contract,
archive=args.archive,
checksum=args.checksum,
token_output=args.request_token,
username=os.environ.get("STATIC_PACKAGE_READ_USER") or None,
package_token=os.environ.get("STATIC_PACKAGE_READ_TOKEN") or None,
)
else:
print(json.dumps(contract, separators=(",", ":")))
except (ContractError, PackageError) as exc:
print(f"error: {exc}", file=sys.stderr)
return 1
return 0
if __name__ == "__main__":
raise SystemExit(main())
delivery/scripts/fetch_hugo.py
+63
View File
@@ -0,0 +1,63 @@
#!/usr/bin/env python3
"""Install the pinned Hugo release after verifying its published SHA-256."""
from __future__ import annotations
import argparse
import hashlib
import io
from pathlib import Path
import sys
import tarfile
import urllib.request
VERSION = "0.150.1"
URL = (
f"https://github.com/gohugoio/hugo/releases/download/v{VERSION}/"
f"hugo_extended_{VERSION}_linux-amd64.tar.gz"
)
SHA256 = "e1248fa077d99232794e38df5ec533494993aafafca1ae3e331a4ed629079ed6"
MAX_BYTES = 128 * 1024 * 1024
class FetchError(RuntimeError):
pass
def extract_hugo(archive: bytes, expected: str, destination: Path) -> Path:
actual = hashlib.sha256(archive).hexdigest()
if actual != expected:
raise FetchError(f"Hugo archive digest mismatch: {actual}")
with tarfile.open(fileobj=io.BytesIO(archive), mode="r:gz") as tar:
member = tar.getmember("hugo")
if not member.isfile():
raise FetchError("Hugo archive member is not a regular file")
source = tar.extractfile(member)
if source is None:
raise FetchError("cannot read the Hugo binary")
destination.mkdir(parents=True, exist_ok=True)
binary = destination / "hugo"
binary.write_bytes(source.read())
binary.chmod(0o755)
return binary
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--destination", type=Path, required=True)
args = parser.parse_args()
try:
with urllib.request.urlopen(URL, timeout=60) as response:
archive = response.read(MAX_BYTES + 1)
if len(archive) > MAX_BYTES:
raise FetchError("Hugo archive exceeds the size limit")
print(extract_hugo(archive, SHA256, args.destination))
except (OSError, KeyError, tarfile.TarError, FetchError) as exc:
print(f"error: {exc}", file=sys.stderr)
return 1
return 0
if __name__ == "__main__":
raise SystemExit(main())
delivery/scripts/generic_package.py
Executable
+266
View File
@@ -0,0 +1,266 @@
#!/usr/bin/env python3
"""Publish a static release as immutable files in Gitea's generic registry."""
from __future__ import annotations
import argparse
import base64
import json
import os
from pathlib import Path
import re
import sys
import urllib.error
import urllib.parse
import urllib.request
try:
from .static_release import ReleaseError, inspect_archive, validate_revision
except ImportError: # Direct script execution adds scripts/ to sys.path.
from static_release import ReleaseError, inspect_archive, validate_revision
COMPONENT_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._+-]*$")
ARCHIVE_NAME = "site.tar.gz"
CHECKSUM_NAME = "site.tar.gz.sha256"
COORDINATE_NAME = "release-coordinate.json"
MAX_RESPONSE_BYTES = 200 * 1024 * 1024
class PackageError(RuntimeError):
pass
def validate_component(value: str, label: str) -> None:
if not COMPONENT_RE.fullmatch(value):
raise PackageError(f"{label} contains unsupported characters")
def validate_registry(registry: str, *, allow_http: bool = False) -> str:
parsed = urllib.parse.urlsplit(registry)
allowed_schemes = {"https"} | ({"http"} if allow_http else set())
if (
parsed.scheme not in allowed_schemes
or not parsed.netloc
or parsed.query
or parsed.fragment
or parsed.username
or parsed.password
):
raise PackageError("registry must be an HTTPS origin without credentials, query or fragment")
return registry.rstrip("/")
def package_file_url(
registry: str,
owner: str,
package: str,
version: str,
filename: str,
*,
allow_http: bool = False,
) -> str:
registry = validate_registry(registry, allow_http=allow_http)
for value, label in ((owner, "owner"), (package, "package"), (filename, "filename")):
validate_component(value, label)
validate_revision(version)
components = [owner, package, version, filename]
owner_part, package_part, version_part, filename_part = (
urllib.parse.quote(value, safe="") for value in components
)
return (
f"{registry}/api/packages/{owner_part}/generic/"
f"{package_part}/{version_part}/{filename_part}"
)
def authorization(username: str | None, token: str | None) -> str | None:
if bool(username) != bool(token):
raise PackageError("package username and token must be supplied together")
if not username:
return None
encoded = base64.b64encode(f"{username}:{token}".encode()).decode("ascii")
return f"Basic {encoded}"
def request(
method: str,
url: str,
*,
username: str | None,
token: str | None,
data: bytes | None = None,
timeout: float = 300.0,
) -> tuple[int, bytes]:
headers = {"User-Agent": "donatella-static-publisher/1"}
auth = authorization(username, token)
if auth:
headers["Authorization"] = auth
if data is not None:
headers["Content-Type"] = "application/octet-stream"
outgoing = urllib.request.Request(url, data=data, headers=headers, method=method)
try:
with urllib.request.urlopen(outgoing, timeout=timeout) as response:
content = response.read(MAX_RESPONSE_BYTES + 1)
if len(content) > MAX_RESPONSE_BYTES:
raise PackageError("package registry response exceeds the size limit")
return response.status, content
except urllib.error.HTTPError as exc:
try:
content = exc.read(MAX_RESPONSE_BYTES + 1)
if len(content) > MAX_RESPONSE_BYTES:
raise PackageError("package registry error response exceeds the size limit")
return exc.code, content
finally:
exc.close()
except (OSError, urllib.error.URLError) as exc:
raise PackageError(f"package registry request failed: {exc}") from exc
def put_immutable(
url: str,
content: bytes,
*,
username: str,
token: str,
timeout: float = 300.0,
) -> None:
status, existing = request("GET", url, username=username, token=token, timeout=timeout)
if status == 200:
if existing != content:
raise PackageError(f"immutable package file already exists with different content: {url}")
return
if status != 404:
raise PackageError(f"package preflight returned HTTP {status}: {url}")
status, _ = request(
"PUT",
url,
username=username,
token=token,
data=content,
timeout=timeout,
)
if status not in {201, 409}:
raise PackageError(f"package upload returned HTTP {status}: {url}")
status, published = request("GET", url, username=username, token=token, timeout=timeout)
if status != 200 or published != content:
raise PackageError(f"published package file did not verify byte-for-byte: {url}")
def coordinate_bytes(revision: str, artifact_url: str, digest: str) -> bytes:
return (
json.dumps(
{
"revision": revision,
"artifact_url": artifact_url,
"artifact_digest": digest,
},
separators=(",", ":"),
)
+ "\n"
).encode("utf-8")
def publish_release(
*,
registry: str,
owner: str,
package: str,
revision: str,
archive: Path,
checksum: Path,
coordinate: Path,
username: str,
token: str,
allow_http: bool = False,
) -> dict[str, str]:
if not username or not token:
raise PackageError("package username and token are required")
if archive.name != ARCHIVE_NAME or checksum.name != CHECKSUM_NAME:
raise PackageError(f"release files must be named {ARCHIVE_NAME} and {CHECKSUM_NAME}")
try:
_, _, digest = inspect_archive(archive, checksum, revision)
except ReleaseError as exc:
raise PackageError(str(exc)) from exc
artifact_url = package_file_url(
registry,
owner,
package,
revision,
ARCHIVE_NAME,
allow_http=allow_http,
)
checksum_url = package_file_url(
registry,
owner,
package,
revision,
CHECKSUM_NAME,
allow_http=allow_http,
)
coordinate_url = package_file_url(
registry,
owner,
package,
revision,
COORDINATE_NAME,
allow_http=allow_http,
)
payload = coordinate_bytes(revision, artifact_url, digest)
if coordinate.resolve() in {archive.resolve(), checksum.resolve()}:
raise PackageError("coordinate output must be separate from the archive and checksum")
coordinate.parent.mkdir(parents=True, exist_ok=True)
coordinate.write_bytes(payload)
for url, content in (
(artifact_url, archive.read_bytes()),
(checksum_url, checksum.read_bytes()),
(coordinate_url, payload),
):
put_immutable(url, content, username=username, token=token)
return {
"revision": revision,
"artifact_url": artifact_url,
"artifact_digest": digest,
}
def parser() -> argparse.ArgumentParser:
command = argparse.ArgumentParser(description=__doc__)
command.add_argument("--registry", required=True)
command.add_argument("--owner", required=True)
command.add_argument("--package", required=True)
command.add_argument("--revision", required=True)
command.add_argument("--archive", type=Path, required=True)
command.add_argument("--checksum", type=Path, required=True)
command.add_argument("--coordinate", type=Path, required=True)
return command
def main() -> int:
args = parser().parse_args()
try:
result = publish_release(
registry=args.registry,
owner=args.owner,
package=args.package,
revision=args.revision,
archive=args.archive,
checksum=args.checksum,
coordinate=args.coordinate,
username=os.environ.get("STATIC_PACKAGE_USER", ""),
token=os.environ.get("STATIC_PACKAGE_TOKEN", ""),
)
except PackageError as exc:
print(f"error: {exc}", file=sys.stderr)
return 1
print(json.dumps(result, separators=(",", ":")))
return 0
if __name__ == "__main__":
raise SystemExit(main())
delivery/scripts/static_release.py
Executable
+646
View File
@@ -0,0 +1,646 @@
#!/usr/bin/env python3
"""Build, inspect, publish, verify and roll back trusted static releases."""
from __future__ import annotations
import argparse
import contextlib
import datetime as dt
import fcntl
import gzip
import hashlib
import io
import json
import os
from pathlib import Path, PurePosixPath
import re
import shutil
import sys
import tarfile
import tempfile
import urllib.error
import urllib.parse
import urllib.request
REVISION_RE = re.compile(r"^[0-9a-f]{40}$")
GENERATION_RE = re.compile(r"^[1-9][0-9]{0,11}$")
DIGEST_RE = re.compile(r"^sha256:[0-9a-f]{64}$")
MAX_FILES = 10_000
# The Sarah Weisman portfolio contains 164,651,446 bytes of site assets.
# Keep one bounded limit for the archive, receiver and extracted release.
MAX_BYTES = 200 * 1024 * 1024
SOURCE_MARKER = Path(".release-source.json")
PUBLIC_MARKER = Path(".well-known/release.json")
# Last desired-state selection this target applied, outside the served tree.
DESIRED_STATE = Path(".desired-state.json")
FORBIDDEN_PARTS = {".git", ".hg", ".svn"}
class ReleaseError(RuntimeError):
pass
class SupersededError(ReleaseError):
"""A newer desired-state generation has already been applied here."""
def validate_revision(revision: str) -> None:
if not REVISION_RE.fullmatch(revision):
raise ReleaseError("revision must be a full 40-character lowercase Git SHA")
def validate_digest(digest: str) -> None:
if not DIGEST_RE.fullmatch(digest):
raise ReleaseError("digest must have the form sha256:<64 lowercase hex characters>")
def validate_order(order: dict) -> dict:
"""Validate the GitOps selection that requested a release.
``desired_generation`` is the ancestor count of ``desired_commit`` on the
GitOps main branch. Main only moves to descendants, so a later selection
always has a larger generation than an earlier one.
"""
if not isinstance(order, dict) or set(order) != {"desired_commit", "desired_generation"}:
raise ReleaseError("order must contain exactly desired_commit and desired_generation")
if not isinstance(order["desired_commit"], str) or not REVISION_RE.fullmatch(order["desired_commit"]):
raise ReleaseError("desired_commit must be a full 40-character lowercase Git SHA")
if not isinstance(order["desired_generation"], str) or not GENERATION_RE.fullmatch(order["desired_generation"]):
raise ReleaseError("desired_generation must be a positive decimal integer without leading zeros")
return order
def load_desired_state(root: Path) -> dict | None:
path = root / DESIRED_STATE
if not path.exists() and not path.is_symlink():
return None
if path.is_symlink() or not path.is_file():
raise ReleaseError("desired-state record must be a regular file")
try:
state = json.loads(path.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError) as exc:
raise ReleaseError(f"invalid desired-state record: {exc}") from exc
if not isinstance(state, dict) or set(state) != {
"desired_commit", "desired_generation", "revision", "artifact_digest",
}:
raise ReleaseError("desired-state record has an unsupported shape")
validate_order({key: state[key] for key in ("desired_commit", "desired_generation")})
validate_revision(state["revision"])
validate_digest(state["artifact_digest"])
return state
def check_order(root: Path, revision: str, digest: str, order: dict) -> None:
"""Refuse a request whose selection is older than the one applied here.
An identical request (same generation, commit and release) is a retry of
the accepted selection and may be applied again.
"""
state = load_desired_state(root)
if state is None:
return
requested = int(order["desired_generation"])
applied = int(state["desired_generation"])
if requested < applied:
raise SupersededError(
f"superseded: generation {requested} ({order['desired_commit']}) is older than "
f"applied generation {applied} ({state['desired_commit']})"
)
if requested == applied and (
order["desired_commit"] != state["desired_commit"]
or revision != state["revision"]
or digest != state["artifact_digest"]
):
raise ReleaseError(
f"generation {requested} was already applied with a different selection"
)
def write_desired_state(root: Path, revision: str, digest: str, order: dict) -> None:
state = {**order, "revision": revision, "artifact_digest": digest}
temporary = root / f".desired-state.next-{os.getpid()}"
temporary.write_text(json.dumps(state, sort_keys=True) + "\n", encoding="utf-8")
os.replace(temporary, root / DESIRED_STATE)
def validate_site_path(path: PurePosixPath) -> None:
if any(part in FORBIDDEN_PARTS for part in path.parts):
raise ReleaseError(f"version-control metadata is forbidden: {path}")
filename = path.name.lower()
if filename == ".env" or filename.startswith(".env.") or filename in {"id_rsa", "id_ed25519"}:
raise ReleaseError(f"common secret filename is forbidden: {path}")
def iter_files(root: Path):
for path in sorted(root.rglob("*")):
relative = PurePosixPath(path.relative_to(root).as_posix())
validate_site_path(relative)
if path.is_symlink():
raise ReleaseError(f"symlinks are forbidden: {relative}")
if path.is_file():
yield path
elif not path.is_dir():
raise ReleaseError(f"unsupported filesystem entry: {relative}")
def validate_tree(root: Path) -> None:
count = 0
size = 0
for path in iter_files(root):
count += 1
size += path.stat().st_size
if count > MAX_FILES or size > MAX_BYTES:
raise ReleaseError("site exceeds the file-count or uncompressed-size limit")
if not (root / "index.html").is_file():
raise ReleaseError("release has no index.html")
def load_source_marker(root: Path) -> dict:
try:
marker = json.loads((root / SOURCE_MARKER).read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError) as exc:
raise ReleaseError(f"invalid or missing source marker: {exc}") from exc
if set(marker) != {"revision"}:
raise ReleaseError("source marker must contain only the revision")
validate_revision(marker["revision"])
validate_tree(root)
return marker
def load_public_marker(root: Path) -> dict:
try:
marker = json.loads((root / PUBLIC_MARKER).read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError) as exc:
raise ReleaseError(f"invalid or missing public release marker: {exc}") from exc
if set(marker) != {"revision", "artifact_digest"}:
raise ReleaseError("public marker must contain revision and artifact_digest")
validate_revision(marker["revision"])
validate_digest(marker["artifact_digest"])
return marker
def build(source: Path, output: Path, revision: str) -> None:
validate_revision(revision)
if not source.is_dir():
raise ReleaseError(f"source directory does not exist: {source}")
if (source / SOURCE_MARKER).exists() or (source / PUBLIC_MARKER).exists():
raise ReleaseError("source must not provide reserved release markers")
validate_tree(source)
source_root = source.resolve()
output_root = output.resolve()
if (
output_root == source_root
or output_root.is_relative_to(source_root)
or source_root.is_relative_to(output_root)
):
raise ReleaseError("build output and source must be separate directory trees")
if output.exists():
if output.is_symlink() or not output.is_dir():
raise ReleaseError(f"build output must be a directory: {output}")
shutil.rmtree(output)
output.mkdir(parents=True)
for path in iter_files(source):
target = output / path.relative_to(source)
target.parent.mkdir(parents=True, exist_ok=True)
shutil.copyfile(path, target)
target.chmod(0o644)
(output / SOURCE_MARKER).write_text(
json.dumps({"revision": revision}, separators=(",", ":")) + "\n",
encoding="utf-8",
)
load_source_marker(output)
def package(directory: Path, archive: Path, checksum: Path) -> str:
load_source_marker(directory)
release_root = directory.resolve()
if archive.resolve().is_relative_to(release_root) or checksum.resolve().is_relative_to(release_root):
raise ReleaseError("archive and checksum must be written outside the release directory")
if archive.resolve() == checksum.resolve():
raise ReleaseError("archive and checksum paths must differ")
archive.parent.mkdir(parents=True, exist_ok=True)
with archive.open("wb") as raw:
with gzip.GzipFile(filename="", mode="wb", fileobj=raw, mtime=0) as compressed:
with tarfile.open(fileobj=compressed, mode="w", format=tarfile.PAX_FORMAT) as tar:
for path in iter_files(directory):
relative = path.relative_to(directory).as_posix()
info = tar.gettarinfo(str(path), arcname=relative)
info.uid = info.gid = 0
info.uname = info.gname = ""
info.mode = 0o644
info.mtime = 0
with path.open("rb") as source:
tar.addfile(info, source)
digest = "sha256:" + hashlib.sha256(archive.read_bytes()).hexdigest()
checksum.write_text(f"{digest} {archive.name}\n", encoding="ascii")
return digest
def expected_archive_digest(checksum: Path, archive: Path) -> str:
try:
fields = checksum.read_text(encoding="ascii").strip().split()
except OSError as exc:
raise ReleaseError(f"cannot read checksum: {exc}") from exc
if len(fields) != 2 or fields[1] != archive.name:
raise ReleaseError("checksum file must contain 'sha256:<digest> <archive-name>'")
validate_digest(fields[0])
return fields[0]
def inspect_archive(
archive: Path,
checksum: Path,
revision: str,
) -> tuple[bytes, list[tarfile.TarInfo], str]:
validate_revision(revision)
expected = expected_archive_digest(checksum, archive)
try:
if archive.stat().st_size > MAX_BYTES:
raise ReleaseError("compressed archive exceeds the size limit")
archive_data = archive.read_bytes()
except OSError as exc:
raise ReleaseError(f"cannot read archive: {exc}") from exc
actual = "sha256:" + hashlib.sha256(archive_data).hexdigest()
if actual != expected:
raise ReleaseError("archive digest mismatch")
members: list[tarfile.TarInfo] = []
member_names: set[str] = set()
total = 0
try:
with tarfile.open(fileobj=io.BytesIO(archive_data), mode="r:gz") as tar:
for member in tar.getmembers():
path = PurePosixPath(member.name)
if path.is_absolute() or not path.parts or any(part in ("", ".", "..") for part in path.parts):
raise ReleaseError(f"unsafe archive path: {member.name!r}")
validate_site_path(path)
if not member.isfile():
raise ReleaseError(f"archive may contain regular files only: {member.name!r}")
if member.name in member_names:
raise ReleaseError(f"duplicate archive path: {member.name!r}")
total += member.size
members.append(member)
member_names.add(member.name)
if len(members) > MAX_FILES or total > MAX_BYTES:
raise ReleaseError("archive exceeds the file-count or uncompressed-size limit")
except (OSError, tarfile.TarError) as exc:
raise ReleaseError(f"invalid archive: {exc}") from exc
names = {member.name for member in members}
if "index.html" not in names or SOURCE_MARKER.as_posix() not in names:
raise ReleaseError("archive must contain index.html and .release-source.json")
if PUBLIC_MARKER.as_posix() in names:
raise ReleaseError("artifact must not provide the publisher-owned public marker")
with tempfile.TemporaryDirectory() as temp:
destination = Path(temp)
extract_members(archive_data, members, destination)
marker = load_source_marker(destination)
if marker["revision"] != revision:
raise ReleaseError("archive revision does not match the requested revision")
return archive_data, members, actual
def extract_members(archive_data: bytes, members: list[tarfile.TarInfo], destination: Path) -> None:
with tarfile.open(fileobj=io.BytesIO(archive_data), mode="r:gz") as tar:
by_name = {member.name: member for member in tar.getmembers()}
for inspected in members:
member = by_name.get(inspected.name)
if member is None or not member.isfile() or member.size != inspected.size:
raise ReleaseError("archive changed while it was being inspected")
target = destination.joinpath(*PurePosixPath(member.name).parts)
target.parent.mkdir(parents=True, exist_ok=True)
source = tar.extractfile(member)
if source is None:
raise ReleaseError(f"cannot extract archive member: {member.name}")
with source, target.open("wb") as output:
shutil.copyfileobj(source, output)
target.chmod(0o644)
def validate_verify_url(url: str) -> None:
parsed = urllib.parse.urlsplit(url)
if parsed.scheme not in {"http", "https"} or not parsed.netloc or parsed.query or parsed.fragment:
raise ReleaseError("verify URL must be an HTTP(S) marker URL without query or fragment")
if not parsed.path.endswith("/.well-known/release.json"):
raise ReleaseError("verify URL must end with /.well-known/release.json")
def read_url_json(url: str, timeout: float) -> dict:
request = urllib.request.Request(url, headers={"Cache-Control": "no-cache"})
try:
with urllib.request.urlopen(request, timeout=timeout) as response:
if response.status != 200:
raise ReleaseError(f"public marker returned HTTP {response.status}")
return json.loads(response.read().decode("utf-8"))
except (OSError, urllib.error.URLError, json.JSONDecodeError) as exc:
raise ReleaseError(f"public marker verification failed: {exc}") from exc
def verify_public(url: str, revision: str, digest: str, timeout: float = 10.0) -> dict:
validate_revision(revision)
validate_digest(digest)
validate_verify_url(url)
marker = read_url_json(url, timeout)
if marker.get("revision") != revision or marker.get("artifact_digest") != digest:
raise ReleaseError("public marker does not identify the expected release")
index_url = url[: -len(PUBLIC_MARKER.as_posix())] + "index.html"
try:
with urllib.request.urlopen(
urllib.request.Request(index_url, headers={"Cache-Control": "no-cache"}),
timeout=timeout,
) as response:
if response.status != 200 or not response.read(1):
raise ReleaseError("public index is missing or empty")
except (OSError, urllib.error.URLError) as exc:
raise ReleaseError(f"public index verification failed: {exc}") from exc
return marker
@contextlib.contextmanager
def release_lock(root: Path):
if (
not root.is_absolute()
or root == Path("/")
or root.resolve(strict=False) != root
or not root.is_dir()
or root.is_symlink()
):
raise ReleaseError(f"release root must be a provisioned directory: {root}")
with (root / ".deploy.lock").open("a+") as handle:
fcntl.flock(handle, fcntl.LOCK_EX)
yield
def current_release(root: Path) -> dict | None:
pointer = root / "current"
if not pointer.exists() and not pointer.is_symlink():
return None
if not pointer.is_symlink():
raise ReleaseError(f"current target must be a managed symlink: {pointer}")
resolved = pointer.resolve(strict=True)
releases = (root / "releases").resolve()
try:
relative = resolved.relative_to(releases)
except ValueError as exc:
raise ReleaseError("current pointer escapes the managed releases directory") from exc
if len(relative.parts) != 1 or not re.fullmatch(r"[0-9a-f]{64}", relative.name):
raise ReleaseError("current pointer has an invalid release target")
marker = load_public_marker(resolved)
if marker["artifact_digest"] != f"sha256:{relative.name}":
raise ReleaseError("current pointer and public marker digest disagree")
return marker
def switch(root: Path, digest: str) -> None:
digest_hex = digest.removeprefix("sha256:")
temporary = root / f".current.next-{os.getpid()}"
if temporary.exists() or temporary.is_symlink():
temporary.unlink()
temporary.symlink_to(Path("releases") / digest_hex)
os.replace(temporary, root / "current")
def record(
root: Path,
action: str,
revision: str,
digest: str,
previous: dict | None,
outcome: str,
actor: str | None,
order: dict | None = None,
) -> None:
entry = {
"at": dt.datetime.now(dt.timezone.utc).isoformat().replace("+00:00", "Z"),
"action": action,
"actor": actor or os.environ.get("GITHUB_ACTOR") or os.environ.get("USER") or "unknown",
"artifact_digest": digest,
"revision": revision,
"previous_revision": previous["revision"] if previous else None,
"previous_artifact_digest": previous["artifact_digest"] if previous else None,
"target": str(root / "current"),
"outcome": outcome,
"desired_commit": order["desired_commit"] if order else None,
"desired_generation": order["desired_generation"] if order else None,
}
with (root / ".deployments.jsonl").open("a", encoding="utf-8") as handle:
handle.write(json.dumps(entry, sort_keys=True) + "\n")
def promote(
root: Path,
revision: str,
digest: str,
*,
verify_url: str | None,
action: str,
actor: str | None,
order: dict | None = None,
) -> None:
validate_revision(revision)
validate_digest(digest)
release = root / "releases" / digest.removeprefix("sha256:")
if not release.is_dir():
raise ReleaseError(f"release is not staged: {digest}")
source = load_source_marker(release)
marker = load_public_marker(release)
if source["revision"] != revision or marker != {"revision": revision, "artifact_digest": digest}:
raise ReleaseError("staged release identity mismatch")
previous = current_release(root)
switch(root, digest)
try:
if verify_url:
verify_public(verify_url, revision, digest)
record(root, action, revision, digest, previous, "verified", actor, order)
except BaseException:
if previous is None:
(root / "current").unlink(missing_ok=True)
else:
switch(root, previous["artifact_digest"])
record(root, action, revision, digest, previous, "rolled-back", actor, order)
raise
if order is not None:
write_desired_state(root, revision, digest, order)
def publish(
archive: Path,
checksum: Path,
root: Path,
revision: str,
verify_url: str | None,
actor: str | None = None,
order: dict | None = None,
) -> None:
if order is not None:
validate_order(order)
archive_data, members, digest = inspect_archive(archive, checksum, revision)
with release_lock(root):
if order is not None:
try:
check_order(root, revision, digest, order)
except SupersededError:
record(root, "publish", revision, digest, current_release(root), "superseded", actor, order)
raise
releases = root / "releases"
releases.mkdir(exist_ok=True)
destination = releases / digest.removeprefix("sha256:")
if destination.exists():
marker = load_public_marker(destination)
if marker != {"revision": revision, "artifact_digest": digest}:
raise ReleaseError("existing immutable release has the wrong identity")
else:
with tempfile.TemporaryDirectory(prefix=".incoming-", dir=releases) as temp:
incoming = Path(temp)
extract_members(archive_data, members, incoming)
source = load_source_marker(incoming)
if source["revision"] != revision:
raise ReleaseError("extracted release revision mismatch")
marker_path = incoming / PUBLIC_MARKER
marker_path.parent.mkdir(parents=True, exist_ok=True)
marker_path.write_text(
json.dumps({"revision": revision, "artifact_digest": digest}, separators=(",", ":")) + "\n",
encoding="utf-8",
)
load_public_marker(incoming)
os.replace(incoming, destination)
promote(
root,
revision,
digest,
verify_url=verify_url,
action="publish",
actor=actor,
order=order,
)
def rollback(
root: Path,
revision: str,
digest: str,
verify_url: str | None,
actor: str | None = None,
) -> None:
with release_lock(root):
promote(root, revision, digest, verify_url=verify_url, action="rollback", actor=actor)
def retention(root: Path, keep: int) -> dict:
"""Report releases outside the retention set. Never deletes anything.
Kept: the current release, the last applied desired selection, and the
``keep`` most recent distinct verified releases from the deployment log.
"""
if keep < 1:
raise ReleaseError("keep must be at least 1")
releases = root / "releases"
present = sorted(
path.name for path in releases.iterdir()
if path.is_dir() and re.fullmatch(r"[0-9a-f]{64}", path.name)
) if releases.is_dir() else []
reasons: dict[str, list[str]] = {}
def keep_digest(digest: str | None, reason: str) -> None:
if digest:
reasons.setdefault(digest.removeprefix("sha256:"), []).append(reason)
current = current_release(root)
keep_digest(current and current["artifact_digest"], "current")
state = load_desired_state(root)
keep_digest(state and state["artifact_digest"], "desired-state")
verified: list[str] = []
log = root / ".deployments.jsonl"
if log.is_file():
for line in reversed(log.read_text(encoding="utf-8").splitlines()):
try:
entry = json.loads(line)
except json.JSONDecodeError:
continue
digest = entry.get("artifact_digest")
if entry.get("outcome") == "verified" and digest not in verified:
verified.append(digest)
for digest in verified[:keep]:
keep_digest(digest, "recent-verified")
return {
"root": str(root),
"keep": {digest: reasons[digest] for digest in present if digest in reasons},
"candidates": [digest for digest in present if digest not in reasons],
}
def parser() -> argparse.ArgumentParser:
command = argparse.ArgumentParser(description=__doc__)
sub = command.add_subparsers(dest="command", required=True)
build_cmd = sub.add_parser("build")
build_cmd.add_argument("--source", type=Path, required=True)
build_cmd.add_argument("--output", type=Path, required=True)
build_cmd.add_argument("--revision", required=True)
package_cmd = sub.add_parser("package")
package_cmd.add_argument("--directory", type=Path, required=True)
package_cmd.add_argument("--archive", type=Path, required=True)
package_cmd.add_argument("--checksum", type=Path, required=True)
inspect_cmd = sub.add_parser("inspect")
inspect_cmd.add_argument("--archive", type=Path, required=True)
inspect_cmd.add_argument("--checksum", type=Path, required=True)
inspect_cmd.add_argument("--revision", required=True)
publish_cmd = sub.add_parser("publish")
publish_cmd.add_argument("--archive", type=Path, required=True)
publish_cmd.add_argument("--checksum", type=Path, required=True)
publish_cmd.add_argument("--root", type=Path, required=True)
publish_cmd.add_argument("--revision", required=True)
publish_cmd.add_argument("--verify-url")
publish_cmd.add_argument("--actor")
verify_cmd = sub.add_parser("verify")
verify_cmd.add_argument("--url", required=True)
verify_cmd.add_argument("--revision", required=True)
verify_cmd.add_argument("--digest", required=True)
rollback_cmd = sub.add_parser("rollback")
rollback_cmd.add_argument("--root", type=Path, required=True)
rollback_cmd.add_argument("--revision", required=True)
rollback_cmd.add_argument("--digest", required=True)
rollback_cmd.add_argument("--verify-url")
rollback_cmd.add_argument("--actor")
retention_cmd = sub.add_parser("retention", help="report prune candidates; never deletes")
retention_cmd.add_argument("--root", type=Path, required=True)
retention_cmd.add_argument("--keep", type=int, default=5)
return command
def main() -> int:
args = parser().parse_args()
try:
if args.command == "build":
build(args.source, args.output, args.revision)
elif args.command == "package":
package(args.directory, args.archive, args.checksum)
elif args.command == "inspect":
inspect_archive(args.archive, args.checksum, args.revision)
elif args.command == "publish":
publish(args.archive, args.checksum, args.root, args.revision, args.verify_url, args.actor)
elif args.command == "verify":
verify_public(args.url, args.revision, args.digest)
elif args.command == "rollback":
rollback(args.root, args.revision, args.digest, args.verify_url, args.actor)
elif args.command == "retention":
print(json.dumps(retention(args.root, args.keep), indent=2, sort_keys=True))
except ReleaseError as exc:
print(f"error: {exc}", file=sys.stderr)
return 1
return 0
if __name__ == "__main__":
raise SystemExit(main())
delivery/site.json
+1
View File
@@ -0,0 +1 @@
{"schema":1,"name":"blog","owner":"public","package":"blog-site","target":"/srv/libretech-static/blog","verify_url":"https://blog.static.librete.ch/.well-known/release.json"}
delivery/tests/test_blog_delivery.py
+104
View File
@@ -0,0 +1,104 @@
from __future__ import annotations
import hashlib
import io
import json
from pathlib import Path
import tarfile
import tempfile
import unittest
from scripts import deploy_contract
from scripts import delivery_gate
from scripts import fetch_hugo
from scripts import generic_package
DELIVERY = Path(__file__).resolve().parents[1]
REPO_ROOT = DELIVERY.parent
WORKFLOWS = REPO_ROOT / ".gitea/workflows"
REVISION = "a" * 40
class BlogSiteTests(unittest.TestCase):
def test_site_is_the_netcup_release_root_not_a_checkout(self):
site = deploy_contract.load_site(DELIVERY / "site.json")
self.assertEqual(site.target, "/srv/libretech-static/blog")
self.assertEqual(site.verify_url, "https://blog.static.librete.ch/.well-known/release.json")
url = generic_package.package_file_url(
deploy_contract.REGISTRY, "public", "blog-site", REVISION, generic_package.ARCHIVE_NAME,
)
request = deploy_contract.validate_request(
site=site,
revision=REVISION,
artifact_url=url,
artifact_digest="sha256:" + "b" * 64,
target=site.target,
verify_url=site.verify_url,
desired_commit="c" * 40,
desired_generation="1",
)
self.assertEqual(request["artifact_url"], url)
with self.assertRaisesRegex(deploy_contract.ContractError, "immutable project package"):
deploy_contract.validate_request(**{**request, "artifact_url": url.replace("public", "libretech")}, site=site)
def test_reviewed_gate_enables_publication_and_deployment(self):
config = DELIVERY / ".delivery/config.json"
for capability in delivery_gate.CAPABILITIES:
delivery_gate.check(config, capability)
def test_hugo_is_pinned_and_verified(self):
self.assertRegex(fetch_hugo.SHA256, r"^[0-9a-f]{64}$")
self.assertIn(f"/v{fetch_hugo.VERSION}/", fetch_hugo.URL)
with tempfile.TemporaryDirectory() as temp:
buffer = io.BytesIO()
with tarfile.open(fileobj=buffer, mode="w:gz") as tar:
info = tarfile.TarInfo("hugo")
info.size = 4
tar.addfile(info, io.BytesIO(b"\x7fELF"))
archive = buffer.getvalue()
binary = fetch_hugo.extract_hugo(archive, hashlib.sha256(archive).hexdigest(), Path(temp))
self.assertEqual(binary.read_bytes(), b"\x7fELF")
with self.assertRaisesRegex(fetch_hugo.FetchError, "digest mismatch"):
fetch_hugo.extract_hugo(archive, "0" * 64, Path(temp))
class BlogWorkflowTests(unittest.TestCase):
def read(self, name):
return (WORKFLOWS / name).read_text(encoding="utf-8")
def test_no_workflow_publishes_from_a_checkout(self):
for workflow in sorted(WORKFLOWS.glob("*.yml")):
text = workflow.read_text(encoding="utf-8")
with self.subTest(workflow=workflow.name):
for forbidden in ("rsync", "scp ", "publishDir", "/var/www"):
self.assertNotIn(forbidden, text)
def test_publication_is_gated_before_any_package_write(self):
workflow = self.read("publish-blog.yml")
_, _, publish = workflow.partition("\n publish:\n")
self.assertIn("vars.BLOG_PACKAGE_PUBLISH_ENABLED == 'true'", publish)
self.assertLess(publish.index("delivery_gate.py package_publish"), publish.index("generic_package.py"))
_, _, validate = workflow.partition("\n validate:\n")
self.assertNotIn("secrets.", validate.partition("\n publish:\n")[0])
def test_deployment_is_gated_ordered_and_site_scoped(self):
workflow = self.read("deploy-blog.yml")
self.assertIn("workflow_dispatch:", workflow)
self.assertNotIn("\n push:", workflow)
self.assertIn("vars.BLOG_STATIC_DEPLOY_ENABLED == 'true'", workflow)
self.assertNotIn("DONATELLA", workflow)
self.assertEqual(workflow.count("--site-config delivery/site.json"), 2)
for field in ("revision", "artifact_url", "artifact_digest", "target", "verify_url",
"desired_commit", "desired_generation"):
self.assertIn(f" {field}:\n", workflow)
self.assertIn("StrictHostKeyChecking yes", workflow)
self.assertIn("steps.receive.outputs.superseded == 'false'", workflow)
self.assertLess(
workflow.index("delivery_gate.py static_deploy"),
workflow.index("ssh static-release-target"),
)
if __name__ == "__main__":
unittest.main()
public/categories/index.xml
-11
View File
@@ -1,11 +0,0 @@
<?xml version="1.0" encoding="utf-8" standalone="yes"?>
<rss version="2.0" xmlns:atom="http://www.w3.org/2005/Atom">
<channel>
<title>Categories on LibreTECH</title>
<link>https://blog.librete.ch/categories/</link>
<description>Recent content in Categories on LibreTECH</description>
<generator>Hugo</generator>
<language>en-us</language>
<atom:link href="https://blog.librete.ch/categories/index.xml" rel="self" type="application/rss+xml" />
</channel>
</rss>
public/index.xml
-11
View File
@@ -1,11 +0,0 @@
<?xml version="1.0" encoding="utf-8" standalone="yes"?>
<rss version="2.0" xmlns:atom="http://www.w3.org/2005/Atom">
<channel>
<title>LibreTECH</title>
<link>https://blog.librete.ch/</link>
<description>Recent content on LibreTECH</description>
<generator>Hugo</generator>
<language>en-us</language>
<atom:link href="https://blog.librete.ch/index.xml" rel="self" type="application/rss+xml" />
</channel>
</rss>
public/sitemap.xml
-11
View File
@@ -1,11 +0,0 @@
<?xml version="1.0" encoding="utf-8" standalone="yes"?>
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"
xmlns:xhtml="http://www.w3.org/1999/xhtml">
<url>
<loc>https://blog.librete.ch/categories/</loc>
</url><url>
<loc>https://blog.librete.ch/</loc>
</url><url>
<loc>https://blog.librete.ch/tags/</loc>
</url>
</urlset>
public/tags/index.xml
-11
View File
@@ -1,11 +0,0 @@
<?xml version="1.0" encoding="utf-8" standalone="yes"?>
<rss version="2.0" xmlns:atom="http://www.w3.org/2005/Atom">
<channel>
<title>Tags on LibreTECH</title>
<link>https://blog.librete.ch/tags/</link>
<description>Recent content in Tags on LibreTECH</description>
<generator>Hugo</generator>
<language>en-us</language>
<atom:link href="https://blog.librete.ch/tags/index.xml" rel="self" type="application/rss+xml" />
</channel>
</rss>