diff --git a/README.md b/README.md index cd4e7f5..bf6a5af 100644 --- a/README.md +++ b/README.md @@ -122,6 +122,9 @@ simulation limits are documented in the The current consumer evidence and remaining external gates are tracked in the [SpaceGame consumer pilot](docs/integration/spacegame-pilot.md) and independent [Unscouted consumer pilot](docs/integration/unscouted-pilot.md). +The fail-closed launch decision, redacted evidence matrix, and two-machine +external-network procedure are in +[production readiness and real-network canary](docs/operations/production-readiness.md). ## Development diff --git a/docs/evidence/production-readiness-v1.json b/docs/evidence/production-readiness-v1.json new file mode 100644 index 0000000..772c12d --- /dev/null +++ b/docs/evidence/production-readiness-v1.json @@ -0,0 +1,124 @@ +{ + "schemaVersion": 1, + "kind": "rendezvous-production-readiness", + "evaluatedCommit": "6bad659c123ad45ad0d9d07f93217c2e8c42d459", + "decision": "not-ready", + "localGates": [ + { + "id": "immutable-release-artifacts", + "status": "pending", + "evidenceRef": "docs/operations/production-readiness.md", + "note": "Rebuild after the readiness tooling checkpoint." + }, + { + "id": "debug-and-release-verification", + "status": "pending", + "evidenceRef": "docs/operations/production-readiness.md", + "note": "Re-run after the readiness tooling checkpoint." + }, + { + "id": "real-consumer-pilots", + "status": "pending", + "evidenceRef": "docs/integration/spacegame-pilot.md", + "note": "Pin and re-run both real consumer revisions." + }, + { + "id": "candidate-capacity-resilience", + "status": "pending", + "evidenceRef": "docs/evidence/capacity/v2/candidate-2cpu.json", + "note": "Re-run the five-minute candidate on the tooling checkpoint." + }, + { + "id": "production-process-recovery", + "status": "pending", + "evidenceRef": "docs/operations/capacity-and-resilience.md", + "note": "Re-run process restart, drain, and rollback gates." + }, + { + "id": "security-privacy-observability", + "status": "pending", + "evidenceRef": "docs/operations/production-readiness.md", + "note": "Re-run the combined release verification matrix." + } + ], + "externalGates": [ + { + "id": "public-package-empty-cache-restore", + "status": "pending", + "evidenceRef": "docs/operations/production-readiness.md", + "note": "The public registry does not currently resolve version 1.0.0." + }, + { + "id": "signed-publication", + "status": "pending", + "evidenceRef": "docs/releases/README.md", + "note": "Protected release credentials and immutable tag publication are required." + }, + { + "id": "source-preserving-udp-ingress", + "status": "pending", + "evidenceRef": "docs/operations/production-readiness.md", + "note": "The public ingress path needs packet-level source and reply validation." + }, + { + "id": "same-lan-direct-canary", + "status": "pending", + "evidenceRef": "docs/operations/production-readiness.md", + "note": "Requires two independently operated game clients." + }, + { + "id": "home-nat-direct-canary", + "status": "pending", + "evidenceRef": "docs/operations/production-readiness.md", + "note": "Requires distinct residential networks." + }, + { + "id": "restrictive-cgnat-typed-failure", + "status": "pending", + "evidenceRef": "docs/operations/production-readiness.md", + "note": "Requires a known restrictive carrier topology." + }, + { + "id": "firewall-blocked-udp-typed-failure", + "status": "pending", + "evidenceRef": "docs/operations/production-readiness.md", + "note": "Requires an independently controlled firewall rule." + }, + { + "id": "ipv6-direct-canary", + "status": "pending", + "evidenceRef": "docs/operations/production-readiness.md", + "note": "Requires two IPv6-capable external clients and public ingress." + }, + { + "id": "public-rate-shaped-capacity", + "status": "pending", + "evidenceRef": "docs/operations/capacity-and-resilience.md", + "note": "The full public HTTP and UDP traffic mix has not been measured." + }, + { + "id": "one-hour-candidate-endurance", + "status": "pending", + "evidenceRef": "docs/operations/capacity-and-resilience.md", + "note": "A production-shaped one-hour candidate run is required." + }, + { + "id": "alert-delivery", + "status": "pending", + "evidenceRef": "docs/operations/incident-runbooks.md", + "note": "A real alert sink must observe trigger and recovery notifications." + }, + { + "id": "cold-standby-rollback-drill", + "status": "pending", + "evidenceRef": "docs/operations/capacity-and-resilience.md", + "note": "The deployment must demonstrate the host-visible recovery objective." + }, + { + "id": "documentation-only-runbook-exercise", + "status": "pending", + "evidenceRef": "docs/operations/incident-runbooks.md", + "note": "An independent operator must execute the runbooks using only the docs." + } + ] +} diff --git a/docs/integration/test-client.md b/docs/integration/test-client.md index 83a9429..a807fee 100644 --- a/docs/integration/test-client.md +++ b/docs/integration/test-client.md @@ -116,6 +116,9 @@ rm deploy/compose/secrets/signing-key codes are stable automation contracts. Informational events use stdout and failures use stderr. +Successful direct-connection and direct-traffic events include the coarse +`addressFamily` value `ipv4` or `ipv6`. They never include the peer address. + The deployment smoke performs the full health, publish, join, mediation, direct traffic, outcome-report, and cleanup flow using bounded waits: @@ -220,3 +223,6 @@ least-scope publisher credential from the deployment secret boundary and set the external service, mediator, and matching scope variables described in the [secure Linux deployment smoke](../deployment/linux.md#http-and-udp-smoke). Run representative external-network tests; loopback success is not NAT coverage. +Use the redacting, bounded +[real-network canary procedure](../operations/production-readiness.md) for formal +production evidence rather than committing raw TestClient JSON. diff --git a/docs/operations/production-readiness.md b/docs/operations/production-readiness.md new file mode 100644 index 0000000..c306c60 --- /dev/null +++ b/docs/operations/production-readiness.md @@ -0,0 +1,191 @@ +# Production-readiness decision and real-network canary + +Tracking: #23 + +Rendezvous v1 is **not production-ready** until every required gate in +[`production-readiness-v1.json`](../evidence/production-readiness-v1.json) is +recorded as `pass`. The machine-checkable decision is intentionally fail-closed: + +```bash +./scripts/check-production-readiness.sh +``` + +Exit `0` means every required gate is present and passing, exit `3` means the +record is valid but at least one gate is pending or failed, and exit `2` means +the record itself is malformed or contains identifier-, endpoint-, account-, or +credential-shaped data. Editing only the top-level decision cannot make the +check pass. + +The checked-in record is an index, not a log archive. It contains one +repository-relative evidence reference and a short categorical note per gate. +Raw packet captures, client event streams, publisher credentials, public or +private network endpoints, listing IDs, and player/account identifiers must not +be committed. + +## Required decision matrix + +The local matrix covers immutable artifacts, Debug and Release verification, +both real game consumers, the candidate capacity/resilience profile, +production-process recovery, and the combined security/privacy/observability +gate. These may be reproduced by the project team on a clean candidate commit. + +The external matrix remains distinct because a local namespace, loopback, +container bridge, or second process on one machine cannot prove it: + +| Gate | Required evidence | +| --- | --- | +| Public package empty-cache restore | A clean machine restores the exact Client and Contracts version using only the documented public sources. | +| Signed publication | The immutable tag publishes packages, image digest, SBOMs, provenance, checksums, and verifiable signatures through the protected release workflow. | +| Source-preserving UDP ingress | Packet capture on the service host proves the mediator observes each peer's real source tuple and replies from the advertised public tuple; no UDP proxy rewrites either direction. | +| Same-LAN direct canary | Two independently operated game clients establish authenticated direct LiteNetLib traffic. | +| Home-NAT direct canary | Host and joiner on distinct residential networks establish authenticated direct LiteNetLib traffic. | +| Restrictive/CGNAT and blocked-UDP canaries | Each bounded join exits `12`, records a typed terminal category, and exposes the game-owned fallback policy without hanging or claiming success. | +| IPv6 direct canary | Two external IPv6 clients record authenticated direct traffic and an observed `ipv6` peer address family. | +| Public rate-shaped capacity | The documented HTTP/UDP workload mix meets its objectives through TLS, Kestrel, JSON, LiteNetLib, kernel sockets, and public ingress. | +| One-hour endurance | The immutable production-shaped candidate completes the one-hour profile without a state, handle, memory, readiness, or latency failure. | +| Alert delivery | A real alert sink receives both trigger and recovery notifications for the rehearsed outage. | +| Cold-standby rollback | Drain, stop, socket release, replacement start, host re-registration, and rollback meet the process and host-visible recovery objectives. | +| Documentation-only exercise | An operator who did not author the runbooks completes key rotation/revocation, outage, restart, re-registration, and rollback using only the checked-in documentation. | + +Failure or missing evidence is blocking. It is never converted into an accepted +risk by changing the wording of the readiness note. + +## Prepare one immutable canary build + +Use the exact release candidate on every canary machine. Verify a clean checkout, +restore in locked mode, and build the TestClient before changing networks: + +```bash +test -z "$(git status --porcelain)" +dotnet restore Rendezvous.slnx --locked-mode +dotnet build Rendezvous.slnx --configuration Release --no-restore +``` + +Keep shell tracing disabled. The host receives a short-lived, least-scope +publisher credential through `RENDEZVOUS_PUBLISHER_CREDENTIAL`; it must never be +put in an argument, coordination file, evidence file, command transcript, or +support message. Set the public HTTPS service URL and advertised UDP mediator +tuple separately. TestClient rejects credentials embedded in the service URL. + +## Run a success canary across two machines + +On the host machine, choose `same-lan`, `home-nat`, or `ipv6-direct`. The +coordination file is mode `0600` and contains only the temporary listing UUID. +It is not evidence; transfer it through an approved private channel, then delete +both copies. + +```bash +set +x +export RENDEZVOUS_PUBLISHER_CREDENTIAL='supplied-by-the-approved-secret-boundary' +export RENDEZVOUS_CANARY_ROLE=host +export RENDEZVOUS_CANARY_TOPOLOGY=home-nat +export RENDEZVOUS_CANARY_ADDRESS_FAMILY=ipv4 +export RENDEZVOUS_CANARY_HTTP_URL='https://service.example.invalid/' +export RENDEZVOUS_CANARY_UDP_ENDPOINT='203.0.113.10:9050' +export RENDEZVOUS_CANARY_COORDINATION_FILE="$HOME/.local/state/rendezvous-canary-listing" +export RENDEZVOUS_CANARY_OUTPUT="$PWD/artifacts/canary/home-nat-host.json" +./scripts/run-real-network-canary.sh +``` + +The host prints only that it is ready and waits for the authenticated exchange. +On the joiner, read the securely transferred UUID without placing it in shell +history and run the matching topology: + +```bash +set +x +read -r RENDEZVOUS_CANARY_LISTING_ID < "$HOME/.local/state/rendezvous-canary-listing" +export RENDEZVOUS_CANARY_LISTING_ID +export RENDEZVOUS_CANARY_ROLE=client-success +export RENDEZVOUS_CANARY_TOPOLOGY=home-nat +export RENDEZVOUS_CANARY_ADDRESS_FAMILY=ipv4 +export RENDEZVOUS_CANARY_HTTP_URL='https://service.example.invalid/' +export RENDEZVOUS_CANARY_UDP_ENDPOINT='203.0.113.10:9050' +export RENDEZVOUS_CANARY_OUTPUT="$PWD/artifacts/canary/home-nat-client.json" +./scripts/run-real-network-canary.sh +unset RENDEZVOUS_CANARY_LISTING_ID +``` + +The host summary requires authenticated direct traffic and deregistration. The +client summary requires connection, authenticated direct traffic, accepted +outcome reporting, and the declared address family observed on the actual peer. +The summaries deliberately contain no network tuple or listing identifier. + +For IPv6, set the topology to `ipv6-direct`, the family to `ipv6`, and use the +deployment's bracketed IPv6 mediator form. Record unsupported operating systems, +console platforms, VPNs, and address families as untested; an IPv4 pass is not +evidence for IPv6 or a platform network policy. + +## Run a bounded failure canary + +Start the host from an independently reachable network as above. On the joiner, +apply the reviewed firewall rule that blocks the relevant UDP path, or use the +known restrictive carrier network, then set `client-expected-failure` and the +matching topology: + +```bash +export RENDEZVOUS_CANARY_ROLE=client-expected-failure +export RENDEZVOUS_CANARY_TOPOLOGY=firewall-blocked-udp +export RENDEZVOUS_CANARY_ADDRESS_FAMILY=ipv4 +export RENDEZVOUS_CANARY_OUTPUT="$PWD/artifacts/canary/firewall-blocked-client.json" +./scripts/run-real-network-canary.sh +``` + +This role passes only when TestClient exits exactly `12`, emits a non-empty typed +authorization/traversal outcome, and emits the authoritative fallback category. +A timeout without the typed terminal outcome, exit `0`, direct-traffic success, +or an unbounded process is a failed canary. Restore the firewall after the drill +and verify normal traffic again. + +## Private diagnostics and retention + +The harness creates raw JSON events under a randomly named `0700`-equivalent +temporary directory with a process `umask` of `077`. Successful raw events are +deleted automatically. On failure they remain in that private directory so the +operator can triage locally; do not attach them to an issue before removing +listing IDs and reviewing every field. Set `RENDEZVOUS_CANARY_KEEP_RAW=true` +only for an approved short-lived diagnostic capture, then delete it manually. + +The sanitized summary contains the commit, clean/dirty tree state, UTC time, +role, declared topology, observed address-family gate, aggregate booleans, and +the retention policy. Formal evidence requires the default clean-tree check. + +## Public ingress proof + +Success through a public hostname is insufficient proof that UDP source/reply +addressing is preserved. During a canary, an authorized operator must capture +only packet headers at the service host and verify: + +1. each authenticated contribution reaches the mediator with the external peer + source tuple visible to the server; +2. introductions are sent from the same advertised public mediator tuple; +3. no load balancer, user-space proxy, service mesh, or destination NAT changes + the source or reply tuple expected by LiteNetLib; and +4. malformed or unauthenticated traffic receives no amplified response. + +Store the approval, capture time window, candidate digest, topology category, +and pass/fail result. Do not retain packet payloads or peer tuples in the +repository. A failed tuple check blocks release even if one canary happened to +connect. + +## Rehearsal and triage + +Run the security, capacity, observability, deployment, rollback, privacy, and +incident procedures against the same immutable candidate. The independent +operator records which runbook revision they followed, start/end time, observed +alerts, recovery time, unexpected decisions, and pass/fail result. Update the +documentation and repeat any failed or ambiguous step. + +Before changing the readiness record, reconcile every open roadmap issue as one +of: `blocking` with an owner and evidence needed, `accepted-v1` with a bounded +documented limitation, or `post-v1` with a filed issue. HA, active-active or +multi-region routing, relays, platform authentication, and scale above the +single-active v1 envelope are not silently accepted; each needs a traceable +post-v1 issue. The current follow-ups are relay decision [#24], HA/multi-region +shared state and routing [#28], scale beyond the measured envelope [#29], and +platform authentication adapters [#30]. Run the checker after every evidence +update. Only its `READY` result may support a production-ready claim. + +[#24]: https://git.finalfactory.de/HeiKyu/Rendezvous/issues/24 +[#28]: https://git.finalfactory.de/HeiKyu/Rendezvous/issues/28 +[#29]: https://git.finalfactory.de/HeiKyu/Rendezvous/issues/29 +[#30]: https://git.finalfactory.de/HeiKyu/Rendezvous/issues/30 diff --git a/eng/check_production_readiness.py b/eng/check_production_readiness.py new file mode 100755 index 0000000..41d71cb --- /dev/null +++ b/eng/check_production_readiness.py @@ -0,0 +1,147 @@ +#!/usr/bin/env python3 +"""Validate the redacted v1 readiness record and emit the release decision.""" + +from __future__ import annotations + +import json +import pathlib +import re +import sys +from typing import Any + + +LOCAL_GATES = { + "immutable-release-artifacts", + "debug-and-release-verification", + "real-consumer-pilots", + "candidate-capacity-resilience", + "production-process-recovery", + "security-privacy-observability", +} +EXTERNAL_GATES = { + "public-package-empty-cache-restore", + "signed-publication", + "source-preserving-udp-ingress", + "same-lan-direct-canary", + "home-nat-direct-canary", + "restrictive-cgnat-typed-failure", + "firewall-blocked-udp-typed-failure", + "ipv6-direct-canary", + "public-rate-shaped-capacity", + "one-hour-candidate-endurance", + "alert-delivery", + "cold-standby-rollback-drill", + "documentation-only-runbook-exercise", +} +STATUSES = {"pass", "pending", "fail"} +FORBIDDEN_KEY_PARTS = { + "address", + "credential", + "endpoint", + "listingid", + "password", + "playerid", + "secret", + "token", + "userid", +} +UUID = re.compile(r"\b[0-9a-fA-F]{8}-[0-9a-fA-F-]{27,}\b") +IPV4 = re.compile(r"(? None: + if isinstance(value, dict): + for key, child in value.items(): + normalized = re.sub(r"[^a-z0-9]", "", key.lower()) + if any(part in normalized for part in FORBIDDEN_KEY_PARTS): + raise InvalidRecord(f"{path}.{key} uses a forbidden sensitive-data key") + reject_sensitive(child, f"{path}.{key}") + elif isinstance(value, list): + for index, child in enumerate(value): + reject_sensitive(child, f"{path}[{index}]") + elif isinstance(value, str): + if UUID.search(value) or IPV4.search(value) or "://" in value or "@" in value: + raise InvalidRecord(f"{path} contains endpoint, identifier, or account-shaped data") + + +def validate_gate_set(items: Any, expected: set[str], path: str) -> list[dict[str, str]]: + if not isinstance(items, list): + raise InvalidRecord(f"{path} must be an array") + gates: list[dict[str, str]] = [] + for index, item in enumerate(items): + if not isinstance(item, dict) or set(item) != {"id", "status", "evidenceRef", "note"}: + raise InvalidRecord(f"{path}[{index}] has an invalid shape") + if not all(isinstance(item[key], str) for key in item): + raise InvalidRecord(f"{path}[{index}] fields must be strings") + if item["status"] not in STATUSES: + raise InvalidRecord(f"{path}[{index}] has an invalid status") + evidence = pathlib.PurePosixPath(item["evidenceRef"]) + if evidence.is_absolute() or ".." in evidence.parts or not item["evidenceRef"]: + raise InvalidRecord(f"{path}[{index}].evidenceRef must be a repository-relative reference") + if len(item["note"]) > 240: + raise InvalidRecord(f"{path}[{index}].note is too long") + gates.append(item) + identifiers = [gate["id"] for gate in gates] + if len(identifiers) != len(set(identifiers)): + raise InvalidRecord(f"{path} contains duplicate gate identifiers") + if set(identifiers) != expected: + missing = sorted(expected - set(identifiers)) + extra = sorted(set(identifiers) - expected) + raise InvalidRecord(f"{path} gate mismatch; missing={missing}, extra={extra}") + return gates + + +def validate(record: Any) -> tuple[bool, list[str]]: + if not isinstance(record, dict) or set(record) != { + "schemaVersion", + "kind", + "evaluatedCommit", + "decision", + "localGates", + "externalGates", + }: + raise InvalidRecord("The top-level readiness record shape is invalid") + if record["schemaVersion"] != 1 or record["kind"] != "rendezvous-production-readiness": + raise InvalidRecord("The readiness schema identity is invalid") + if not isinstance(record["evaluatedCommit"], str) or not COMMIT.fullmatch(record["evaluatedCommit"]): + raise InvalidRecord("evaluatedCommit must be a full lowercase Git commit") + reject_sensitive(record) + gates = validate_gate_set(record["localGates"], LOCAL_GATES, "$.localGates") + gates += validate_gate_set(record["externalGates"], EXTERNAL_GATES, "$.externalGates") + blockers = sorted(gate["id"] for gate in gates if gate["status"] != "pass") + ready = not blockers + expected_decision = "ready" if ready else "not-ready" + if record["decision"] != expected_decision: + raise InvalidRecord( + f"decision must be {expected_decision!r} for the recorded gate statuses" + ) + return ready, blockers + + +def main() -> int: + if len(sys.argv) != 2: + print("usage: check_production_readiness.py RECORD", file=sys.stderr) + return 2 + try: + with open(sys.argv[1], "r", encoding="utf-8") as source: + record = json.load(source) + ready, blockers = validate(record) + except (OSError, json.JSONDecodeError, InvalidRecord) as error: + print(f"INVALID: {error}", file=sys.stderr) + return 2 + if not ready: + print(f"NOT READY: {len(blockers)} required gate(s) are not passing.") + for blocker in blockers: + print(f"- {blocker}") + return 3 + print("READY: every required v1 production gate is recorded as passing.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/eng/consumer-revisions.json b/eng/consumer-revisions.json index f39f0ed..213e52c 100644 --- a/eng/consumer-revisions.json +++ b/eng/consumer-revisions.json @@ -4,13 +4,13 @@ { "name": "SpaceGame", "repository": "https://git.finalfactory.de/Kyuubi/SpaceGame.git", - "revision": "77519b0cc418a27f8d408ae2d7b8812fbe087c04", + "revision": "f3f5bc29810c362656cd7143bec1ddc2cfaf9f22", "project": "SpaceGame.csproj" }, { "name": "Unscouted", "repository": "https://git.finalfactory.de/HeiKyu/Unscouted.git", - "revision": "7807dbee86eb8b98e702f1eb89c88adff728f635", + "revision": "f0574a7de82aadff6495ca5657dfc19cf7c2f67c", "project": "Net.Core/Net.Core.csproj" } ] diff --git a/scripts/check-production-readiness.sh b/scripts/check-production-readiness.sh new file mode 100755 index 0000000..0fc2f62 --- /dev/null +++ b/scripts/check-production-readiness.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +RECORD="${1:-$ROOT/docs/evidence/production-readiness-v1.json}" + +exec python3 "$ROOT/eng/check_production_readiness.py" "$RECORD" diff --git a/scripts/run-real-network-canary.sh b/scripts/run-real-network-canary.sh new file mode 100755 index 0000000..c72f37d --- /dev/null +++ b/scripts/run-real-network-canary.sh @@ -0,0 +1,234 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +PROJECT="$ROOT/src/FinalFactory.Rendezvous.TestClient/FinalFactory.Rendezvous.TestClient.csproj" +ROLE="${RENDEZVOUS_CANARY_ROLE:-}" +TOPOLOGY="${RENDEZVOUS_CANARY_TOPOLOGY:-}" +ADDRESS_FAMILY="${RENDEZVOUS_CANARY_ADDRESS_FAMILY:-ipv4}" +SERVICE_URL="${RENDEZVOUS_CANARY_HTTP_URL:-}" +MEDIATOR="${RENDEZVOUS_CANARY_UDP_ENDPOINT:-}" +GAME_ID="${RENDEZVOUS_CANARY_GAME_ID:-space-game}" +ENVIRONMENT_ID="${RENDEZVOUS_CANARY_ENVIRONMENT_ID:-production-canary}" +REGION="${RENDEZVOUS_CANARY_REGION:-production-canary}" +PROTOCOL_VERSION="${RENDEZVOUS_CANARY_PROTOCOL_VERSION:-1}" +TIMEOUT_SECONDS="${RENDEZVOUS_CANARY_TIMEOUT_SECONDS:-60}" +RUN_SECONDS="${RENDEZVOUS_CANARY_RUN_SECONDS:-900}" +OUTPUT="${RENDEZVOUS_CANARY_OUTPUT:-$ROOT/artifacts/canary/${ROLE:-unknown}-${TOPOLOGY:-unknown}.json}" +COORDINATION_FILE="${RENDEZVOUS_CANARY_COORDINATION_FILE:-}" +LISTING_ID="${RENDEZVOUS_CANARY_LISTING_ID:-}" +REQUIRE_CLEAN="${RENDEZVOUS_CANARY_REQUIRE_CLEAN:-true}" +KEEP_RAW="${RENDEZVOUS_CANARY_KEEP_RAW:-false}" + +usage() { + printf '%s\n' \ + 'Set RENDEZVOUS_CANARY_ROLE to host, client-success, or client-expected-failure.' \ + 'Also set RENDEZVOUS_CANARY_TOPOLOGY, RENDEZVOUS_CANARY_HTTP_URL, and' \ + 'RENDEZVOUS_CANARY_UDP_ENDPOINT. See docs/operations/production-readiness.md.' >&2 + exit 2 +} + +for command in date dotnet git jq mktemp; do + command -v "$command" >/dev/null || { + printf 'Missing required command: %s\n' "$command" >&2 + exit 2 + } +done + +case "$ROLE" in + host|client-success|client-expected-failure) ;; + *) usage ;; +esac +case "$TOPOLOGY" in + same-lan|home-nat|firewall-blocked-udp|restrictive-cgnat|ipv6-direct) ;; + *) usage ;; +esac +case "$ADDRESS_FAMILY" in + ipv4|ipv6) ;; + *) printf 'RENDEZVOUS_CANARY_ADDRESS_FAMILY must be ipv4 or ipv6.\n' >&2; exit 2 ;; +esac +if [[ "$TOPOLOGY" == ipv6-direct && "$ADDRESS_FAMILY" != ipv6 ]]; then + printf 'The ipv6-direct topology requires RENDEZVOUS_CANARY_ADDRESS_FAMILY=ipv6.\n' >&2 + exit 2 +fi +if [[ "$TOPOLOGY" =~ ^(firewall-blocked-udp|restrictive-cgnat)$ \ + && "$ROLE" == client-success ]]; then + printf 'Failure topologies must use the client-expected-failure role.\n' >&2 + exit 2 +fi +if [[ -z "$SERVICE_URL" || -z "$MEDIATOR" ]]; then + usage +fi +if [[ ! "$TIMEOUT_SECONDS" =~ ^[0-9]+$ ]] \ + || (( TIMEOUT_SECONDS < 1 || TIMEOUT_SECONDS > 300 )); then + printf 'RENDEZVOUS_CANARY_TIMEOUT_SECONDS must be an integer from 1 through 300.\n' >&2 + exit 2 +fi +if [[ ! "$RUN_SECONDS" =~ ^[0-9]+$ ]] \ + || (( RUN_SECONDS < 60 || RUN_SECONDS > 3600 )); then + printf 'RENDEZVOUS_CANARY_RUN_SECONDS must be an integer from 60 through 3600.\n' >&2 + exit 2 +fi +if [[ ! "$PROTOCOL_VERSION" =~ ^[0-9]+$ ]] || (( PROTOCOL_VERSION < 1 )); then + printf 'RENDEZVOUS_CANARY_PROTOCOL_VERSION must be a positive integer.\n' >&2 + exit 2 +fi +if [[ "$REQUIRE_CLEAN" != true && "$REQUIRE_CLEAN" != false ]]; then + printf 'RENDEZVOUS_CANARY_REQUIRE_CLEAN must be true or false.\n' >&2 + exit 2 +fi +if [[ "$KEEP_RAW" != true && "$KEEP_RAW" != false ]]; then + printf 'RENDEZVOUS_CANARY_KEEP_RAW must be true or false.\n' >&2 + exit 2 +fi + +cd "$ROOT" +commit="$(git rev-parse HEAD)" +tree_state=clean +if [[ -n "$(git status --porcelain)" ]]; then + tree_state=dirty +fi +if [[ "$REQUIRE_CLEAN" == true && "$tree_state" != clean ]]; then + printf 'Formal canary evidence requires a clean source tree.\n' >&2 + exit 2 +fi + +if [[ "$ROLE" == host ]]; then + if [[ -z "$COORDINATION_FILE" ]]; then + printf 'The host role requires RENDEZVOUS_CANARY_COORDINATION_FILE.\n' >&2 + exit 2 + fi + if [[ -z "${RENDEZVOUS_PUBLISHER_CREDENTIAL:-}" ]]; then + printf 'The host role requires RENDEZVOUS_PUBLISHER_CREDENTIAL.\n' >&2 + exit 2 + fi +else + if [[ ! "$LISTING_ID" =~ ^[0-9a-fA-F-]{36}$ ]]; then + printf 'A client role requires a UUID in RENDEZVOUS_CANARY_LISTING_ID.\n' >&2 + exit 2 + fi +fi + +umask 077 +raw_dir="$(mktemp -d "${TMPDIR:-/tmp}/rendezvous-canary.XXXXXXXX")" +raw_log="$raw_dir/events.jsonl" +run_succeeded=false +host_pid='' +cleanup() { + local status="$?" + if [[ -n "$host_pid" ]] && kill -0 "$host_pid" 2>/dev/null; then + kill -TERM "$host_pid" 2>/dev/null || true + wait "$host_pid" 2>/dev/null || true + fi + if [[ "$run_succeeded" == true && "$KEEP_RAW" == false ]]; then + rm -rf "$raw_dir" + else + printf 'Private raw canary events retained at %s\n' "$raw_dir" >&2 + fi + return "$status" +} +trap cleanup EXIT +trap 'exit 130' INT +trap 'exit 143' TERM + +common_arguments=( + --service "$SERVICE_URL" + --mediator "$MEDIATOR" + --game "$GAME_ID" + --environment "$ENVIRONMENT_ID" + --region "$REGION" + --protocol "$PROTOCOL_VERSION" + --script + --json + --timeout-seconds "$TIMEOUT_SECONDS" +) + +exit_code=0 +if [[ "$ROLE" == host ]]; then + dotnet run --project "$PROJECT" --configuration Release --no-build -- \ + host "${common_arguments[@]}" --exit-after-echo --run-seconds "$RUN_SECONDS" \ + >"$raw_log" 2>&1 & + host_pid="$!" + ready=false + for ((iteration = 0; iteration < TIMEOUT_SECONDS * 4; iteration++)); do + if jq -e 'select(.event == "host.ready" and .status == "ready")' "$raw_log" \ + >/dev/null 2>&1; then + ready=true + break + fi + if ! kill -0 "$host_pid" 2>/dev/null; then + break + fi + sleep 0.25 + done + if [[ "$ready" != true ]]; then + printf 'The canary host did not become ready within the bounded startup window.\n' >&2 + kill -TERM "$host_pid" 2>/dev/null || true + wait "$host_pid" 2>/dev/null || true + exit 1 + fi + observed_listing="$(jq -r 'select(.event == "host.registered") | .listingId' "$raw_log" | tail -n 1)" + if [[ ! "$observed_listing" =~ ^[0-9a-f-]{36}$ ]]; then + printf 'The canary host did not produce a valid coordination identifier.\n' >&2 + kill -TERM "$host_pid" 2>/dev/null || true + wait "$host_pid" 2>/dev/null || true + exit 1 + fi + coordination_parent="$(dirname "$COORDINATION_FILE")" + mkdir -p "$coordination_parent" + coordination_temp="$COORDINATION_FILE.tmp.$$" + printf '%s\n' "$observed_listing" >"$coordination_temp" + chmod 600 "$coordination_temp" + mv "$coordination_temp" "$COORDINATION_FILE" + printf 'Host ready; securely transfer the private coordination file to the client operator.\n' + set +e + wait "$host_pid" + exit_code="$?" + set -e +elif [[ "$ROLE" == client-success ]]; then + set +e + dotnet run --project "$PROJECT" --configuration Release --no-build -- \ + join "${common_arguments[@]}" --listing "$LISTING_ID" >"$raw_log" 2>&1 + exit_code="$?" + set -e +else + set +e + dotnet run --project "$PROJECT" --configuration Release --no-build -- \ + join "${common_arguments[@]}" --listing "$LISTING_ID" >"$raw_log" 2>&1 + exit_code="$?" + set -e +fi + +checks='{}' +if [[ "$ROLE" == host ]]; then + [[ "$exit_code" -eq 0 ]] + jq -e --arg family "$ADDRESS_FAMILY" 'select(.event == "host.direct-traffic" and .status == "verified" and .addressFamily == $family)' "$raw_log" >/dev/null + jq -e 'select(.event == "host.deregistered" and .status == "complete")' "$raw_log" >/dev/null + checks='{"authenticatedDirectTraffic":true,"deregistered":true}' +elif [[ "$ROLE" == client-success ]]; then + [[ "$exit_code" -eq 0 ]] + jq -e --arg family "$ADDRESS_FAMILY" 'select(.event == "join.connected" and .status == "connected" and .addressFamily == $family)' "$raw_log" >/dev/null + jq -e --arg family "$ADDRESS_FAMILY" 'select(.event == "join.direct-traffic" and .status == "verified" and .addressFamily == $family)' "$raw_log" >/dev/null + jq -e 'select(.event == "join.outcome-report" and .status == "accepted")' "$raw_log" >/dev/null + checks='{"authenticatedDirectTraffic":true,"typedOutcomeReported":true}' +else + [[ "$exit_code" -eq 12 ]] + jq -e 'select((.event == "join.traversal" or .event == "join.authorization") and .status == "failed" and (.outcome | type == "string") and (.outcome | length > 0))' "$raw_log" >/dev/null + jq -e 'select(.event == "join.fallback" and (.status == "available" or .status == "unavailable") and (.outcome | type == "string") and (.outcome | length > 0))' "$raw_log" >/dev/null + checks='{"boundedTypedFailure":true,"fallbackPolicyReported":true}' +fi + +mkdir -p "$(dirname "$OUTPUT")" +jq -n \ + --arg commit "$commit" \ + --arg treeState "$tree_state" \ + --arg timestampUtc "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ + --arg role "$ROLE" \ + --arg topology "$TOPOLOGY" \ + --arg addressFamily "$ADDRESS_FAMILY" \ + --argjson checks "$checks" \ + '{schemaVersion:1,kind:"rendezvous-real-network-canary",commit:$commit,treeState:$treeState,timestampUtc:$timestampUtc,role:$role,topology:$topology,addressFamily:$addressFamily,result:"pass",checks:$checks,dataRetention:{rawEvents:"deleted-after-success",identifiers:"not-in-summary",networkEndpoints:"not-in-summary"}}' \ + >"$OUTPUT" + +run_succeeded=true +printf 'Real-network canary passed; sanitized evidence: %s\n' "$OUTPUT" diff --git a/scripts/verify-real-consumers.sh b/scripts/verify-real-consumers.sh index 1a843b6..73a9b60 100755 --- a/scripts/verify-real-consumers.sh +++ b/scripts/verify-real-consumers.sh @@ -47,6 +47,8 @@ for ((index = 0; index < count; index++)); do cat >"$targets" < + + diff --git a/src/FinalFactory.Rendezvous.TestClient/RendezvousCommandRunner.cs b/src/FinalFactory.Rendezvous.TestClient/RendezvousCommandRunner.cs index ca88aae..6410358 100644 --- a/src/FinalFactory.Rendezvous.TestClient/RendezvousCommandRunner.cs +++ b/src/FinalFactory.Rendezvous.TestClient/RendezvousCommandRunner.cs @@ -114,11 +114,12 @@ internal sealed class RendezvousCommandRunner : ITestClientCommandRunner listingId: session.ListingId.ToString(), displayName: options.DisplayName); echo = new DirectEchoProtocol(events.GameplayEvents, host: true); - echo.ExchangeCompleted += _ => output.Write( + echo.ExchangeCompleted += peer => output.Write( "host.direct-traffic", "verified", phase: "direct-traffic", - endpointType: "peer-to-peer"); + endpointType: "peer-to-peer", + addressFamily: AddressFamilyName(peer.Address)); coordinator = new RendezvousHostCoordinator( manager, events, @@ -485,6 +486,7 @@ internal sealed class RendezvousCommandRunner : ITestClientCommandRunner "connected", phase: "direct-connection", endpointType: endpointType, + addressFamily: AddressFamilyName(peer.Address), elapsedMilliseconds: ToMilliseconds(outcome.Elapsed)); await ReportOutcomeAsync(coordinator, joins, output, cancellationToken).ConfigureAwait(false); echo.BeginJoin(peer); @@ -507,7 +509,8 @@ internal sealed class RendezvousCommandRunner : ITestClientCommandRunner "join.direct-traffic", "verified", phase: "direct-traffic", - endpointType: endpointType); + endpointType: endpointType, + addressFamily: AddressFamilyName(peer.Address)); peer.Disconnect(); manager.PollEvents(); return TestClientExitCode.Success; @@ -765,6 +768,9 @@ internal sealed class RendezvousCommandRunner : ITestClientCommandRunner return privateAddress ? "private" : "public"; } + private static string AddressFamilyName(IPAddress address) => + address.AddressFamily == AddressFamily.InterNetworkV6 ? "ipv6" : "ipv4"; + private static long ToMilliseconds(TimeSpan elapsed) => (long)Math.Min(long.MaxValue, Math.Max(0, elapsed.TotalMilliseconds)); diff --git a/src/FinalFactory.Rendezvous.TestClient/TestClientOutput.cs b/src/FinalFactory.Rendezvous.TestClient/TestClientOutput.cs index c4582f4..4ad70f9 100644 --- a/src/FinalFactory.Rendezvous.TestClient/TestClientOutput.cs +++ b/src/FinalFactory.Rendezvous.TestClient/TestClientOutput.cs @@ -24,6 +24,7 @@ internal sealed class TestClientOutput(TextWriter standardOutput, TextWriter sta string? displayName = null, string? outcome = null, string? endpointType = null, + string? addressFamily = null, int? count = null, long? elapsedMilliseconds = null, string? message = null) => WriteCore( @@ -37,6 +38,7 @@ internal sealed class TestClientOutput(TextWriter standardOutput, TextWriter sta DisplayName = SafeText(displayName), Outcome = SafeToken(outcome), EndpointType = SafeToken(endpointType), + AddressFamily = SafeToken(addressFamily), Count = count, ElapsedMilliseconds = elapsedMilliseconds, Message = SafeText(message), @@ -92,6 +94,7 @@ internal sealed class TestClientOutput(TextWriter standardOutput, TextWriter sta Append(line, "name", item.DisplayName, quote: true); Append(line, "outcome", item.Outcome); Append(line, "endpoint", item.EndpointType); + Append(line, "addressFamily", item.AddressFamily); if (item.Count.HasValue) { Append(line, "count", item.Count.Value.ToString(System.Globalization.CultureInfo.InvariantCulture)); @@ -174,6 +177,7 @@ internal sealed class TestClientOutput(TextWriter standardOutput, TextWriter sta public string? DisplayName { get; init; } public string? Outcome { get; init; } public string? EndpointType { get; init; } + public string? AddressFamily { get; init; } public int? Count { get; init; } public long? ElapsedMilliseconds { get; init; } public string? Message { get; init; } diff --git a/tests/FinalFactory.Rendezvous.Tests/Release/ReleaseCompatibilityTests.cs b/tests/FinalFactory.Rendezvous.Tests/Release/ReleaseCompatibilityTests.cs index c9f5424..cac0e31 100644 --- a/tests/FinalFactory.Rendezvous.Tests/Release/ReleaseCompatibilityTests.cs +++ b/tests/FinalFactory.Rendezvous.Tests/Release/ReleaseCompatibilityTests.cs @@ -143,6 +143,10 @@ public sealed class ReleaseCompatibilityTests pinnedConsumers.Select(static item => item.GetProperty("name").GetString()!).ToArray()); Assert.All(pinnedConsumers, static item => Assert.Matches("^[0-9a-f]{40}$", item.GetProperty("revision").GetString())); + + string realConsumerGate = File.ReadAllText(Path.Combine(root, "scripts", "verify-real-consumers.sh")); + Assert.Contains("", realConsumerGate, StringComparison.Ordinal); + Assert.Contains("", realConsumerGate, StringComparison.Ordinal); } [Fact] @@ -164,6 +168,49 @@ public sealed class ReleaseCompatibilityTests Assert.Equal(actual, declared); } + [Fact] + public void ProductionReadinessRecordIsFailClosedAndCanaryEvidenceIsRedacted() + { + string root = FindRepositoryRoot(); + using JsonDocument readiness = JsonDocument.Parse(File.ReadAllText(Path.Combine( + root, + "docs/evidence/production-readiness-v1.json"))); + JsonElement document = readiness.RootElement; + Assert.Equal(1, document.GetProperty("schemaVersion").GetInt32()); + Assert.Equal("rendezvous-production-readiness", document.GetProperty("kind").GetString()); + Assert.Matches("^[0-9a-f]{40}$", document.GetProperty("evaluatedCommit").GetString()); + + JsonElement[] local = document.GetProperty("localGates").EnumerateArray().ToArray(); + JsonElement[] external = document.GetProperty("externalGates").EnumerateArray().ToArray(); + Assert.Equal(6, local.Length); + Assert.Equal(13, external.Length); + JsonElement[] gates = local.Concat(external).ToArray(); + Assert.Equal(gates.Length, gates.Select(static gate => gate.GetProperty("id").GetString()).Distinct().Count()); + Assert.All(gates, static gate => + { + Assert.True(gate.GetProperty("status").GetString() is "pass" or "pending" or "fail"); + string evidence = Assert.IsType(gate.GetProperty("evidenceRef").GetString()); + Assert.False(Path.IsPathRooted(evidence)); + Assert.DoesNotContain("..", evidence, StringComparison.Ordinal); + Assert.True(gate.GetProperty("note").GetString()!.Length <= 240); + }); + bool allPass = gates.All(static gate => gate.GetProperty("status").GetString() == "pass"); + Assert.Equal(allPass ? "ready" : "not-ready", document.GetProperty("decision").GetString()); + + string canary = File.ReadAllText(Path.Combine(root, "scripts/run-real-network-canary.sh")); + Assert.Contains("umask 077", canary, StringComparison.Ordinal); + Assert.Contains("client-expected-failure", canary, StringComparison.Ordinal); + Assert.Contains("exit_code\" -eq 12", canary, StringComparison.Ordinal); + Assert.Contains(".addressFamily == $family", canary, StringComparison.Ordinal); + Assert.Contains("identifiers:\"not-in-summary\"", canary, StringComparison.Ordinal); + Assert.DoesNotContain("jq -c . \"$raw_log\"", canary, StringComparison.Ordinal); + + string checker = File.ReadAllText(Path.Combine(root, "eng/check_production_readiness.py")); + Assert.Contains("return 3", checker, StringComparison.Ordinal); + Assert.Contains("FORBIDDEN_KEY_PARTS", checker, StringComparison.Ordinal); + Assert.Contains("decision must be", checker, StringComparison.Ordinal); + } + private static string Property(XDocument document, string name) => document.Descendants(name).Single().Value; diff --git a/tests/FinalFactory.Rendezvous.Tests/TestClient/TestClientProcessIntegrationTests.cs b/tests/FinalFactory.Rendezvous.Tests/TestClient/TestClientProcessIntegrationTests.cs index e6a2550..02f2b32 100644 --- a/tests/FinalFactory.Rendezvous.Tests/TestClient/TestClientProcessIntegrationTests.cs +++ b/tests/FinalFactory.Rendezvous.Tests/TestClient/TestClientProcessIntegrationTests.cs @@ -178,12 +178,17 @@ public sealed class TestClientProcessIntegrationTests Assert.Contains( join.JsonEvents(), item => item.GetProperty("event").GetString() == "join.connected" - && item.GetProperty("endpointType").GetString() is "loopback" or "private"); + && item.GetProperty("endpointType").GetString() is "loopback" or "private" + && item.GetProperty("addressFamily").GetString() == "ipv4"); Assert.True(join.HasEvent("join.punch", "started"), join.DiagnosticText()); Assert.True(join.HasEvent("join.direct-connect", "started"), join.DiagnosticText()); Assert.True(join.HasEvent("join.direct-traffic", "verified"), join.DiagnosticText()); Assert.True(join.HasEvent("join.outcome-report", "accepted"), join.DiagnosticText()); Assert.True(host.HasEvent("host.direct-traffic", "verified"), host.DiagnosticText()); + Assert.Contains( + host.JsonEvents(), + item => item.GetProperty("event").GetString() == "host.direct-traffic" + && item.GetProperty("addressFamily").GetString() == "ipv4"); Assert.True(host.HasEvent("host.punch", "started"), host.DiagnosticText()); Assert.True(host.HasEvent("host.direct-connect", "connected"), host.DiagnosticText()); Assert.True(host.HasEvent("host.deregistered", "complete"), host.DiagnosticText());