feat(operations): add production readiness gate (#23)

This commit is contained in:
KyuubiYoru
2026-07-16 22:19:51 +02:00
parent 6bad659c12
commit 00d5ff7764
13 changed files with 782 additions and 6 deletions
+3
View File
@@ -122,6 +122,9 @@ simulation limits are documented in the
The current consumer evidence and remaining external gates are tracked in the
[SpaceGame consumer pilot](docs/integration/spacegame-pilot.md) and independent
[Unscouted consumer pilot](docs/integration/unscouted-pilot.md).
The fail-closed launch decision, redacted evidence matrix, and two-machine
external-network procedure are in
[production readiness and real-network canary](docs/operations/production-readiness.md).
## Development
+124
View File
@@ -0,0 +1,124 @@
{
"schemaVersion": 1,
"kind": "rendezvous-production-readiness",
"evaluatedCommit": "6bad659c123ad45ad0d9d07f93217c2e8c42d459",
"decision": "not-ready",
"localGates": [
{
"id": "immutable-release-artifacts",
"status": "pending",
"evidenceRef": "docs/operations/production-readiness.md",
"note": "Rebuild after the readiness tooling checkpoint."
},
{
"id": "debug-and-release-verification",
"status": "pending",
"evidenceRef": "docs/operations/production-readiness.md",
"note": "Re-run after the readiness tooling checkpoint."
},
{
"id": "real-consumer-pilots",
"status": "pending",
"evidenceRef": "docs/integration/spacegame-pilot.md",
"note": "Pin and re-run both real consumer revisions."
},
{
"id": "candidate-capacity-resilience",
"status": "pending",
"evidenceRef": "docs/evidence/capacity/v2/candidate-2cpu.json",
"note": "Re-run the five-minute candidate on the tooling checkpoint."
},
{
"id": "production-process-recovery",
"status": "pending",
"evidenceRef": "docs/operations/capacity-and-resilience.md",
"note": "Re-run process restart, drain, and rollback gates."
},
{
"id": "security-privacy-observability",
"status": "pending",
"evidenceRef": "docs/operations/production-readiness.md",
"note": "Re-run the combined release verification matrix."
}
],
"externalGates": [
{
"id": "public-package-empty-cache-restore",
"status": "pending",
"evidenceRef": "docs/operations/production-readiness.md",
"note": "The public registry does not currently resolve version 1.0.0."
},
{
"id": "signed-publication",
"status": "pending",
"evidenceRef": "docs/releases/README.md",
"note": "Protected release credentials and immutable tag publication are required."
},
{
"id": "source-preserving-udp-ingress",
"status": "pending",
"evidenceRef": "docs/operations/production-readiness.md",
"note": "The public ingress path needs packet-level source and reply validation."
},
{
"id": "same-lan-direct-canary",
"status": "pending",
"evidenceRef": "docs/operations/production-readiness.md",
"note": "Requires two independently operated game clients."
},
{
"id": "home-nat-direct-canary",
"status": "pending",
"evidenceRef": "docs/operations/production-readiness.md",
"note": "Requires distinct residential networks."
},
{
"id": "restrictive-cgnat-typed-failure",
"status": "pending",
"evidenceRef": "docs/operations/production-readiness.md",
"note": "Requires a known restrictive carrier topology."
},
{
"id": "firewall-blocked-udp-typed-failure",
"status": "pending",
"evidenceRef": "docs/operations/production-readiness.md",
"note": "Requires an independently controlled firewall rule."
},
{
"id": "ipv6-direct-canary",
"status": "pending",
"evidenceRef": "docs/operations/production-readiness.md",
"note": "Requires two IPv6-capable external clients and public ingress."
},
{
"id": "public-rate-shaped-capacity",
"status": "pending",
"evidenceRef": "docs/operations/capacity-and-resilience.md",
"note": "The full public HTTP and UDP traffic mix has not been measured."
},
{
"id": "one-hour-candidate-endurance",
"status": "pending",
"evidenceRef": "docs/operations/capacity-and-resilience.md",
"note": "A production-shaped one-hour candidate run is required."
},
{
"id": "alert-delivery",
"status": "pending",
"evidenceRef": "docs/operations/incident-runbooks.md",
"note": "A real alert sink must observe trigger and recovery notifications."
},
{
"id": "cold-standby-rollback-drill",
"status": "pending",
"evidenceRef": "docs/operations/capacity-and-resilience.md",
"note": "The deployment must demonstrate the host-visible recovery objective."
},
{
"id": "documentation-only-runbook-exercise",
"status": "pending",
"evidenceRef": "docs/operations/incident-runbooks.md",
"note": "An independent operator must execute the runbooks using only the docs."
}
]
}
+6
View File
@@ -116,6 +116,9 @@ rm deploy/compose/secrets/signing-key
codes are stable automation contracts. Informational events use stdout and
failures use stderr.
Successful direct-connection and direct-traffic events include the coarse
`addressFamily` value `ipv4` or `ipv6`. They never include the peer address.
The deployment smoke performs the full health, publish, join, mediation, direct
traffic, outcome-report, and cleanup flow using bounded waits:
@@ -220,3 +223,6 @@ least-scope publisher credential from the deployment secret boundary and set the
external service, mediator, and matching scope variables described in the
[secure Linux deployment smoke](../deployment/linux.md#http-and-udp-smoke).
Run representative external-network tests; loopback success is not NAT coverage.
Use the redacting, bounded
[real-network canary procedure](../operations/production-readiness.md) for formal
production evidence rather than committing raw TestClient JSON.
+191
View File
@@ -0,0 +1,191 @@
# Production-readiness decision and real-network canary
Tracking: #23
Rendezvous v1 is **not production-ready** until every required gate in
[`production-readiness-v1.json`](../evidence/production-readiness-v1.json) is
recorded as `pass`. The machine-checkable decision is intentionally fail-closed:
```bash
./scripts/check-production-readiness.sh
```
Exit `0` means every required gate is present and passing, exit `3` means the
record is valid but at least one gate is pending or failed, and exit `2` means
the record itself is malformed or contains identifier-, endpoint-, account-, or
credential-shaped data. Editing only the top-level decision cannot make the
check pass.
The checked-in record is an index, not a log archive. It contains one
repository-relative evidence reference and a short categorical note per gate.
Raw packet captures, client event streams, publisher credentials, public or
private network endpoints, listing IDs, and player/account identifiers must not
be committed.
## Required decision matrix
The local matrix covers immutable artifacts, Debug and Release verification,
both real game consumers, the candidate capacity/resilience profile,
production-process recovery, and the combined security/privacy/observability
gate. These may be reproduced by the project team on a clean candidate commit.
The external matrix remains distinct because a local namespace, loopback,
container bridge, or second process on one machine cannot prove it:
| Gate | Required evidence |
| --- | --- |
| Public package empty-cache restore | A clean machine restores the exact Client and Contracts version using only the documented public sources. |
| Signed publication | The immutable tag publishes packages, image digest, SBOMs, provenance, checksums, and verifiable signatures through the protected release workflow. |
| Source-preserving UDP ingress | Packet capture on the service host proves the mediator observes each peer's real source tuple and replies from the advertised public tuple; no UDP proxy rewrites either direction. |
| Same-LAN direct canary | Two independently operated game clients establish authenticated direct LiteNetLib traffic. |
| Home-NAT direct canary | Host and joiner on distinct residential networks establish authenticated direct LiteNetLib traffic. |
| Restrictive/CGNAT and blocked-UDP canaries | Each bounded join exits `12`, records a typed terminal category, and exposes the game-owned fallback policy without hanging or claiming success. |
| IPv6 direct canary | Two external IPv6 clients record authenticated direct traffic and an observed `ipv6` peer address family. |
| Public rate-shaped capacity | The documented HTTP/UDP workload mix meets its objectives through TLS, Kestrel, JSON, LiteNetLib, kernel sockets, and public ingress. |
| One-hour endurance | The immutable production-shaped candidate completes the one-hour profile without a state, handle, memory, readiness, or latency failure. |
| Alert delivery | A real alert sink receives both trigger and recovery notifications for the rehearsed outage. |
| Cold-standby rollback | Drain, stop, socket release, replacement start, host re-registration, and rollback meet the process and host-visible recovery objectives. |
| Documentation-only exercise | An operator who did not author the runbooks completes key rotation/revocation, outage, restart, re-registration, and rollback using only the checked-in documentation. |
Failure or missing evidence is blocking. It is never converted into an accepted
risk by changing the wording of the readiness note.
## Prepare one immutable canary build
Use the exact release candidate on every canary machine. Verify a clean checkout,
restore in locked mode, and build the TestClient before changing networks:
```bash
test -z "$(git status --porcelain)"
dotnet restore Rendezvous.slnx --locked-mode
dotnet build Rendezvous.slnx --configuration Release --no-restore
```
Keep shell tracing disabled. The host receives a short-lived, least-scope
publisher credential through `RENDEZVOUS_PUBLISHER_CREDENTIAL`; it must never be
put in an argument, coordination file, evidence file, command transcript, or
support message. Set the public HTTPS service URL and advertised UDP mediator
tuple separately. TestClient rejects credentials embedded in the service URL.
## Run a success canary across two machines
On the host machine, choose `same-lan`, `home-nat`, or `ipv6-direct`. The
coordination file is mode `0600` and contains only the temporary listing UUID.
It is not evidence; transfer it through an approved private channel, then delete
both copies.
```bash
set +x
export RENDEZVOUS_PUBLISHER_CREDENTIAL='supplied-by-the-approved-secret-boundary'
export RENDEZVOUS_CANARY_ROLE=host
export RENDEZVOUS_CANARY_TOPOLOGY=home-nat
export RENDEZVOUS_CANARY_ADDRESS_FAMILY=ipv4
export RENDEZVOUS_CANARY_HTTP_URL='https://service.example.invalid/'
export RENDEZVOUS_CANARY_UDP_ENDPOINT='203.0.113.10:9050'
export RENDEZVOUS_CANARY_COORDINATION_FILE="$HOME/.local/state/rendezvous-canary-listing"
export RENDEZVOUS_CANARY_OUTPUT="$PWD/artifacts/canary/home-nat-host.json"
./scripts/run-real-network-canary.sh
```
The host prints only that it is ready and waits for the authenticated exchange.
On the joiner, read the securely transferred UUID without placing it in shell
history and run the matching topology:
```bash
set +x
read -r RENDEZVOUS_CANARY_LISTING_ID < "$HOME/.local/state/rendezvous-canary-listing"
export RENDEZVOUS_CANARY_LISTING_ID
export RENDEZVOUS_CANARY_ROLE=client-success
export RENDEZVOUS_CANARY_TOPOLOGY=home-nat
export RENDEZVOUS_CANARY_ADDRESS_FAMILY=ipv4
export RENDEZVOUS_CANARY_HTTP_URL='https://service.example.invalid/'
export RENDEZVOUS_CANARY_UDP_ENDPOINT='203.0.113.10:9050'
export RENDEZVOUS_CANARY_OUTPUT="$PWD/artifacts/canary/home-nat-client.json"
./scripts/run-real-network-canary.sh
unset RENDEZVOUS_CANARY_LISTING_ID
```
The host summary requires authenticated direct traffic and deregistration. The
client summary requires connection, authenticated direct traffic, accepted
outcome reporting, and the declared address family observed on the actual peer.
The summaries deliberately contain no network tuple or listing identifier.
For IPv6, set the topology to `ipv6-direct`, the family to `ipv6`, and use the
deployment's bracketed IPv6 mediator form. Record unsupported operating systems,
console platforms, VPNs, and address families as untested; an IPv4 pass is not
evidence for IPv6 or a platform network policy.
## Run a bounded failure canary
Start the host from an independently reachable network as above. On the joiner,
apply the reviewed firewall rule that blocks the relevant UDP path, or use the
known restrictive carrier network, then set `client-expected-failure` and the
matching topology:
```bash
export RENDEZVOUS_CANARY_ROLE=client-expected-failure
export RENDEZVOUS_CANARY_TOPOLOGY=firewall-blocked-udp
export RENDEZVOUS_CANARY_ADDRESS_FAMILY=ipv4
export RENDEZVOUS_CANARY_OUTPUT="$PWD/artifacts/canary/firewall-blocked-client.json"
./scripts/run-real-network-canary.sh
```
This role passes only when TestClient exits exactly `12`, emits a non-empty typed
authorization/traversal outcome, and emits the authoritative fallback category.
A timeout without the typed terminal outcome, exit `0`, direct-traffic success,
or an unbounded process is a failed canary. Restore the firewall after the drill
and verify normal traffic again.
## Private diagnostics and retention
The harness creates raw JSON events under a randomly named `0700`-equivalent
temporary directory with a process `umask` of `077`. Successful raw events are
deleted automatically. On failure they remain in that private directory so the
operator can triage locally; do not attach them to an issue before removing
listing IDs and reviewing every field. Set `RENDEZVOUS_CANARY_KEEP_RAW=true`
only for an approved short-lived diagnostic capture, then delete it manually.
The sanitized summary contains the commit, clean/dirty tree state, UTC time,
role, declared topology, observed address-family gate, aggregate booleans, and
the retention policy. Formal evidence requires the default clean-tree check.
## Public ingress proof
Success through a public hostname is insufficient proof that UDP source/reply
addressing is preserved. During a canary, an authorized operator must capture
only packet headers at the service host and verify:
1. each authenticated contribution reaches the mediator with the external peer
source tuple visible to the server;
2. introductions are sent from the same advertised public mediator tuple;
3. no load balancer, user-space proxy, service mesh, or destination NAT changes
the source or reply tuple expected by LiteNetLib; and
4. malformed or unauthenticated traffic receives no amplified response.
Store the approval, capture time window, candidate digest, topology category,
and pass/fail result. Do not retain packet payloads or peer tuples in the
repository. A failed tuple check blocks release even if one canary happened to
connect.
## Rehearsal and triage
Run the security, capacity, observability, deployment, rollback, privacy, and
incident procedures against the same immutable candidate. The independent
operator records which runbook revision they followed, start/end time, observed
alerts, recovery time, unexpected decisions, and pass/fail result. Update the
documentation and repeat any failed or ambiguous step.
Before changing the readiness record, reconcile every open roadmap issue as one
of: `blocking` with an owner and evidence needed, `accepted-v1` with a bounded
documented limitation, or `post-v1` with a filed issue. HA, active-active or
multi-region routing, relays, platform authentication, and scale above the
single-active v1 envelope are not silently accepted; each needs a traceable
post-v1 issue. The current follow-ups are relay decision [#24], HA/multi-region
shared state and routing [#28], scale beyond the measured envelope [#29], and
platform authentication adapters [#30]. Run the checker after every evidence
update. Only its `READY` result may support a production-ready claim.
[#24]: https://git.finalfactory.de/HeiKyu/Rendezvous/issues/24
[#28]: https://git.finalfactory.de/HeiKyu/Rendezvous/issues/28
[#29]: https://git.finalfactory.de/HeiKyu/Rendezvous/issues/29
[#30]: https://git.finalfactory.de/HeiKyu/Rendezvous/issues/30
+147
View File
@@ -0,0 +1,147 @@
#!/usr/bin/env python3
"""Validate the redacted v1 readiness record and emit the release decision."""
from __future__ import annotations
import json
import pathlib
import re
import sys
from typing import Any
LOCAL_GATES = {
"immutable-release-artifacts",
"debug-and-release-verification",
"real-consumer-pilots",
"candidate-capacity-resilience",
"production-process-recovery",
"security-privacy-observability",
}
EXTERNAL_GATES = {
"public-package-empty-cache-restore",
"signed-publication",
"source-preserving-udp-ingress",
"same-lan-direct-canary",
"home-nat-direct-canary",
"restrictive-cgnat-typed-failure",
"firewall-blocked-udp-typed-failure",
"ipv6-direct-canary",
"public-rate-shaped-capacity",
"one-hour-candidate-endurance",
"alert-delivery",
"cold-standby-rollback-drill",
"documentation-only-runbook-exercise",
}
STATUSES = {"pass", "pending", "fail"}
FORBIDDEN_KEY_PARTS = {
"address",
"credential",
"endpoint",
"listingid",
"password",
"playerid",
"secret",
"token",
"userid",
}
UUID = re.compile(r"\b[0-9a-fA-F]{8}-[0-9a-fA-F-]{27,}\b")
IPV4 = re.compile(r"(?<![0-9])(?:[0-9]{1,3}\.){3}[0-9]{1,3}(?![0-9])")
COMMIT = re.compile(r"[0-9a-f]{40}")
class InvalidRecord(ValueError):
pass
def reject_sensitive(value: Any, path: str = "$") -> None:
if isinstance(value, dict):
for key, child in value.items():
normalized = re.sub(r"[^a-z0-9]", "", key.lower())
if any(part in normalized for part in FORBIDDEN_KEY_PARTS):
raise InvalidRecord(f"{path}.{key} uses a forbidden sensitive-data key")
reject_sensitive(child, f"{path}.{key}")
elif isinstance(value, list):
for index, child in enumerate(value):
reject_sensitive(child, f"{path}[{index}]")
elif isinstance(value, str):
if UUID.search(value) or IPV4.search(value) or "://" in value or "@" in value:
raise InvalidRecord(f"{path} contains endpoint, identifier, or account-shaped data")
def validate_gate_set(items: Any, expected: set[str], path: str) -> list[dict[str, str]]:
if not isinstance(items, list):
raise InvalidRecord(f"{path} must be an array")
gates: list[dict[str, str]] = []
for index, item in enumerate(items):
if not isinstance(item, dict) or set(item) != {"id", "status", "evidenceRef", "note"}:
raise InvalidRecord(f"{path}[{index}] has an invalid shape")
if not all(isinstance(item[key], str) for key in item):
raise InvalidRecord(f"{path}[{index}] fields must be strings")
if item["status"] not in STATUSES:
raise InvalidRecord(f"{path}[{index}] has an invalid status")
evidence = pathlib.PurePosixPath(item["evidenceRef"])
if evidence.is_absolute() or ".." in evidence.parts or not item["evidenceRef"]:
raise InvalidRecord(f"{path}[{index}].evidenceRef must be a repository-relative reference")
if len(item["note"]) > 240:
raise InvalidRecord(f"{path}[{index}].note is too long")
gates.append(item)
identifiers = [gate["id"] for gate in gates]
if len(identifiers) != len(set(identifiers)):
raise InvalidRecord(f"{path} contains duplicate gate identifiers")
if set(identifiers) != expected:
missing = sorted(expected - set(identifiers))
extra = sorted(set(identifiers) - expected)
raise InvalidRecord(f"{path} gate mismatch; missing={missing}, extra={extra}")
return gates
def validate(record: Any) -> tuple[bool, list[str]]:
if not isinstance(record, dict) or set(record) != {
"schemaVersion",
"kind",
"evaluatedCommit",
"decision",
"localGates",
"externalGates",
}:
raise InvalidRecord("The top-level readiness record shape is invalid")
if record["schemaVersion"] != 1 or record["kind"] != "rendezvous-production-readiness":
raise InvalidRecord("The readiness schema identity is invalid")
if not isinstance(record["evaluatedCommit"], str) or not COMMIT.fullmatch(record["evaluatedCommit"]):
raise InvalidRecord("evaluatedCommit must be a full lowercase Git commit")
reject_sensitive(record)
gates = validate_gate_set(record["localGates"], LOCAL_GATES, "$.localGates")
gates += validate_gate_set(record["externalGates"], EXTERNAL_GATES, "$.externalGates")
blockers = sorted(gate["id"] for gate in gates if gate["status"] != "pass")
ready = not blockers
expected_decision = "ready" if ready else "not-ready"
if record["decision"] != expected_decision:
raise InvalidRecord(
f"decision must be {expected_decision!r} for the recorded gate statuses"
)
return ready, blockers
def main() -> int:
if len(sys.argv) != 2:
print("usage: check_production_readiness.py RECORD", file=sys.stderr)
return 2
try:
with open(sys.argv[1], "r", encoding="utf-8") as source:
record = json.load(source)
ready, blockers = validate(record)
except (OSError, json.JSONDecodeError, InvalidRecord) as error:
print(f"INVALID: {error}", file=sys.stderr)
return 2
if not ready:
print(f"NOT READY: {len(blockers)} required gate(s) are not passing.")
for blocker in blockers:
print(f"- {blocker}")
return 3
print("READY: every required v1 production gate is recorded as passing.")
return 0
if __name__ == "__main__":
raise SystemExit(main())
+2 -2
View File
@@ -4,13 +4,13 @@
{
"name": "SpaceGame",
"repository": "https://git.finalfactory.de/Kyuubi/SpaceGame.git",
"revision": "77519b0cc418a27f8d408ae2d7b8812fbe087c04",
"revision": "f3f5bc29810c362656cd7143bec1ddc2cfaf9f22",
"project": "SpaceGame.csproj"
},
{
"name": "Unscouted",
"repository": "https://git.finalfactory.de/HeiKyu/Unscouted.git",
"revision": "7807dbee86eb8b98e702f1eb89c88adff728f635",
"revision": "f0574a7de82aadff6495ca5657dfc19cf7c2f67c",
"project": "Net.Core/Net.Core.csproj"
}
]
+7
View File
@@ -0,0 +1,7 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
RECORD="${1:-$ROOT/docs/evidence/production-readiness-v1.json}"
exec python3 "$ROOT/eng/check_production_readiness.py" "$RECORD"
+234
View File
@@ -0,0 +1,234 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
PROJECT="$ROOT/src/FinalFactory.Rendezvous.TestClient/FinalFactory.Rendezvous.TestClient.csproj"
ROLE="${RENDEZVOUS_CANARY_ROLE:-}"
TOPOLOGY="${RENDEZVOUS_CANARY_TOPOLOGY:-}"
ADDRESS_FAMILY="${RENDEZVOUS_CANARY_ADDRESS_FAMILY:-ipv4}"
SERVICE_URL="${RENDEZVOUS_CANARY_HTTP_URL:-}"
MEDIATOR="${RENDEZVOUS_CANARY_UDP_ENDPOINT:-}"
GAME_ID="${RENDEZVOUS_CANARY_GAME_ID:-space-game}"
ENVIRONMENT_ID="${RENDEZVOUS_CANARY_ENVIRONMENT_ID:-production-canary}"
REGION="${RENDEZVOUS_CANARY_REGION:-production-canary}"
PROTOCOL_VERSION="${RENDEZVOUS_CANARY_PROTOCOL_VERSION:-1}"
TIMEOUT_SECONDS="${RENDEZVOUS_CANARY_TIMEOUT_SECONDS:-60}"
RUN_SECONDS="${RENDEZVOUS_CANARY_RUN_SECONDS:-900}"
OUTPUT="${RENDEZVOUS_CANARY_OUTPUT:-$ROOT/artifacts/canary/${ROLE:-unknown}-${TOPOLOGY:-unknown}.json}"
COORDINATION_FILE="${RENDEZVOUS_CANARY_COORDINATION_FILE:-}"
LISTING_ID="${RENDEZVOUS_CANARY_LISTING_ID:-}"
REQUIRE_CLEAN="${RENDEZVOUS_CANARY_REQUIRE_CLEAN:-true}"
KEEP_RAW="${RENDEZVOUS_CANARY_KEEP_RAW:-false}"
usage() {
printf '%s\n' \
'Set RENDEZVOUS_CANARY_ROLE to host, client-success, or client-expected-failure.' \
'Also set RENDEZVOUS_CANARY_TOPOLOGY, RENDEZVOUS_CANARY_HTTP_URL, and' \
'RENDEZVOUS_CANARY_UDP_ENDPOINT. See docs/operations/production-readiness.md.' >&2
exit 2
}
for command in date dotnet git jq mktemp; do
command -v "$command" >/dev/null || {
printf 'Missing required command: %s\n' "$command" >&2
exit 2
}
done
case "$ROLE" in
host|client-success|client-expected-failure) ;;
*) usage ;;
esac
case "$TOPOLOGY" in
same-lan|home-nat|firewall-blocked-udp|restrictive-cgnat|ipv6-direct) ;;
*) usage ;;
esac
case "$ADDRESS_FAMILY" in
ipv4|ipv6) ;;
*) printf 'RENDEZVOUS_CANARY_ADDRESS_FAMILY must be ipv4 or ipv6.\n' >&2; exit 2 ;;
esac
if [[ "$TOPOLOGY" == ipv6-direct && "$ADDRESS_FAMILY" != ipv6 ]]; then
printf 'The ipv6-direct topology requires RENDEZVOUS_CANARY_ADDRESS_FAMILY=ipv6.\n' >&2
exit 2
fi
if [[ "$TOPOLOGY" =~ ^(firewall-blocked-udp|restrictive-cgnat)$ \
&& "$ROLE" == client-success ]]; then
printf 'Failure topologies must use the client-expected-failure role.\n' >&2
exit 2
fi
if [[ -z "$SERVICE_URL" || -z "$MEDIATOR" ]]; then
usage
fi
if [[ ! "$TIMEOUT_SECONDS" =~ ^[0-9]+$ ]] \
|| (( TIMEOUT_SECONDS < 1 || TIMEOUT_SECONDS > 300 )); then
printf 'RENDEZVOUS_CANARY_TIMEOUT_SECONDS must be an integer from 1 through 300.\n' >&2
exit 2
fi
if [[ ! "$RUN_SECONDS" =~ ^[0-9]+$ ]] \
|| (( RUN_SECONDS < 60 || RUN_SECONDS > 3600 )); then
printf 'RENDEZVOUS_CANARY_RUN_SECONDS must be an integer from 60 through 3600.\n' >&2
exit 2
fi
if [[ ! "$PROTOCOL_VERSION" =~ ^[0-9]+$ ]] || (( PROTOCOL_VERSION < 1 )); then
printf 'RENDEZVOUS_CANARY_PROTOCOL_VERSION must be a positive integer.\n' >&2
exit 2
fi
if [[ "$REQUIRE_CLEAN" != true && "$REQUIRE_CLEAN" != false ]]; then
printf 'RENDEZVOUS_CANARY_REQUIRE_CLEAN must be true or false.\n' >&2
exit 2
fi
if [[ "$KEEP_RAW" != true && "$KEEP_RAW" != false ]]; then
printf 'RENDEZVOUS_CANARY_KEEP_RAW must be true or false.\n' >&2
exit 2
fi
cd "$ROOT"
commit="$(git rev-parse HEAD)"
tree_state=clean
if [[ -n "$(git status --porcelain)" ]]; then
tree_state=dirty
fi
if [[ "$REQUIRE_CLEAN" == true && "$tree_state" != clean ]]; then
printf 'Formal canary evidence requires a clean source tree.\n' >&2
exit 2
fi
if [[ "$ROLE" == host ]]; then
if [[ -z "$COORDINATION_FILE" ]]; then
printf 'The host role requires RENDEZVOUS_CANARY_COORDINATION_FILE.\n' >&2
exit 2
fi
if [[ -z "${RENDEZVOUS_PUBLISHER_CREDENTIAL:-}" ]]; then
printf 'The host role requires RENDEZVOUS_PUBLISHER_CREDENTIAL.\n' >&2
exit 2
fi
else
if [[ ! "$LISTING_ID" =~ ^[0-9a-fA-F-]{36}$ ]]; then
printf 'A client role requires a UUID in RENDEZVOUS_CANARY_LISTING_ID.\n' >&2
exit 2
fi
fi
umask 077
raw_dir="$(mktemp -d "${TMPDIR:-/tmp}/rendezvous-canary.XXXXXXXX")"
raw_log="$raw_dir/events.jsonl"
run_succeeded=false
host_pid=''
cleanup() {
local status="$?"
if [[ -n "$host_pid" ]] && kill -0 "$host_pid" 2>/dev/null; then
kill -TERM "$host_pid" 2>/dev/null || true
wait "$host_pid" 2>/dev/null || true
fi
if [[ "$run_succeeded" == true && "$KEEP_RAW" == false ]]; then
rm -rf "$raw_dir"
else
printf 'Private raw canary events retained at %s\n' "$raw_dir" >&2
fi
return "$status"
}
trap cleanup EXIT
trap 'exit 130' INT
trap 'exit 143' TERM
common_arguments=(
--service "$SERVICE_URL"
--mediator "$MEDIATOR"
--game "$GAME_ID"
--environment "$ENVIRONMENT_ID"
--region "$REGION"
--protocol "$PROTOCOL_VERSION"
--script
--json
--timeout-seconds "$TIMEOUT_SECONDS"
)
exit_code=0
if [[ "$ROLE" == host ]]; then
dotnet run --project "$PROJECT" --configuration Release --no-build -- \
host "${common_arguments[@]}" --exit-after-echo --run-seconds "$RUN_SECONDS" \
>"$raw_log" 2>&1 &
host_pid="$!"
ready=false
for ((iteration = 0; iteration < TIMEOUT_SECONDS * 4; iteration++)); do
if jq -e 'select(.event == "host.ready" and .status == "ready")' "$raw_log" \
>/dev/null 2>&1; then
ready=true
break
fi
if ! kill -0 "$host_pid" 2>/dev/null; then
break
fi
sleep 0.25
done
if [[ "$ready" != true ]]; then
printf 'The canary host did not become ready within the bounded startup window.\n' >&2
kill -TERM "$host_pid" 2>/dev/null || true
wait "$host_pid" 2>/dev/null || true
exit 1
fi
observed_listing="$(jq -r 'select(.event == "host.registered") | .listingId' "$raw_log" | tail -n 1)"
if [[ ! "$observed_listing" =~ ^[0-9a-f-]{36}$ ]]; then
printf 'The canary host did not produce a valid coordination identifier.\n' >&2
kill -TERM "$host_pid" 2>/dev/null || true
wait "$host_pid" 2>/dev/null || true
exit 1
fi
coordination_parent="$(dirname "$COORDINATION_FILE")"
mkdir -p "$coordination_parent"
coordination_temp="$COORDINATION_FILE.tmp.$$"
printf '%s\n' "$observed_listing" >"$coordination_temp"
chmod 600 "$coordination_temp"
mv "$coordination_temp" "$COORDINATION_FILE"
printf 'Host ready; securely transfer the private coordination file to the client operator.\n'
set +e
wait "$host_pid"
exit_code="$?"
set -e
elif [[ "$ROLE" == client-success ]]; then
set +e
dotnet run --project "$PROJECT" --configuration Release --no-build -- \
join "${common_arguments[@]}" --listing "$LISTING_ID" >"$raw_log" 2>&1
exit_code="$?"
set -e
else
set +e
dotnet run --project "$PROJECT" --configuration Release --no-build -- \
join "${common_arguments[@]}" --listing "$LISTING_ID" >"$raw_log" 2>&1
exit_code="$?"
set -e
fi
checks='{}'
if [[ "$ROLE" == host ]]; then
[[ "$exit_code" -eq 0 ]]
jq -e --arg family "$ADDRESS_FAMILY" 'select(.event == "host.direct-traffic" and .status == "verified" and .addressFamily == $family)' "$raw_log" >/dev/null
jq -e 'select(.event == "host.deregistered" and .status == "complete")' "$raw_log" >/dev/null
checks='{"authenticatedDirectTraffic":true,"deregistered":true}'
elif [[ "$ROLE" == client-success ]]; then
[[ "$exit_code" -eq 0 ]]
jq -e --arg family "$ADDRESS_FAMILY" 'select(.event == "join.connected" and .status == "connected" and .addressFamily == $family)' "$raw_log" >/dev/null
jq -e --arg family "$ADDRESS_FAMILY" 'select(.event == "join.direct-traffic" and .status == "verified" and .addressFamily == $family)' "$raw_log" >/dev/null
jq -e 'select(.event == "join.outcome-report" and .status == "accepted")' "$raw_log" >/dev/null
checks='{"authenticatedDirectTraffic":true,"typedOutcomeReported":true}'
else
[[ "$exit_code" -eq 12 ]]
jq -e 'select((.event == "join.traversal" or .event == "join.authorization") and .status == "failed" and (.outcome | type == "string") and (.outcome | length > 0))' "$raw_log" >/dev/null
jq -e 'select(.event == "join.fallback" and (.status == "available" or .status == "unavailable") and (.outcome | type == "string") and (.outcome | length > 0))' "$raw_log" >/dev/null
checks='{"boundedTypedFailure":true,"fallbackPolicyReported":true}'
fi
mkdir -p "$(dirname "$OUTPUT")"
jq -n \
--arg commit "$commit" \
--arg treeState "$tree_state" \
--arg timestampUtc "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \
--arg role "$ROLE" \
--arg topology "$TOPOLOGY" \
--arg addressFamily "$ADDRESS_FAMILY" \
--argjson checks "$checks" \
'{schemaVersion:1,kind:"rendezvous-real-network-canary",commit:$commit,treeState:$treeState,timestampUtc:$timestampUtc,role:$role,topology:$topology,addressFamily:$addressFamily,result:"pass",checks:$checks,dataRetention:{rawEvents:"deleted-after-success",identifiers:"not-in-summary",networkEndpoints:"not-in-summary"}}' \
>"$OUTPUT"
run_succeeded=true
printf 'Real-network canary passed; sanitized evidence: %s\n' "$OUTPUT"
+2
View File
@@ -47,6 +47,8 @@ for ((index = 0; index < count; index++)); do
cat >"$targets" <<EOF
<Project>
<ItemGroup Condition="'\$(MSBuildProjectFullPath)' == '$project'">
<PackageReference Remove="FinalFactory.Rendezvous.Client" />
<PackageReference Remove="FinalFactory.Rendezvous.Contracts" />
<PackageReference Include="FinalFactory.Rendezvous.Client" Version="[$version]" />
<PackageReference Include="FinalFactory.Rendezvous.Contracts" Version="[$version]" />
</ItemGroup>
@@ -114,11 +114,12 @@ internal sealed class RendezvousCommandRunner : ITestClientCommandRunner
listingId: session.ListingId.ToString(),
displayName: options.DisplayName);
echo = new DirectEchoProtocol(events.GameplayEvents, host: true);
echo.ExchangeCompleted += _ => output.Write(
echo.ExchangeCompleted += peer => output.Write(
"host.direct-traffic",
"verified",
phase: "direct-traffic",
endpointType: "peer-to-peer");
endpointType: "peer-to-peer",
addressFamily: AddressFamilyName(peer.Address));
coordinator = new RendezvousHostCoordinator(
manager,
events,
@@ -485,6 +486,7 @@ internal sealed class RendezvousCommandRunner : ITestClientCommandRunner
"connected",
phase: "direct-connection",
endpointType: endpointType,
addressFamily: AddressFamilyName(peer.Address),
elapsedMilliseconds: ToMilliseconds(outcome.Elapsed));
await ReportOutcomeAsync(coordinator, joins, output, cancellationToken).ConfigureAwait(false);
echo.BeginJoin(peer);
@@ -507,7 +509,8 @@ internal sealed class RendezvousCommandRunner : ITestClientCommandRunner
"join.direct-traffic",
"verified",
phase: "direct-traffic",
endpointType: endpointType);
endpointType: endpointType,
addressFamily: AddressFamilyName(peer.Address));
peer.Disconnect();
manager.PollEvents();
return TestClientExitCode.Success;
@@ -765,6 +768,9 @@ internal sealed class RendezvousCommandRunner : ITestClientCommandRunner
return privateAddress ? "private" : "public";
}
private static string AddressFamilyName(IPAddress address) =>
address.AddressFamily == AddressFamily.InterNetworkV6 ? "ipv6" : "ipv4";
private static long ToMilliseconds(TimeSpan elapsed) =>
(long)Math.Min(long.MaxValue, Math.Max(0, elapsed.TotalMilliseconds));
@@ -24,6 +24,7 @@ internal sealed class TestClientOutput(TextWriter standardOutput, TextWriter sta
string? displayName = null,
string? outcome = null,
string? endpointType = null,
string? addressFamily = null,
int? count = null,
long? elapsedMilliseconds = null,
string? message = null) => WriteCore(
@@ -37,6 +38,7 @@ internal sealed class TestClientOutput(TextWriter standardOutput, TextWriter sta
DisplayName = SafeText(displayName),
Outcome = SafeToken(outcome),
EndpointType = SafeToken(endpointType),
AddressFamily = SafeToken(addressFamily),
Count = count,
ElapsedMilliseconds = elapsedMilliseconds,
Message = SafeText(message),
@@ -92,6 +94,7 @@ internal sealed class TestClientOutput(TextWriter standardOutput, TextWriter sta
Append(line, "name", item.DisplayName, quote: true);
Append(line, "outcome", item.Outcome);
Append(line, "endpoint", item.EndpointType);
Append(line, "addressFamily", item.AddressFamily);
if (item.Count.HasValue)
{
Append(line, "count", item.Count.Value.ToString(System.Globalization.CultureInfo.InvariantCulture));
@@ -174,6 +177,7 @@ internal sealed class TestClientOutput(TextWriter standardOutput, TextWriter sta
public string? DisplayName { get; init; }
public string? Outcome { get; init; }
public string? EndpointType { get; init; }
public string? AddressFamily { get; init; }
public int? Count { get; init; }
public long? ElapsedMilliseconds { get; init; }
public string? Message { get; init; }
@@ -143,6 +143,10 @@ public sealed class ReleaseCompatibilityTests
pinnedConsumers.Select(static item => item.GetProperty("name").GetString()!).ToArray());
Assert.All(pinnedConsumers, static item =>
Assert.Matches("^[0-9a-f]{40}$", item.GetProperty("revision").GetString()));
string realConsumerGate = File.ReadAllText(Path.Combine(root, "scripts", "verify-real-consumers.sh"));
Assert.Contains("<PackageReference Remove=\"FinalFactory.Rendezvous.Client\" />", realConsumerGate, StringComparison.Ordinal);
Assert.Contains("<PackageReference Remove=\"FinalFactory.Rendezvous.Contracts\" />", realConsumerGate, StringComparison.Ordinal);
}
[Fact]
@@ -164,6 +168,49 @@ public sealed class ReleaseCompatibilityTests
Assert.Equal(actual, declared);
}
[Fact]
public void ProductionReadinessRecordIsFailClosedAndCanaryEvidenceIsRedacted()
{
string root = FindRepositoryRoot();
using JsonDocument readiness = JsonDocument.Parse(File.ReadAllText(Path.Combine(
root,
"docs/evidence/production-readiness-v1.json")));
JsonElement document = readiness.RootElement;
Assert.Equal(1, document.GetProperty("schemaVersion").GetInt32());
Assert.Equal("rendezvous-production-readiness", document.GetProperty("kind").GetString());
Assert.Matches("^[0-9a-f]{40}$", document.GetProperty("evaluatedCommit").GetString());
JsonElement[] local = document.GetProperty("localGates").EnumerateArray().ToArray();
JsonElement[] external = document.GetProperty("externalGates").EnumerateArray().ToArray();
Assert.Equal(6, local.Length);
Assert.Equal(13, external.Length);
JsonElement[] gates = local.Concat(external).ToArray();
Assert.Equal(gates.Length, gates.Select(static gate => gate.GetProperty("id").GetString()).Distinct().Count());
Assert.All(gates, static gate =>
{
Assert.True(gate.GetProperty("status").GetString() is "pass" or "pending" or "fail");
string evidence = Assert.IsType<string>(gate.GetProperty("evidenceRef").GetString());
Assert.False(Path.IsPathRooted(evidence));
Assert.DoesNotContain("..", evidence, StringComparison.Ordinal);
Assert.True(gate.GetProperty("note").GetString()!.Length <= 240);
});
bool allPass = gates.All(static gate => gate.GetProperty("status").GetString() == "pass");
Assert.Equal(allPass ? "ready" : "not-ready", document.GetProperty("decision").GetString());
string canary = File.ReadAllText(Path.Combine(root, "scripts/run-real-network-canary.sh"));
Assert.Contains("umask 077", canary, StringComparison.Ordinal);
Assert.Contains("client-expected-failure", canary, StringComparison.Ordinal);
Assert.Contains("exit_code\" -eq 12", canary, StringComparison.Ordinal);
Assert.Contains(".addressFamily == $family", canary, StringComparison.Ordinal);
Assert.Contains("identifiers:\"not-in-summary\"", canary, StringComparison.Ordinal);
Assert.DoesNotContain("jq -c . \"$raw_log\"", canary, StringComparison.Ordinal);
string checker = File.ReadAllText(Path.Combine(root, "eng/check_production_readiness.py"));
Assert.Contains("return 3", checker, StringComparison.Ordinal);
Assert.Contains("FORBIDDEN_KEY_PARTS", checker, StringComparison.Ordinal);
Assert.Contains("decision must be", checker, StringComparison.Ordinal);
}
private static string Property(XDocument document, string name) =>
document.Descendants(name).Single().Value;
@@ -178,12 +178,17 @@ public sealed class TestClientProcessIntegrationTests
Assert.Contains(
join.JsonEvents(),
item => item.GetProperty("event").GetString() == "join.connected"
&& item.GetProperty("endpointType").GetString() is "loopback" or "private");
&& item.GetProperty("endpointType").GetString() is "loopback" or "private"
&& item.GetProperty("addressFamily").GetString() == "ipv4");
Assert.True(join.HasEvent("join.punch", "started"), join.DiagnosticText());
Assert.True(join.HasEvent("join.direct-connect", "started"), join.DiagnosticText());
Assert.True(join.HasEvent("join.direct-traffic", "verified"), join.DiagnosticText());
Assert.True(join.HasEvent("join.outcome-report", "accepted"), join.DiagnosticText());
Assert.True(host.HasEvent("host.direct-traffic", "verified"), host.DiagnosticText());
Assert.Contains(
host.JsonEvents(),
item => item.GetProperty("event").GetString() == "host.direct-traffic"
&& item.GetProperty("addressFamily").GetString() == "ipv4");
Assert.True(host.HasEvent("host.punch", "started"), host.DiagnosticText());
Assert.True(host.HasEvent("host.direct-connect", "connected"), host.DiagnosticText());
Assert.True(host.HasEvent("host.deregistered", "complete"), host.DiagnosticText());