Compare commits

..

16 Commits

Author SHA1 Message Date
KyuubiYoru 589f802e2e fix(ci): smoke through service namespace (#1)
quality-gate / quality (push) Successful in 2m26s
quality-gate / container (push) Successful in 1m34s
2026-07-17 02:16:21 +02:00
KyuubiYoru 4423503bba fix(ci): reduce readiness probe pressure (#1)
quality-gate / quality (push) Successful in 2m40s
quality-gate / container (push) Failing after 4m34s
2026-07-17 02:06:39 +02:00
KyuubiYoru 74ae126ea7 fix(ci): allow cold container startup (#1)
quality-gate / quality (push) Successful in 2m39s
quality-gate / container (push) Failing after 3m11s
2026-07-17 01:59:30 +02:00
KyuubiYoru a77d4b801c fix(ci): isolate deployment smoke client (#1)
quality-gate / quality (push) Successful in 2m29s
quality-gate / container (push) Failing after 2m23s
2026-07-17 01:53:50 +02:00
KyuubiYoru 769336e424 fix(ci): run smoke on Compose network (#1)
quality-gate / quality (push) Successful in 2m37s
quality-gate / container (push) Failing after 1m39s
2026-07-17 01:42:28 +02:00
KyuubiYoru 41be7fbce5 fix(ci): join service to runner network (#1)
quality-gate / quality (push) Successful in 2m37s
quality-gate / container (push) Failing after 1m40s
2026-07-17 01:36:47 +02:00
KyuubiYoru 1bca034a52 fix(ci): route smoke through Docker gateway (#1)
quality-gate / quality (push) Successful in 2m37s
quality-gate / container (push) Failing after 1m51s
2026-07-17 01:29:30 +02:00
KyuubiYoru 8d98d888b3 fix(ci): resolve sibling-container bind sources (#1)
quality-gate / quality (push) Successful in 2m36s
quality-gate / container (push) Failing after 1m46s
2026-07-17 01:23:16 +02:00
KyuubiYoru ef07685d22 fix(ci): correct Docker inspect templates (#1)
quality-gate / quality (push) Successful in 2m36s
quality-gate / container (push) Failing after 1m42s
2026-07-17 01:14:44 +02:00
KyuubiYoru f9cb7f47e5 fix(ci): isolate Compose host ports (#1)
quality-gate / quality (push) Successful in 2m46s
quality-gate / container (push) Failing after 1m28s
2026-07-17 01:07:56 +02:00
KyuubiYoru de753b99e6 fix(deploy): validate complete smoke logs (#1)
quality-gate / quality (push) Successful in 2m47s
quality-gate / container (push) Failing after 2m18s
2026-07-17 00:58:28 +02:00
KyuubiYoru c3b3629515 fix(deploy): gate smoke readiness atomically (#1)
quality-gate / quality (push) Failing after 2m14s
quality-gate / container (push) Has been skipped
2026-07-17 00:54:04 +02:00
KyuubiYoru e8c07ee22a fix(deploy): synchronize smoke listing lookup (#1)
quality-gate / quality (push) Failing after 2m14s
quality-gate / container (push) Has been skipped
2026-07-17 00:48:13 +02:00
KyuubiYoru d42e6c99a3 fix(ci): provision TestClient runtime (#1)
quality-gate / quality (push) Failing after 2m18s
quality-gate / container (push) Has been skipped
2026-07-17 00:42:16 +02:00
KyuubiYoru c2f63db3ad test(client): stabilize stream integration gate (#1)
quality-gate / quality (push) Failing after 1m28s
quality-gate / container (push) Has been skipped
2026-07-17 00:34:29 +02:00
KyuubiYoru 99885f8c8c feat(observability): add diagnostic dashboards (#27)
quality-gate / quality (push) Failing after 1m31s
quality-gate / container (push) Has been skipped
2026-07-17 00:22:46 +02:00
38 changed files with 3386 additions and 63 deletions
+49 -13
View File
@@ -21,7 +21,9 @@ jobs:
- name: Install .NET SDK - name: Install .NET SDK
uses: actions/setup-dotnet@67a3573c9a986a3f9c594539f4ab511d57bb3ce9 # v4.3.1 uses: actions/setup-dotnet@67a3573c9a986a3f9c594539f4ab511d57bb3ce9 # v4.3.1
with: with:
dotnet-version: 10.0.301 dotnet-version: |
8.0.128
10.0.301
- name: Restore locked dependencies - name: Restore locked dependencies
run: dotnet restore Rendezvous.slnx --locked-mode run: dotnet restore Rendezvous.slnx --locked-mode
@@ -45,6 +47,12 @@ jobs:
- name: Verify formatting and analyzers - name: Verify formatting and analyzers
run: dotnet format Rendezvous.slnx --verify-no-changes --no-restore run: dotnet format Rendezvous.slnx --verify-no-changes --no-restore
- name: Test dependency-free diagnostic dashboard
run: ./scripts/test-diagnostic-dashboard.sh
- name: Test observability dashboard provisioning
run: ./scripts/test-observability-assets.sh
- name: Build - name: Build
run: dotnet build Rendezvous.slnx --configuration Release --no-restore run: dotnet build Rendezvous.slnx --configuration Release --no-restore
@@ -121,7 +129,9 @@ jobs:
- name: Install .NET SDK - name: Install .NET SDK
uses: actions/setup-dotnet@67a3573c9a986a3f9c594539f4ab511d57bb3ce9 # v4.3.1 uses: actions/setup-dotnet@67a3573c9a986a3f9c594539f4ab511d57bb3ce9 # v4.3.1
with: with:
dotnet-version: 10.0.301 dotnet-version: |
8.0.128
10.0.301
- name: Build deployment diagnostic - name: Build deployment diagnostic
run: | run: |
@@ -140,12 +150,29 @@ jobs:
rm -f "$secret" rm -f "$secret"
} }
trap cleanup EXIT trap cleanup EXIT
umask 077
install -d -m 0700 deploy/compose/secrets install -d -m 0700 deploy/compose/secrets
openssl rand -out "$secret" 32 openssl rand -out "$secret" 32
chmod 0600 "$secret"
publisher_credential="$(RENDEZVOUS_SMOKE_LOCAL_KEY="$secret" \
./scripts/mint-local-publisher-credential.sh)"
export RENDEZVOUS_PUBLISHER_CREDENTIAL="$publisher_credential"
chmod 0444 "$secret" chmod 0444 "$secret"
export RENDEZVOUS_UID=1654 export RENDEZVOUS_UID=1654
export RENDEZVOUS_GID=1654 export RENDEZVOUS_GID=1654
port_suffix="$(( ${GITHUB_RUN_ID:-$$} % 10000 ))"
export RENDEZVOUS_HTTP_HOST_PORT="$(( 20000 + port_suffix ))"
export RENDEZVOUS_UDP_HOST_PORT="$(( 40000 + port_suffix ))"
runner_workspace_source="$(docker inspect "$HOSTNAME" | jq -er \
--arg destination "$GITHUB_WORKSPACE" \
'.[0].Mounts[] | select(.Destination == $destination) | .Source')"
export RENDEZVOUS_CONFIG_SOURCE="$runner_workspace_source/deploy/compose/appsettings.Production.json"
export RENDEZVOUS_SECRET_SOURCE="$runner_workspace_source/deploy/compose/secrets/signing-key"
export SOURCE_REVISION_ID="$GITHUB_SHA" export SOURCE_REVISION_ID="$GITHUB_SHA"
smoke_client_image="rendezvous-smoke-client:${GITHUB_SHA}"
docker build --file eng/release-builder.Dockerfile \
--target release-builder \
--tag "$smoke_client_image" .
docker compose -f "$compose_file" build \ docker compose -f "$compose_file" build \
--build-arg SOURCE_REVISION_ID="$SOURCE_REVISION_ID" --build-arg SOURCE_REVISION_ID="$SOURCE_REVISION_ID"
docker compose -f "$compose_file" up --no-build --detach docker compose -f "$compose_file" up --no-build --detach
@@ -153,19 +180,28 @@ jobs:
test -n "$container_id" test -n "$container_id"
test "$(docker inspect --format '{{.Config.User}}' "$container_id")" = "1654:1654" test "$(docker inspect --format '{{.Config.User}}' "$container_id")" = "1654:1654"
test "$(docker inspect --format '{{.HostConfig.ReadonlyRootfs}}' "$container_id")" = "true" test "$(docker inspect --format '{{.HostConfig.ReadonlyRootfs}}' "$container_id")" = "true"
test "$(docker inspect --format '{{range .Mounts}}{{if eq .Destination \"/app/appsettings.Production.json\"}}{{.RW}}{{end}}{{end}}' "$container_id")" = "false" test "$(docker inspect --format '{{range .Mounts}}{{if eq .Destination "/app/appsettings.Production.json"}}{{.RW}}{{end}}{{end}}' "$container_id")" = "false"
test "$(docker inspect --format '{{range .Mounts}}{{if eq .Destination \"/run/secrets/rendezvous-signing-key\"}}{{.RW}}{{end}}{{end}}' "$container_id")" = "false" test "$(docker inspect --format '{{range .Mounts}}{{if eq .Destination "/run/secrets/rendezvous-signing-key"}}{{.RW}}{{end}}{{end}}' "$container_id")" = "false"
for attempt in {1..100}; do docker run --rm \
if curl --fail --silent http://127.0.0.1:8080/health/ready >/dev/null 2>&1; then --network "container:${container_id}" \
break --user "$(id -u):$(id -g)" \
fi --env HOME=/tmp \
if (( attempt == 100 )); then --env RENDEZVOUS_SMOKE_HTTP_URL=http://127.0.0.1:8080/ \
--env RENDEZVOUS_SMOKE_UDP_ENDPOINT=127.0.0.1:9050 \
--env RENDEZVOUS_PUBLISHER_CREDENTIAL \
--volume "$runner_workspace_source:/source:ro" \
--workdir /source \
"$smoke_client_image" \
bash -lc '
for attempt in {1..180}; do
curl --fail --silent "${RENDEZVOUS_SMOKE_HTTP_URL%/}/health/ready" >/dev/null 2>&1 && exec ./scripts/smoke-deployment.sh
sleep 1
done
exit 1
' || {
docker compose -f "$compose_file" logs rendezvous docker compose -f "$compose_file" logs rendezvous
exit 1 exit 1
fi }
sleep 0.1
done
./scripts/smoke-deployment.sh
docker compose -f "$compose_file" stop --timeout 40 rendezvous docker compose -f "$compose_file" stop --timeout 40 rendezvous
test "$(docker inspect --format '{{.State.Running}}' "$container_id")" = "false" test "$(docker inspect --format '{{.State.Running}}' "$container_id")" = "false"
test "$(docker inspect --format '{{.State.ExitCode}}' "$container_id")" = "0" test "$(docker inspect --format '{{.State.ExitCode}}' "$container_id")" = "0"
+37 -9
View File
@@ -20,10 +20,12 @@ jobs:
with: with:
fetch-depth: 0 fetch-depth: 0
- name: Install pinned .NET SDK - name: Install pinned .NET SDKs
uses: actions/setup-dotnet@67a3573c9a986a3f9c594539f4ab511d57bb3ce9 # v4.3.1 uses: actions/setup-dotnet@67a3573c9a986a3f9c594539f4ab511d57bb3ce9 # v4.3.1
with: with:
dotnet-version: 10.0.301 dotnet-version: |
8.0.128
10.0.301
- name: Install pinned Buildx and BuildKit - name: Install pinned Buildx and BuildKit
uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1
@@ -116,22 +118,48 @@ jobs:
rm -f "$secret" rm -f "$secret"
} }
trap cleanup EXIT trap cleanup EXIT
umask 077
install -d -m 0700 deploy/compose/secrets install -d -m 0700 deploy/compose/secrets
openssl rand -out "$secret" 32 openssl rand -out "$secret" 32
chmod 0600 "$secret"
publisher_credential="$(RENDEZVOUS_SMOKE_LOCAL_KEY="$secret" \
./scripts/mint-local-publisher-credential.sh)"
export RENDEZVOUS_PUBLISHER_CREDENTIAL="$publisher_credential"
chmod 0444 "$secret" chmod 0444 "$secret"
export RENDEZVOUS_UID=1654 export RENDEZVOUS_UID=1654
export RENDEZVOUS_GID=1654 export RENDEZVOUS_GID=1654
port_suffix="$(( ${GITHUB_RUN_ID:-$$} % 10000 ))"
export RENDEZVOUS_HTTP_HOST_PORT="$(( 20000 + port_suffix ))"
export RENDEZVOUS_UDP_HOST_PORT="$(( 40000 + port_suffix ))"
runner_workspace_source="$(docker inspect "$HOSTNAME" | jq -er \
--arg destination "$GITHUB_WORKSPACE" \
'.[0].Mounts[] | select(.Destination == $destination) | .Source')"
export RENDEZVOUS_CONFIG_SOURCE="$runner_workspace_source/deploy/compose/appsettings.Production.json"
export RENDEZVOUS_SECRET_SOURCE="$runner_workspace_source/deploy/compose/secrets/signing-key"
export RENDEZVOUS_IMAGE="git.finalfactory.de/heikyu/rendezvous:${RENDEZVOUS_VERSION}" export RENDEZVOUS_IMAGE="git.finalfactory.de/heikyu/rendezvous:${RENDEZVOUS_VERSION}"
docker compose -f deploy/compose/compose.yaml up --detach --no-build docker compose -f deploy/compose/compose.yaml up --detach --no-build
for attempt in {1..100}; do container_id="$(docker compose -f deploy/compose/compose.yaml ps -q rendezvous)"
curl --fail --silent http://127.0.0.1:8080/health/ready >/dev/null 2>&1 && break test -n "$container_id"
if (( attempt == 100 )); then docker run --rm \
--network "container:${container_id}" \
--user "$(id -u):$(id -g)" \
--env HOME=/tmp \
--env RENDEZVOUS_SMOKE_HTTP_URL=http://127.0.0.1:8080/ \
--env RENDEZVOUS_SMOKE_UDP_ENDPOINT=127.0.0.1:9050 \
--env RENDEZVOUS_PUBLISHER_CREDENTIAL \
--volume "$runner_workspace_source:/source:ro" \
--workdir /source \
"$RENDEZVOUS_RELEASE_BUILDER" \
bash -lc '
for attempt in {1..180}; do
curl --fail --silent "${RENDEZVOUS_SMOKE_HTTP_URL%/}/health/ready" >/dev/null 2>&1 && exec ./scripts/smoke-deployment.sh
sleep 1
done
exit 1
' || {
docker compose -f deploy/compose/compose.yaml logs rendezvous docker compose -f deploy/compose/compose.yaml logs rendezvous
exit 1 exit 1
fi }
sleep 0.1
done
./scripts/smoke-deployment.sh
- name: Scan candidate for high and critical vulnerabilities - name: Scan candidate for high and critical vulnerabilities
uses: aquasecurity/trivy-action@57a97c7e7821a5776cebc9bb87c984fa69cba8f1 # v0.35.0, post-incident safe SHA uses: aquasecurity/trivy-action@57a97c7e7821a5776cebc9bb87c984fa69cba8f1 # v0.35.0, post-incident safe SHA
+10 -6
View File
@@ -84,10 +84,10 @@ directory leases, authenticated join attempts, LiteNetLib mediator, caller-owned
SDK coordination, typed connection outcomes, thin public-SDK diagnostic client, SDK coordination, typed connection outcomes, thin public-SDK diagnostic client,
deterministic NAT topology harness, hostile-input controls, deterministic NAT topology harness, hostile-input controls,
observability/operator surface, secure single-active Linux deployment, and observability/operator surface, secure single-active Linux deployment, and
numeric capacity/resilience gates, and reproducible signed release pipeline are numeric capacity/resilience gates, reproducible signed release pipeline, consumer
implemented. Consumer pilots and final production-readiness gates remain in progress; pilot integrations, and production-readiness decision framework are implemented.
participating games must not treat the current repository as a finished production Deployment-specific external canaries and optional roadmap follow-ups remain;
service until those gates land. participating games must not treat a checkout alone as production approval.
The ratified v1 boundaries, trust decisions, privacy rules, safety budgets, and The ratified v1 boundaries, trust decisions, privacy rules, safety budgets, and
threat model are indexed in [the architecture documentation](docs/architecture/README.md). threat model are indexed in [the architecture documentation](docs/architecture/README.md).
@@ -100,6 +100,8 @@ defined in [hostile-input and overload protection](docs/security/abuse-protectio
Health semantics, bounded telemetry, alerting, audit privacy, and the authenticated Health semantics, bounded telemetry, alerting, audit privacy, and the authenticated
operator controls are defined in the operator controls are defined in the
[observability and operator runbook](docs/operations/observability-and-operator-runbook.md). [observability and operator runbook](docs/operations/observability-and-operator-runbook.md).
The optional public session diagnostic and the private provisioned Grafana stack
are described in [diagnostic dashboards](docs/operations/diagnostic-dashboards.md).
Concrete detect/contain/recover/verify procedures are in the Concrete detect/contain/recover/verify procedures are in the
[incident and change runbooks](docs/operations/incident-runbooks.md). [incident and change runbooks](docs/operations/incident-runbooks.md).
The pinned non-root container, production topology, graceful drain, Linux The pinned non-root container, production topology, graceful drain, Linux
@@ -132,8 +134,10 @@ external-network procedure are in
## Development ## Development
The repository pins .NET SDK 10.0.301. From a clean clone, run the same gates as The repository pins .NET SDK 10.0.301. The full process-test gate also requires
CI from the repository root: the .NET 8 runtime because the public TestClient deliberately targets `net8.0`;
CI installs SDK 8.0.128 to supply the pinned 8.0.28 runtime. From a clean clone,
run the same gates as CI from the repository root:
```bash ```bash
dotnet restore Rendezvous.slnx --locked-mode dotnet restore Rendezvous.slnx --locked-mode
@@ -14,6 +14,29 @@
"ListenAddress": "0.0.0.0", "ListenAddress": "0.0.0.0",
"Port": 9050 "Port": 9050
}, },
"Diagnostics": {
"Enabled": false,
"PollIntervalSeconds": 10,
"MaximumRenderedSessions": 100,
"Scopes": [
{
"GameId": "space-game",
"EnvironmentId": "smoke",
"ProtocolVersions": [1, 2],
"Regions": ["local"]
},
{
"GameId": "unscouted",
"EnvironmentId": "smoke",
"ProtocolVersions": [1],
"Regions": ["local"]
}
]
},
"Metrics": {
"Enabled": false,
"BearerTokenSecretReference": "file:/run/secrets/rendezvous-metrics-token"
},
"AbuseProtection": { "AbuseProtection": {
"TrustedProxyAddresses": ["127.0.0.1"], "TrustedProxyAddresses": ["127.0.0.1"],
"OperatorAllowedAddresses": ["127.0.0.1"] "OperatorAllowedAddresses": ["127.0.0.1"]
+6 -4
View File
@@ -27,9 +27,11 @@ services:
environment: environment:
ASPNETCORE_ENVIRONMENT: Production ASPNETCORE_ENVIRONMENT: Production
ASPNETCORE_HTTP_PORTS: "8080" ASPNETCORE_HTTP_PORTS: "8080"
Rendezvous__Diagnostics__Enabled: "${RENDEZVOUS_DIAGNOSTICS_ENABLED:-false}"
Rendezvous__Metrics__Enabled: "${RENDEZVOUS_METRICS_ENABLED:-false}"
volumes: volumes:
- ./appsettings.Production.json:/app/appsettings.Production.json:ro - ${RENDEZVOUS_CONFIG_SOURCE:-./appsettings.Production.json}:/app/appsettings.Production.json:ro
- ./secrets/signing-key:/run/secrets/rendezvous-signing-key:ro - ${RENDEZVOUS_SECRET_SOURCE:-./secrets/signing-key}:/run/secrets/rendezvous-signing-key:ro
ports: ports:
- "127.0.0.1:8080:8080/tcp" - "127.0.0.1:${RENDEZVOUS_HTTP_HOST_PORT:-8080}:8080/tcp"
- "9050:9050/udp" - "${RENDEZVOUS_UDP_HOST_PORT:-9050}:9050/udp"
+71
View File
@@ -0,0 +1,71 @@
services:
rendezvous:
environment:
Rendezvous__Metrics__Enabled: "true"
volumes:
- ../observability/secrets/rendezvous-metrics-token:/run/secrets/rendezvous-metrics-token:ro
prometheus:
image: prom/prometheus:v3.13.1@sha256:3c42b892cf723fa54d2f262c37a0e1f80aa8c8ddb1da7b9b0df9455a35a7f893
user: "65534:65534"
read_only: true
init: true
cap_drop:
- ALL
security_opt:
- no-new-privileges:true
pids_limit: 128
mem_limit: 512m
cpus: 1.0
restart: unless-stopped
command:
- --config.file=/etc/prometheus/prometheus.yml
- --storage.tsdb.path=/prometheus
- --storage.tsdb.retention.time=15d
volumes:
- ../observability/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
- ../observability/secrets/rendezvous-metrics-token:/run/secrets/rendezvous-metrics-token:ro
- prometheus-data:/prometheus
depends_on:
- rendezvous
grafana:
image: grafana/grafana:13.1.0@sha256:121a7a9ece6dc10b969f1f96eed64b4f07dfac0d0b8abc070f7cb83bbde86f63
user: "472:472"
read_only: true
init: true
cap_drop:
- ALL
security_opt:
- no-new-privileges:true
pids_limit: 128
mem_limit: 512m
cpus: 1.0
restart: unless-stopped
environment:
GF_ANALYTICS_REPORTING_ENABLED: "false"
GF_ANALYTICS_CHECK_FOR_UPDATES: "false"
GF_PLUGINS_PREINSTALL_DISABLED: "true"
GF_PLUGINS_PREINSTALL_AUTO_UPDATE: "false"
GF_SECURITY_ADMIN_USER: "rendezvous-admin"
GF_SECURITY_ADMIN_PASSWORD__FILE: /run/secrets/grafana-admin-password
GF_USERS_ALLOW_SIGN_UP: "false"
GF_AUTH_ANONYMOUS_ENABLED: "false"
GF_SERVER_DOMAIN: localhost
GF_SERVER_ROOT_URL: http://localhost:3000
tmpfs:
- /tmp:rw,noexec,nosuid,nodev,size=32m,uid=472,gid=472,mode=0700
volumes:
- ../observability/grafana/provisioning/datasources:/etc/grafana/provisioning/datasources:ro
- ../observability/grafana/provisioning/dashboards:/etc/grafana/provisioning/dashboards:ro
- ../observability/grafana/dashboards:/var/lib/grafana/dashboards:ro
- ../observability/secrets/grafana-admin-password:/run/secrets/grafana-admin-password:ro
- grafana-data:/var/lib/grafana
ports:
- "127.0.0.1:3000:3000/tcp"
depends_on:
- prometheus
volumes:
prometheus-data:
grafana-data:
@@ -0,0 +1,292 @@
{
"annotations": {"list": []},
"description": "Privacy-safe operational view of the single-active Rendezvous service. Capacity percentages use the approved 25,000 listing and 10,000 active-attempt launch envelope.",
"editable": false,
"fiscalYearStartMonth": 0,
"graphTooltip": 1,
"id": null,
"links": [],
"liveNow": false,
"panels": [
{
"id": 1,
"title": "Service",
"description": "Prometheus can authenticate to and scrape the Rendezvous process.",
"type": "stat",
"datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"},
"gridPos": {"h": 4, "w": 4, "x": 0, "y": 0},
"fieldConfig": {"defaults": {"mappings": [{"options": {"0": {"color": "red", "text": "DOWN"}, "1": {"color": "green", "text": "UP"}}, "type": "value"}], "thresholds": {"mode": "absolute", "steps": [{"color": "red", "value": null}, {"color": "green", "value": 1}]}}, "overrides": []},
"options": {"colorMode": "background", "graphMode": "none", "justifyMode": "center", "orientation": "auto", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "textMode": "auto", "wideLayout": true},
"targets": [{"editorMode": "code", "expr": "up{job=\"rendezvous\"}", "legendFormat": "Rendezvous", "range": true, "refId": "A"}]
},
{
"id": 2,
"title": "Store",
"type": "stat",
"datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"},
"gridPos": {"h": 4, "w": 4, "x": 4, "y": 0},
"fieldConfig": {"defaults": {"mappings": [{"options": {"0": {"color": "red", "text": "UNAVAILABLE"}, "1": {"color": "green", "text": "AVAILABLE"}}, "type": "value"}], "thresholds": {"mode": "absolute", "steps": [{"color": "red", "value": null}, {"color": "green", "value": 1}]}}, "overrides": []},
"options": {"colorMode": "background", "graphMode": "none", "justifyMode": "center", "orientation": "auto", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "textMode": "auto", "wideLayout": true},
"targets": [{"editorMode": "code", "expr": "rendezvous_store_available", "legendFormat": "Store", "range": true, "refId": "A"}]
},
{
"id": 3,
"title": "Drain",
"type": "stat",
"datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"},
"gridPos": {"h": 4, "w": 4, "x": 8, "y": 0},
"fieldConfig": {"defaults": {"mappings": [{"options": {"0": {"color": "green", "text": "ACCEPTING"}, "1": {"color": "orange", "text": "DRAINING"}}, "type": "value"}], "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}, {"color": "orange", "value": 1}]}}, "overrides": []},
"options": {"colorMode": "background", "graphMode": "none", "justifyMode": "center", "orientation": "auto", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "textMode": "auto", "wideLayout": true},
"targets": [{"editorMode": "code", "expr": "rendezvous_store_draining", "legendFormat": "Drain", "range": true, "refId": "A"}]
},
{
"id": 4,
"title": "Listings",
"description": "Percentage of the approved 25,000-listing single-process envelope.",
"type": "gauge",
"datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"},
"gridPos": {"h": 4, "w": 6, "x": 12, "y": 0},
"fieldConfig": {"defaults": {"max": 100, "min": 0, "unit": "percent", "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}, {"color": "orange", "value": 70}, {"color": "red", "value": 90}]}}, "overrides": []},
"options": {"minVizHeight": 75, "minVizWidth": 75, "orientation": "auto", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "showThresholdLabels": false, "showThresholdMarkers": true, "sizing": "auto"},
"targets": [{"editorMode": "code", "expr": "100 * rendezvous_store_active_listings / 25000", "legendFormat": "Listings", "range": true, "refId": "A"}]
},
{
"id": 5,
"title": "Join attempts",
"description": "Percentage of the approved 10,000 active-attempt single-process envelope.",
"type": "gauge",
"datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"},
"gridPos": {"h": 4, "w": 6, "x": 18, "y": 0},
"fieldConfig": {"defaults": {"max": 100, "min": 0, "unit": "percent", "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}, {"color": "orange", "value": 70}, {"color": "red", "value": 90}]}}, "overrides": []},
"options": {"minVizHeight": 75, "minVizWidth": 75, "orientation": "auto", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "showThresholdLabels": false, "showThresholdMarkers": true, "sizing": "auto"},
"targets": [{"editorMode": "code", "expr": "100 * rendezvous_store_active_attempts / 10000", "legendFormat": "Attempts", "range": true, "refId": "A"}]
},
{
"id": 6,
"title": "HTTP request rate by operation",
"description": "Registration, renewal, browse, and join issuance appear as bounded operation names.",
"type": "timeseries",
"datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"},
"gridPos": {"h": 8, "w": 8, "x": 0, "y": 4},
"fieldConfig": {"defaults": {"unit": "reqps"}, "overrides": []},
"options": {"legend": {"calcs": ["lastNotNull"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
"targets": [{"editorMode": "code", "expr": "sum by (operation) (rate(rendezvous_http_requests_total[5m]))", "legendFormat": "{{operation}}", "range": true, "refId": "A"}]
},
{
"id": 7,
"title": "HTTP p95 latency",
"type": "timeseries",
"datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"},
"gridPos": {"h": 8, "w": 8, "x": 8, "y": 4},
"fieldConfig": {"defaults": {"unit": "ms", "custom": {"thresholdsStyle": {"mode": "line"}}, "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}, {"color": "red", "value": 200}]}}, "overrides": []},
"options": {"legend": {"calcs": ["lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
"targets": [{"editorMode": "code", "expr": "histogram_quantile(0.95, sum by (le, operation) (rate(rendezvous_http_duration_milliseconds_bucket[5m])))", "legendFormat": "{{operation}}", "range": true, "refId": "A"}]
},
{
"id": 8,
"title": "HTTP error and shedding rate",
"type": "timeseries",
"datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"},
"gridPos": {"h": 8, "w": 8, "x": 16, "y": 4},
"fieldConfig": {"defaults": {"unit": "reqps"}, "overrides": []},
"options": {"legend": {"calcs": ["lastNotNull"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
"targets": [{"editorMode": "code", "expr": "sum by (status_code) (rate(rendezvous_http_requests_total{status_code=~\"4..|5..\"}[5m]))", "legendFormat": "HTTP {{status_code}}", "range": true, "refId": "A"}]
},
{
"id": 9,
"title": "Browser live-update load",
"description": "Active public SSE clients, tenant scopes, and bounded replay entries.",
"type": "timeseries",
"datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"},
"gridPos": {"h": 7, "w": 8, "x": 0, "y": 12},
"fieldConfig": {"defaults": {"unit": "short"}, "overrides": []},
"options": {"legend": {"calcs": ["lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
"targets": [
{"editorMode": "code", "expr": "rendezvous_browser_sse_subscribers", "legendFormat": "Subscribers", "range": true, "refId": "A"},
{"editorMode": "code", "expr": "rendezvous_browser_sse_tenants", "legendFormat": "Tenant scopes", "range": true, "refId": "B"},
{"editorMode": "code", "expr": "rendezvous_browser_replay_entries", "legendFormat": "Replay entries", "range": true, "refId": "C"}
]
},
{
"id": 10,
"title": "UDP mediation results",
"type": "timeseries",
"datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"},
"gridPos": {"h": 7, "w": 8, "x": 8, "y": 12},
"fieldConfig": {"defaults": {"unit": "pps"}, "overrides": []},
"options": {"legend": {"calcs": ["lastNotNull"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
"targets": [{"editorMode": "code", "expr": "sum by (operation, result) (rate(rendezvous_udp_results_total[5m]))", "legendFormat": "{{operation}} · {{result}}", "range": true, "refId": "A"}]
},
{
"id": 11,
"title": "UDP p95 processing latency",
"type": "timeseries",
"datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"},
"gridPos": {"h": 7, "w": 8, "x": 16, "y": 12},
"fieldConfig": {"defaults": {"unit": "ms", "custom": {"thresholdsStyle": {"mode": "line"}}, "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}, {"color": "red", "value": 100}]}}, "overrides": []},
"options": {"legend": {"calcs": ["lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
"targets": [{"editorMode": "code", "expr": "histogram_quantile(0.95, sum by (le, operation) (rate(rendezvous_udp_duration_milliseconds_bucket[5m])))", "legendFormat": "{{operation}}", "range": true, "refId": "A"}]
},
{
"id": 12,
"title": "UDP ingress and admitted response budget",
"description": "Traffic bytes observed by the process and the conservative maximum response budget admitted for successful introductions; this is not actual egress.",
"type": "timeseries",
"datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"},
"gridPos": {"h": 7, "w": 8, "x": 0, "y": 19},
"fieldConfig": {"defaults": {"unit": "Bps"}, "overrides": []},
"options": {"legend": {"calcs": ["lastNotNull"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
"targets": [
{"editorMode": "code", "expr": "sum by (operation) (rate(rendezvous_udp_received_bytes_total[5m]))", "legendFormat": "Ingress · {{operation}}", "range": true, "refId": "A"},
{"editorMode": "code", "expr": "sum by (operation) (rate(rendezvous_udp_response_budget_bytes_total[5m]))", "legendFormat": "Response budget · {{operation}}", "range": true, "refId": "B"}
]
},
{
"id": 13,
"title": "Admission-control drops",
"type": "timeseries",
"datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"},
"gridPos": {"h": 7, "w": 8, "x": 8, "y": 19},
"fieldConfig": {"defaults": {"unit": "ops"}, "overrides": []},
"options": {"legend": {"calcs": ["lastNotNull"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
"targets": [{"editorMode": "code", "expr": "sum by (transport, partition) (rate(rendezvous_limiter_drops_total[5m]))", "legendFormat": "{{transport}} · {{partition}}", "range": true, "refId": "A"}]
},
{
"id": 14,
"title": "Direct-connect outcomes",
"type": "timeseries",
"datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"},
"gridPos": {"h": 7, "w": 8, "x": 16, "y": 19},
"fieldConfig": {"defaults": {"unit": "ops"}, "overrides": []},
"options": {"legend": {"calcs": ["lastNotNull"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
"targets": [{"editorMode": "code", "expr": "sum by (outcome, elapsed_bucket) (rate(rendezvous_connection_outcomes_total[5m]))", "legendFormat": "{{outcome}} · {{elapsed_bucket}}", "range": true, "refId": "A"}]
},
{
"id": 15,
"title": "Pairing p95 latency",
"type": "timeseries",
"datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"},
"gridPos": {"h": 7, "w": 8, "x": 0, "y": 26},
"fieldConfig": {"defaults": {"unit": "ms", "custom": {"thresholdsStyle": {"mode": "line"}}, "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}, {"color": "red", "value": 100}]}}, "overrides": []},
"options": {"legend": {"calcs": ["lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
"targets": [{"editorMode": "code", "expr": "histogram_quantile(0.95, rate(rendezvous_pairing_latency_milliseconds_bucket[5m]))", "legendFormat": "Pairing p95", "range": true, "refId": "A"}]
},
{
"id": 16,
"title": "Presence and expiry state",
"type": "timeseries",
"datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"},
"gridPos": {"h": 7, "w": 8, "x": 8, "y": 26},
"fieldConfig": {"defaults": {"unit": "short"}, "overrides": []},
"options": {"legend": {"calcs": ["lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
"targets": [
{"editorMode": "code", "expr": "rendezvous_store_fresh_presence_bindings", "legendFormat": "Fresh presence", "range": true, "refId": "A"},
{"editorMode": "code", "expr": "rendezvous_store_awaiting_presence_listings", "legendFormat": "Awaiting presence", "range": true, "refId": "B"},
{"editorMode": "code", "expr": "rate(rendezvous_store_expiry_churn_total[5m])", "legendFormat": "Expiry churn/s", "range": true, "refId": "C"}
]
},
{
"id": 17,
"title": "Signing key state",
"type": "timeseries",
"datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"},
"gridPos": {"h": 7, "w": 8, "x": 16, "y": 26},
"fieldConfig": {"defaults": {"unit": "short"}, "overrides": []},
"options": {"legend": {"calcs": ["lastNotNull"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
"targets": [{"editorMode": "code", "expr": "rendezvous_signing_keys", "legendFormat": "{{state}}", "range": true, "refId": "A"}]
},
{
"id": 18,
"title": "Minimum signing window",
"description": "Page below seven days; escalate below 24 hours.",
"type": "stat",
"datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"},
"gridPos": {"h": 5, "w": 6, "x": 0, "y": 33},
"fieldConfig": {"defaults": {"unit": "s", "thresholds": {"mode": "absolute", "steps": [{"color": "red", "value": null}, {"color": "orange", "value": 86400}, {"color": "green", "value": 604800}]}}, "overrides": []},
"options": {"colorMode": "background", "graphMode": "area", "justifyMode": "center", "orientation": "auto", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "textMode": "auto", "wideLayout": true},
"targets": [{"editorMode": "code", "expr": "rendezvous_signing_key_sign_seconds_remaining", "legendFormat": "Signing window", "range": true, "refId": "A"}]
},
{
"id": 19,
"title": "Operator authentication",
"type": "timeseries",
"datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"},
"gridPos": {"h": 5, "w": 6, "x": 6, "y": 33},
"fieldConfig": {"defaults": {"unit": "ops"}, "overrides": []},
"options": {"legend": {"calcs": ["lastNotNull"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
"targets": [{"editorMode": "code", "expr": "sum by (result) (rate(rendezvous_operator_authentication_total[5m]))", "legendFormat": "{{result}}", "range": true, "refId": "A"}]
},
{
"id": 20,
"title": "Privileged audit outcomes",
"type": "timeseries",
"datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"},
"gridPos": {"h": 5, "w": 6, "x": 12, "y": 33},
"fieldConfig": {"defaults": {"unit": "ops"}, "overrides": []},
"options": {"legend": {"calcs": ["lastNotNull"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
"targets": [{"editorMode": "code", "expr": "sum by (action, result) (rate(rendezvous_audit_events_total[5m]))", "legendFormat": "{{action}} · {{result}}", "range": true, "refId": "A"}]
},
{
"id": 21,
"title": "Metrics access",
"type": "timeseries",
"datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"},
"gridPos": {"h": 5, "w": 6, "x": 18, "y": 33},
"fieldConfig": {"defaults": {"unit": "ops"}, "overrides": []},
"options": {"legend": {"calcs": ["lastNotNull"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
"targets": [{"editorMode": "code", "expr": "sum by (result) (rate(rendezvous_metrics_scrapes_total[5m]))", "legendFormat": "{{result}}", "range": true, "refId": "A"}]
},
{
"id": 22,
"title": "Process memory",
"type": "timeseries",
"datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"},
"gridPos": {"h": 7, "w": 8, "x": 0, "y": 38},
"fieldConfig": {"defaults": {"unit": "bytes", "custom": {"thresholdsStyle": {"mode": "line"}}, "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}, {"color": "orange", "value": 1073741824}, {"color": "red", "value": 1610612736}]}}, "overrides": []},
"options": {"legend": {"calcs": ["lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
"targets": [
{"editorMode": "code", "expr": "process_resident_memory_bytes", "legendFormat": "Resident", "range": true, "refId": "A"},
{"editorMode": "code", "expr": "dotnet_gc_heap_size_bytes", "legendFormat": "Managed heap", "range": true, "refId": "B"}
]
},
{
"id": 23,
"title": "Process CPU and threads",
"type": "timeseries",
"datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"},
"gridPos": {"h": 7, "w": 8, "x": 8, "y": 38},
"fieldConfig": {"defaults": {"unit": "short"}, "overrides": [{"matcher": {"id": "byName", "options": "CPU cores"}, "properties": [{"id": "unit", "value": "cores"}]}]},
"options": {"legend": {"calcs": ["lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
"targets": [
{"editorMode": "code", "expr": "rate(process_cpu_seconds_total[5m])", "legendFormat": "CPU cores", "range": true, "refId": "A"},
{"editorMode": "code", "expr": "process_threads", "legendFormat": "Process threads", "range": true, "refId": "B"},
{"editorMode": "code", "expr": "dotnet_thread_pool_threads", "legendFormat": "Thread-pool threads", "range": true, "refId": "C"}
]
},
{
"id": 24,
"title": "Descriptor and GC pressure",
"type": "timeseries",
"datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"},
"gridPos": {"h": 7, "w": 8, "x": 16, "y": 38},
"fieldConfig": {"defaults": {"unit": "short"}, "overrides": []},
"options": {"legend": {"calcs": ["lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
"targets": [
{"editorMode": "code", "expr": "process_open_file_descriptors", "legendFormat": "Open descriptors", "range": true, "refId": "A"},
{"editorMode": "code", "expr": "sum(rate(dotnet_gc_collections_total[5m]))", "legendFormat": "GC collections/s", "range": true, "refId": "B"},
{"editorMode": "code", "expr": "dotnet_thread_pool_available_worker_threads", "legendFormat": "Available workers", "range": true, "refId": "C"}
]
}
],
"refresh": "15s",
"schemaVersion": 41,
"tags": ["rendezvous", "operations", "privacy-safe"],
"templating": {"list": []},
"time": {"from": "now-1h", "to": "now"},
"timepicker": {},
"timezone": "browser",
"title": "Rendezvous operational overview",
"uid": "rendezvous-overview",
"version": 1,
"weekStart": ""
}
@@ -0,0 +1,13 @@
apiVersion: 1
providers:
- name: Rendezvous
orgId: 1
folder: Rendezvous
type: file
disableDeletion: true
allowUiUpdates: false
updateIntervalSeconds: 30
options:
path: /var/lib/grafana/dashboards
foldersFromFilesStructure: false
@@ -0,0 +1,17 @@
apiVersion: 1
deleteDatasources:
- name: Rendezvous Prometheus
orgId: 1
datasources:
- name: Rendezvous Prometheus
uid: rendezvous-prometheus
type: prometheus
access: proxy
url: http://prometheus:9090
isDefault: true
editable: false
jsonData:
httpMethod: POST
timeInterval: 15s
@@ -0,0 +1,14 @@
global:
scrape_interval: 15s
evaluation_interval: 15s
external_labels:
service: rendezvous
scrape_configs:
- job_name: rendezvous
scheme: http
metrics_path: /metrics
bearer_token_file: /run/secrets/rendezvous-metrics-token
static_configs:
- targets:
- rendezvous:8080
+2
View File
@@ -0,0 +1,2 @@
*
!.gitignore
+2
View File
@@ -29,6 +29,8 @@ engine, transport, or server dependency therefore fails the normal test gate.
## Supported toolchain ## Supported toolchain
- Build SDK: .NET SDK 10.0.301, pinned by `global.json`. - Build SDK: .NET SDK 10.0.301, pinned by `global.json`.
- Verification runtime: .NET SDK 8.0.128 supplies the pinned .NET 8.0.28
runtime used by TestClient process tests and release verification.
- Server runtime: .NET 10 LTS. - Server runtime: .NET 10 LTS.
- Client/contracts compatibility target: .NET Standard 2.1, consumable by the - Client/contracts compatibility target: .NET Standard 2.1, consumable by the
.NET 8-or-later runtime used by current Godot 4 C# projects. .NET 8-or-later runtime used by current Godot 4 C# projects.
+16
View File
@@ -42,6 +42,22 @@ test "$RENDEZVOUS_UID" -ne 0
docker compose -f deploy/compose/compose.yaml up --build --detach docker compose -f deploy/compose/compose.yaml up --build --detach
``` ```
The example defaults to host TCP 8080 and UDP 9050. On a shared host, set
`RENDEZVOUS_HTTP_HOST_PORT` and `RENDEZVOUS_UDP_HOST_PORT` before starting
Compose, then point `RENDEZVOUS_SMOKE_HTTP_URL` and
`RENDEZVOUS_SMOKE_UDP_ENDPOINT` at those published ports. The container ports
and production-advertised service ports remain 8080/9050. Sibling-container CI
runners may also set `RENDEZVOUS_CONFIG_SOURCE` and
`RENDEZVOUS_SECRET_SOURCE` to host-visible absolute bind-source paths; local
operators should normally keep the checked-in relative defaults. CI runs its
ephemeral smoke client as a sidecar in the service container's network namespace
to avoid runner-specific bridge and host-routing policy; it does not widen the
default loopback HTTP publication. It mints the disposable publisher credential
while the generated key is still owner-private, then makes the key read-only for
the non-root service container. Automation probes readiness once per second and
allows up to three minutes for a cold, resource-constrained image to become
ready; the protocol smoke retains its separate, stricter scenario timeout.
`deploy/compose/appsettings.Production.json` is a local/private-bridge smoke `deploy/compose/appsettings.Production.json` is a local/private-bridge smoke
profile, not an Internet template: TCP is published only on host loopback, the profile, not an Internet template: TCP is published only on host loopback, the
explicit `rendezvous` host name serves isolated clients on the Compose network, explicit `rendezvous` host name serves isolated clients on the Compose network,
+123
View File
@@ -0,0 +1,123 @@
# Diagnostic dashboards
Tracking: #27
Rendezvous provides two deliberately separate, optional views. The public
session diagnostic helps a player or integration operator understand safe
session-list state using only the public browse contract. The private Grafana
dashboard exposes aggregate operational health through authenticated metrics.
Neither view grants operator privileges or exposes player identity, endpoints,
credentials, capabilities, or raw session metadata beyond the explicitly
allowlisted public browse fields.
## Public read-only session diagnostic
The static diagnostic is disabled by default. Enable it only for approved
game/environment scopes and keep the allowlist narrow:
```json
"Diagnostics": {
"Enabled": true,
"PollIntervalSeconds": 10,
"MaximumRenderedSessions": 100,
"Scopes": [
{
"GameId": "space-game",
"EnvironmentId": "smoke",
"ProtocolVersions": [1, 2],
"Regions": ["local"]
}
]
}
```
Open `/diagnostics` on the same origin as Rendezvous. The page cannot choose a
different backend, request private visibility, join a session, or call the
operator surface. It renders a bounded snapshot, then applies ordered SSE
updates. A replay reset, corrupt cursor, incomplete snapshot, transport failure,
or deliberate reconnect returns to a fresh authoritative snapshot and bounded
polling. Apply filter changes explicitly; **Reset filters** restores the
configured defaults, while **Reconnect now** tests recovery without changing
the selection.
The page uses semantic HTML, labelled controls, visible keyboard focus, status
text in addition to color, a reduced-motion mode, and a 320-pixel reflow. It
creates untrusted content with `textContent` only. The endpoint sets a restrictive
same-origin content-security policy, denies framing, disables MIME sniffing and
browser capabilities, and marks every asset/config response `no-store`.
This is a diagnostics convenience, not a game browser, management console, or
availability monitor. Disable it independently by setting `Enabled` to `false`;
all diagnostic paths then return `404` without affecting game traffic, metrics,
or health endpoints.
## Private Prometheus and Grafana view
The observability overlay pins Prometheus 3.13.1 and Grafana 13.1.0 by immutable
multi-platform image digest. Prometheus is not published to the host. Grafana is
bound to host loopback, disables anonymous access and sign-up, and reads its
administrator password from a file. The service and Prometheus share only the
metrics bearer-token file. All three secrets remain ignored by Git.
Create independent random secrets, then start the base service and overlay:
```bash
install -d -m 0700 deploy/compose/secrets deploy/observability/secrets
umask 077
openssl rand -out deploy/compose/secrets/signing-key 32
openssl rand -hex 32 >deploy/observability/secrets/rendezvous-metrics-token
openssl rand -base64 36 >deploy/observability/secrets/grafana-admin-password
export RENDEZVOUS_UID="$(id -u)"
export RENDEZVOUS_GID="$(id -g)"
test "$RENDEZVOUS_UID" -ne 0
docker compose \
-f deploy/compose/compose.yaml \
-f deploy/observability/compose.yaml \
up --build --detach
```
Visit `http://127.0.0.1:3000`, sign in as `rendezvous-admin`, and open the
**Rendezvous operational overview** folder/dashboard. The provisioned panels
cover scrape/store/drain health, listing and join capacity, HTTP volume/errors
and p95, browse/SSE load, lease and join operations, UDP results/latency/bytes,
admission drops, connection outcomes, pairing latency, presence/expiry state,
signing windows, security/audit results, and process/GC/descriptor pressure.
Capacity gauges use the approved single-process envelope of 25,000 listings and
10,000 active attempts, with 70% warning and 90% critical thresholds. The UDP
response series is a conservative admitted maximum, not observed egress.
Validate merged configuration and checked-in dashboard structure before every
rollout:
```bash
RENDEZVOUS_UID="$(id -u)" RENDEZVOUS_GID="$(id -g)" \
docker compose \
-f deploy/compose/compose.yaml \
-f deploy/observability/compose.yaml \
config --quiet
./scripts/test-observability-assets.sh
```
For a real deployment, keep Grafana on a private authenticated management
network instead of host loopback, replace the local admin login with the
organization's supported identity boundary, enforce TLS at the edge, and set
retention to the approved operational period. Do not make Prometheus public.
Provisioning is read-only so local UI edits cannot silently drift from source.
## Verify, rotate, and disable
After startup, verify the dashboard shows `UP`, store `AVAILABLE`, a nonzero
signing window, and changing request/UDP panels during a smoke run. Confirm an
unauthenticated `/metrics` request returns `404`, the bearer-authenticated
collector target is healthy, Prometheus is not bound on a host port, Grafana is
not anonymously accessible, and dashboard query labels contain no identifiers.
Rotate metrics access by writing a new 32-128 character token to the secret file
with private permissions and restarting Rendezvous and Prometheus together.
Rotate the Grafana administrator password through the same protected secret
workflow. Delete both secret files after a disposable local run.
To disable aggregate observability independently, stop/remove the overlay and
set `Rendezvous:Metrics:Enabled` to `false`; `/metrics` returns `404` and the
core service continues. To disable only the public session diagnostic, leave the
overlay running and set `Rendezvous:Diagnostics:Enabled` to `false`.
@@ -2,9 +2,10 @@
This runbook defines the production signals and privileged controls for the This runbook defines the production signals and privileged controls for the
Rendezvous service. The service emits `System.Diagnostics.Metrics` instruments Rendezvous service. The service emits `System.Diagnostics.Metrics` instruments
from the `FinalFactory.Rendezvous` meter and distributed-tracing activities from from the `FinalFactory.Rendezvous` meter, distributed-tracing activities from
`FinalFactory.Rendezvous.Server`. Connect those sources to the deployment's `FinalFactory.Rendezvous.Server`, and an optional bearer-protected Prometheus
OpenTelemetry or equivalent collector. Do not add identifiers to metric labels. endpoint. Connect only a private collector network. Do not add identifiers to
metric labels.
Concrete detect/contain/recover/verify procedures for abuse, key compromise, Concrete detect/contain/recover/verify procedures for abuse, key compromise,
targeted revocation, restart, rollback, saturation, privacy incidents, and targeted revocation, restart, rollback, saturation, privacy incidents, and
@@ -38,6 +39,25 @@ dependency upgrades are in the [incident and change runbooks](incident-runbooks.
| `rendezvous.store.expiry_churn` | Cumulative natural expiry activity | none | | `rendezvous.store.expiry_churn` | Cumulative natural expiry activity | none |
| `rendezvous.store.available` | Store health (`1` available, `0` unavailable) | none | | `rendezvous.store.available` | Store health (`1` available, `0` unavailable) | none |
The Prometheus exporter additionally exposes active/fresh/awaiting store state,
drain state, SSE subscriber/tenant/replay gauges, bounded HTTP and UDP
histograms, UDP ingress and conservative admitted-response budgets, signing-key
state/window gauges, and process/.NET pressure. Its only labels are the fixed
operation, status, result, transport, partition, action, outcome, elapsed-bucket,
key-state, and GC-generation dimensions. Unknown or unsafe values normalize to
`other`; identifiers, metadata, addresses, endpoints, tokens, and capabilities
are never labels.
The exporter is disabled by default. Enabling `Rendezvous:Metrics:Enabled`
requires `BearerTokenSecretReference` to be an external `env:` or absolute
`file:` secret containing 32-128 visible ASCII bytes. Unauthorized requests get
the same `404` as a disabled endpoint, and accepted responses are `no-store`.
Expose `/metrics` only to the private collector network, rotate its token as a
deployment secret, and never place the token in a URL, Compose environment
value, dashboard, log, or issue. The checked-in Prometheus/Grafana provisioning
and its verification procedure are in
[diagnostic dashboards](diagnostic-dashboards.md).
HTTP responses include `X-Rendezvous-Correlation-ID`. It is a generated trace ID HTTP responses include `X-Rendezvous-Correlation-ID`. It is a generated trace ID
or random value, never a caller-supplied session or player identifier. UDP and or random value, never a caller-supplied session or player identifier. UDP and
HTTP activities contain operation-level data only. Logs and traces must not add HTTP activities contain operation-level data only. Logs and traces must not add
+3
View File
@@ -1,11 +1,14 @@
# syntax=docker/dockerfile:1.7@sha256:a57df69d0ea827fb7266491f2813635de6f17269be881f696fbfdf2d83dda33e # syntax=docker/dockerfile:1.7@sha256:a57df69d0ea827fb7266491f2813635de6f17269be881f696fbfdf2d83dda33e
FROM python:3.12.11-slim-bookworm@sha256:c00fc7b44d844b6da22861ec24af43968a5200eac4ec607b4725d585165d6b49 AS release-python FROM python:3.12.11-slim-bookworm@sha256:c00fc7b44d844b6da22861ec24af43968a5200eac4ec607b4725d585165d6b49 AS release-python
FROM ghcr.io/jqlang/jq:1.8.1@sha256:95de8f005ca027686a1ca3b0853e2bb219062438015862816159f3f25a4d4230 AS release-jq FROM ghcr.io/jqlang/jq:1.8.1@sha256:95de8f005ca027686a1ca3b0853e2bb219062438015862816159f3f25a4d4230 AS release-jq
FROM mcr.microsoft.com/dotnet/runtime:8.0.28-noble@sha256:19a311642eb7ee9c985bd71b9e5618123879df4e8d948f7388a41b0ee4788e25 AS dotnet-runtime-8
FROM mcr.microsoft.com/dotnet/sdk:10.0.301-noble@sha256:ea8bde36c11b6e7eec2656d0e59101d4462f6bd630730f2c8201ed0572b295d5 AS release-builder FROM mcr.microsoft.com/dotnet/sdk:10.0.301-noble@sha256:ea8bde36c11b6e7eec2656d0e59101d4462f6bd630730f2c8201ed0572b295d5 AS release-builder
COPY --from=release-python /usr/local/ /usr/local/ COPY --from=release-python /usr/local/ /usr/local/
COPY --from=release-jq /jq /usr/local/bin/jq COPY --from=release-jq /jq /usr/local/bin/jq
COPY --from=dotnet-runtime-8 /usr/share/dotnet/shared/Microsoft.NETCore.App/8.0.28/ /usr/share/dotnet/shared/Microsoft.NETCore.App/8.0.28/
RUN dotnet --version \ RUN dotnet --version \
&& dotnet --list-runtimes \
&& python3 --version \ && python3 --version \
&& git --version \ && git --version \
&& tar --version \ && tar --version \
+31 -15
View File
@@ -58,6 +58,14 @@ temp_dir="$(mktemp -d)"
host_log="$temp_dir/host.jsonl" host_log="$temp_dir/host.jsonl"
join_log="$temp_dir/join.jsonl" join_log="$temp_dir/join.jsonl"
host_pid='' host_pid=''
sanitize_log() {
jq -Rrc 'fromjson? | {
event: (.event // "unknown"),
status: (.status // "unknown"),
phase: (.phase // "unknown"),
hasListingId: (((.listingId // "") | length) > 0)
}' "$1"
}
cleanup() { cleanup() {
local status="$?" local status="$?"
if [[ -n "$host_pid" ]] && kill -0 "$host_pid" 2>/dev/null; then if [[ -n "$host_pid" ]] && kill -0 "$host_pid" 2>/dev/null; then
@@ -66,8 +74,8 @@ cleanup() {
fi fi
if [[ "$status" -ne 0 ]]; then if [[ "$status" -ne 0 ]]; then
printf 'Deployment smoke failed; sanitized diagnostic events follow.\n' >&2 printf 'Deployment smoke failed; sanitized diagnostic events follow.\n' >&2
[[ -f "$host_log" ]] && jq -c . "$host_log" >&2 || true [[ -f "$host_log" ]] && sanitize_log "$host_log" >&2 || true
[[ -f "$join_log" ]] && jq -c . "$join_log" >&2 || true [[ -f "$join_log" ]] && sanitize_log "$join_log" >&2 || true
fi fi
rm -rf "$temp_dir" rm -rf "$temp_dir"
return "$status" return "$status"
@@ -84,26 +92,24 @@ dotnet run --project "$PROJECT" --configuration "$BUILD_CONFIGURATION" --no-buil
host_pid="$!" host_pid="$!"
ready=false ready=false
listing_id=''
for ((iteration = 0; iteration < TIMEOUT_SECONDS * 4; iteration++)); do for ((iteration = 0; iteration < TIMEOUT_SECONDS * 4; iteration++)); do
if jq -e 'select(.event == "host.ready")' "$host_log" >/dev/null 2>&1; then if listing_id="$(jq -er '
select(.event == "host.ready" and .status == "ready")
| .listingId // empty
' "$host_log" 2>/dev/null | tail -n 1)" && [[ -n "$listing_id" ]]; then
ready=true ready=true
break break
fi fi
if ! kill -0 "$host_pid" 2>/dev/null; then if ! kill -0 "$host_pid" 2>/dev/null; then
printf 'Host diagnostic stopped before it became ready.\n' >&2 printf 'Host diagnostic stopped before it became ready.\n' >&2
jq -c . "$host_log" >&2 || true sanitize_log "$host_log" >&2 || true
exit 1 exit 1
fi fi
sleep 0.25 sleep 0.25
done done
if [[ "$ready" != true ]]; then if [[ "$ready" != true ]]; then
printf 'Host diagnostic did not become ready within %s seconds.\n' "$TIMEOUT_SECONDS" >&2 printf 'Host diagnostic did not report a ready listing within %s seconds.\n' "$TIMEOUT_SECONDS" >&2
exit 1
fi
listing_id="$(jq -r 'select(.event == "host.registered") | .listingId' "$host_log" | tail -n 1)"
if [[ -z "$listing_id" || "$listing_id" == null ]]; then
printf 'Host diagnostic did not report a listing ID.\n' >&2
exit 1 exit 1
fi fi
@@ -115,9 +121,19 @@ dotnet run --project "$PROJECT" --configuration "$BUILD_CONFIGURATION" --no-buil
wait "$host_pid" wait "$host_pid"
host_pid='' host_pid=''
jq -e 'select(.event == "host.direct-traffic" and .status == "verified")' "$host_log" >/dev/null if ! jq -s -e '
jq -e 'select(.event == "host.deregistered" and .status == "complete")' "$host_log" >/dev/null any(.[]; .event == "host.direct-traffic" and .status == "verified")
jq -e 'select(.event == "join.direct-traffic" and .status == "verified")' "$join_log" >/dev/null and any(.[]; .event == "host.deregistered" and .status == "complete")
jq -e 'select(.event == "join.outcome-report" and .status == "accepted")' "$join_log" >/dev/null ' "$host_log" >/dev/null; then
printf 'Host diagnostic did not complete authenticated traffic and deregistration.\n' >&2
exit 1
fi
if ! jq -s -e '
any(.[]; .event == "join.direct-traffic" and .status == "verified")
and any(.[]; .event == "join.outcome-report" and .status == "accepted")
' "$join_log" >/dev/null; then
printf 'Join diagnostic did not complete authenticated traffic and outcome reporting.\n' >&2
exit 1
fi
printf 'Rendezvous deployment smoke passed: HTTP live/ready and authenticated UDP mediation/direct traffic.\n' printf 'Rendezvous deployment smoke passed: HTTP live/ready and authenticated UDP mediation/direct traffic.\n'
+6
View File
@@ -0,0 +1,6 @@
#!/usr/bin/env bash
set -euo pipefail
root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
cd "$root"
node --test tests/Diagnostics/*.test.mjs
+4
View File
@@ -0,0 +1,4 @@
#!/usr/bin/env bash
set -euo pipefail
node --test tests/Observability/observability-assets.test.mjs
@@ -143,6 +143,39 @@ internal sealed class SessionChangeJournal
public SessionChangeJournalOptions Options => _options; public SessionChangeJournalOptions Options => _options;
public int ReplayCount
{
get
{
lock (_gate)
{
return _changes.Count;
}
}
}
public int SubscriberCount
{
get
{
lock (_gate)
{
return _subscribers;
}
}
}
public int SubscribedTenantCount
{
get
{
lock (_gate)
{
return _subscribersByTenant.Count;
}
}
}
public long CurrentRevision public long CurrentRevision
{ {
get get
@@ -0,0 +1,615 @@
const CONTRACT_VERSION = 1;
const BROWSER_PAGE_LIMIT = 100;
const REQUEST_TIMEOUT_MS = 10_000;
const STREAM_FAILURE_LIMIT = 3;
export class SessionProjection {
constructor(maximumSessions = 100) {
if (!Number.isInteger(maximumSessions) || maximumSessions < 1 || maximumSessions > 500) {
throw new RangeError("maximumSessions must be between 1 and 500");
}
this.maximumSessions = maximumSessions;
this.entries = new Map();
this.cursor = "";
this.hasMore = false;
}
replace(items, cursor, hasMore = false, updatedAt = Date.now()) {
if (!Array.isArray(items) || items.length > this.maximumSessions) {
return "overflow";
}
const next = new Map();
for (const session of items) {
if (!isPublicSession(session) || next.has(session.listingId)) {
return "invalid";
}
next.set(session.listingId, { session, updatedAt });
}
this.entries = next;
this.cursor = validCursor(cursor) ? cursor : "";
this.hasMore = Boolean(hasMore);
return "applied";
}
apply(event, updatedAt = Date.now()) {
if (!event || event.contractVersion !== CONTRACT_VERSION || !validCursor(event.cursor)) {
return "invalid";
}
if (event.kind === "sessionUpsert" && isPublicSession(event.session) && event.listingId == null) {
if (!this.entries.has(event.session.listingId)
&& this.entries.size >= this.maximumSessions) {
return "overflow";
}
this.entries.set(event.session.listingId, { session: event.session, updatedAt });
this.cursor = event.cursor;
return "applied";
}
if (event.kind === "sessionRemove"
&& typeof event.listingId === "string"
&& event.listingId.length > 0
&& event.session == null) {
this.entries.delete(event.listingId);
this.cursor = event.cursor;
return this.hasMore ? "refresh" : "applied";
}
if (event.kind === "keepalive" && event.session == null && event.listingId == null) {
this.cursor = event.cursor;
return "keepalive";
}
if (event.kind === "reset" && event.session == null && event.listingId == null) {
this.cursor = event.cursor;
return "reset";
}
return "invalid";
}
sessions() {
return [...this.entries.values()]
.sort((left, right) => left.session.listingId.localeCompare(right.session.listingId));
}
}
export function safeText(value, maximumLength = 160) {
const text = typeof value === "string" ? value : String(value ?? "");
const visible = text.replace(/[\u0000-\u001f\u007f]/gu, "");
return visible.length <= maximumLength
? visible
: `${visible.slice(0, Math.max(0, maximumLength - 1))}`;
}
export function buildBrowseUrl(filter) {
validateFilter(filter);
const query = new URLSearchParams({
contractVersion: String(CONTRACT_VERSION),
gameId: filter.gameId,
environmentId: filter.environmentId,
protocolVersion: String(filter.protocolVersion),
regionId: filter.regionId,
pageSize: String(Math.min(BROWSER_PAGE_LIMIT, filter.pageSize ?? BROWSER_PAGE_LIMIT)),
excludeFull: String(Boolean(filter.excludeFull)),
});
return `/v1/sessions?${query}`;
}
export function buildStreamUrl(filter, cursor) {
validateFilter(filter);
if (!validCursor(cursor)) {
throw new TypeError("A bounded stream cursor is required");
}
const query = new URLSearchParams({
contractVersion: String(CONTRACT_VERSION),
gameId: filter.gameId,
environmentId: filter.environmentId,
protocolVersion: String(filter.protocolVersion),
regionId: filter.regionId,
excludeFull: String(Boolean(filter.excludeFull)),
streamCursor: cursor,
});
return `/v1/sessions/stream?${query}`;
}
export function chooseSnapshotTransport(hasMore, pollingOnly) {
if (pollingOnly) {
return "pollingOnly";
}
return hasMore ? "boundedPolling" : "stream";
}
function validCursor(value) {
return typeof value === "string"
&& value.length > 0
&& value.length <= 1024
&& /^[\x21-\x7e]+$/u.test(value);
}
function isPublicSession(session) {
return session != null
&& session.contractVersion === CONTRACT_VERSION
&& typeof session.listingId === "string"
&& session.listingId.length > 0
&& typeof session.gameId === "string"
&& typeof session.environmentId === "string"
&& typeof session.regionId === "string"
&& Number.isInteger(session.protocolVersion)
&& session.protocolVersion > 0
&& typeof session.buildVersion === "string"
&& typeof session.displayName === "string"
&& session.visibility === "public"
&& session.capacity != null
&& Number.isInteger(session.capacity.currentPlayers)
&& Number.isInteger(session.capacity.maximumPlayers)
&& session.capacity.currentPlayers >= 0
&& session.capacity.maximumPlayers >= 1
&& session.capacity.currentPlayers <= session.capacity.maximumPlayers
&& session.metadata != null
&& typeof session.metadata === "object"
&& !Array.isArray(session.metadata);
}
function validateFilter(filter) {
if (!filter
|| typeof filter.gameId !== "string"
|| typeof filter.environmentId !== "string"
|| typeof filter.regionId !== "string"
|| !Number.isInteger(filter.protocolVersion)
|| filter.protocolVersion <= 0) {
throw new TypeError("The selected diagnostic filter is invalid");
}
}
function startDashboard() {
const elements = {
form: document.querySelector("#filters"),
game: document.querySelector("#game-filter"),
environment: document.querySelector("#environment-filter"),
protocol: document.querySelector("#protocol-filter"),
region: document.querySelector("#region-filter"),
capacity: document.querySelector("#capacity-filter"),
error: document.querySelector("#filter-error"),
reconnect: document.querySelector("#reconnect-button"),
reset: document.querySelector("#reset-button"),
poll: document.querySelector("#poll-button"),
streamStatus: document.querySelector("#stream-status"),
transportState: document.querySelector("#transport-state"),
projectionState: document.querySelector("#projection-state"),
lastUpdate: document.querySelector("#last-update"),
sessionCount: document.querySelector("#session-count"),
results: document.querySelector(".results-panel"),
resultsSummary: document.querySelector("#results-summary"),
list: document.querySelector("#session-list"),
empty: document.querySelector("#empty-state"),
};
const state = {
configuration: null,
projection: null,
activeFilter: null,
source: null,
pollingTimer: null,
requestController: null,
streamFailures: 0,
pollingOnly: false,
renderPending: false,
lastSuccess: 0,
};
function setStatus(kind, message, transport = message) {
document.body.dataset.streamState = kind;
elements.streamStatus.textContent = message;
elements.transportState.textContent = transport;
}
function setFormError(message = "") {
elements.error.textContent = message;
elements.error.hidden = message.length === 0;
}
function populate(select, values, previous) {
select.replaceChildren();
for (const value of values) {
const option = document.createElement("option");
option.value = String(value);
option.textContent = safeText(value, 80);
select.append(option);
}
if (values.some((value) => String(value) === previous)) {
select.value = previous;
}
}
function selectedScope() {
return state.configuration?.scopes.find((scope) =>
scope.gameId === elements.game.value
&& scope.environmentId === elements.environment.value) ?? null;
}
function updateEnvironmentOptions() {
const prior = elements.environment.value;
const environments = state.configuration.scopes
.filter((scope) => scope.gameId === elements.game.value)
.map((scope) => scope.environmentId);
populate(elements.environment, environments, prior);
updateScopeOptions();
}
function updateScopeOptions() {
const scope = selectedScope();
populate(elements.protocol, scope?.protocolVersions ?? [], elements.protocol.value);
populate(elements.region, scope?.regions ?? [], elements.region.value);
}
function currentFilter() {
const scope = selectedScope();
const protocolVersion = Number(elements.protocol.value);
if (!scope
|| !scope.protocolVersions.includes(protocolVersion)
|| !scope.regions.includes(elements.region.value)) {
throw new TypeError("Choose one of the configured game, environment, protocol, and region combinations.");
}
return {
gameId: scope.gameId,
environmentId: scope.environmentId,
protocolVersion,
regionId: elements.region.value,
excludeFull: elements.capacity.value === "open",
pageSize: Math.min(BROWSER_PAGE_LIMIT, state.configuration.maximumRenderedSessions),
};
}
function stopLiveWork() {
if (state.source) {
state.source.close();
state.source = null;
}
if (state.pollingTimer) {
clearTimeout(state.pollingTimer);
state.pollingTimer = null;
}
if (state.requestController) {
state.requestController.abort();
state.requestController = null;
}
}
function markSuccess(message) {
state.lastSuccess = Date.now();
elements.lastUpdate.dateTime = new Date(state.lastSuccess).toISOString();
elements.lastUpdate.textContent = "Just now";
elements.resultsSummary.textContent = message;
}
function scheduleRender() {
if (state.renderPending) {
return;
}
state.renderPending = true;
requestAnimationFrame(() => {
state.renderPending = false;
renderProjection();
});
}
function appendDetail(list, term, description) {
const dt = document.createElement("dt");
dt.textContent = term;
const dd = document.createElement("dd");
dd.textContent = safeText(description, 160);
list.append(dt, dd);
}
function sessionCard(entry) {
const session = entry.session;
const article = document.createElement("article");
article.className = "session-card";
article.setAttribute("role", "listitem");
article.dataset.updatedAt = String(entry.updatedAt);
const heading = document.createElement("div");
heading.className = "card-heading";
const title = document.createElement("h3");
title.textContent = safeText(session.displayName, 120);
const region = document.createElement("span");
region.className = "region-badge";
region.textContent = safeText(session.regionId, 48);
heading.append(title, region);
const details = document.createElement("dl");
details.className = "session-detail";
appendDetail(details, "Build", session.buildVersion);
appendDetail(details, "Protocol", session.protocolVersion);
appendDetail(details, "Visibility", "Public");
appendDetail(details, "Presence", "Recently verified");
const capacity = document.createElement("div");
capacity.className = "capacity-row";
const capacityCopy = document.createElement("div");
capacityCopy.className = "capacity-copy";
const capacityLabel = document.createElement("span");
capacityLabel.textContent = "Advisory capacity";
const capacityValue = document.createElement("span");
capacityValue.textContent = `${session.capacity.currentPlayers} / ${session.capacity.maximumPlayers}`;
capacityCopy.append(capacityLabel, capacityValue);
const meter = document.createElement("meter");
meter.min = 0;
meter.max = session.capacity.maximumPlayers;
meter.value = session.capacity.currentPlayers;
meter.setAttribute("aria-label", `Advisory capacity ${capacityValue.textContent}`);
capacity.append(capacityCopy, meter);
const metadata = document.createElement("ul");
metadata.className = "metadata-list";
for (const [key, value] of Object.entries(session.metadata).slice(0, 16)) {
const item = document.createElement("li");
item.textContent = `${safeText(key, 48)}: ${safeText(value, 96)}`;
metadata.append(item);
}
if (metadata.childElementCount === 0) {
const item = document.createElement("li");
item.textContent = "No public metadata";
metadata.append(item);
}
const updated = document.createElement("p");
updated.className = "updated-age";
updated.textContent = "Updated just now";
article.append(heading, details, capacity, metadata, updated);
return article;
}
function renderProjection() {
const entries = state.projection?.sessions() ?? [];
const fragment = document.createDocumentFragment();
for (const entry of entries) {
fragment.append(sessionCard(entry));
}
elements.list.replaceChildren(fragment);
elements.sessionCount.textContent = String(entries.length);
elements.empty.hidden = entries.length !== 0;
elements.projectionState.textContent = state.projection?.hasMore
? "Bounded snapshot; polling"
: "Snapshot plus ordered deltas";
elements.results.setAttribute("aria-busy", "false");
}
function updateAges() {
const now = Date.now();
if (state.lastSuccess > 0) {
const age = Math.max(0, Math.floor((now - state.lastSuccess) / 1000));
elements.lastUpdate.textContent = age < 5 ? "Just now" : `${age} seconds ago`;
}
for (const card of elements.list.querySelectorAll(".session-card")) {
const age = Math.max(0, Math.floor((now - Number(card.dataset.updatedAt)) / 1000));
const copy = card.querySelector(".updated-age");
copy.textContent = age < 5 ? "Updated just now" : `Updated ${age} seconds ago`;
}
}
async function readJson(url) {
const controller = new AbortController();
state.requestController = controller;
const timeout = setTimeout(() => controller.abort(), REQUEST_TIMEOUT_MS);
try {
const response = await fetch(url, {
cache: "no-store",
credentials: "same-origin",
headers: { Accept: "application/json" },
signal: controller.signal,
});
if (!response.ok) {
throw new Error(`The service returned HTTP ${response.status}.`);
}
return await response.json();
} finally {
clearTimeout(timeout);
if (state.requestController === controller) {
state.requestController = null;
}
}
}
function schedulePolling() {
if (state.pollingTimer) {
clearTimeout(state.pollingTimer);
}
state.pollingTimer = setTimeout(
() => fetchSnapshot("poll").catch(showServiceError),
state.configuration.pollIntervalSeconds * 1000,
);
}
function usePolling(message = "Polling fallback active") {
if (state.source) {
state.source.close();
state.source = null;
}
setStatus("polling", message, "Polling fallback");
schedulePolling();
}
function showServiceError(error) {
const message = error?.name === "AbortError"
? "The service did not answer within the request deadline."
: safeText(error?.message || "The service is unavailable.", 200);
setStatus("error", "Service unavailable", "Unavailable");
elements.resultsSummary.textContent = `${message} Retrying with bounded polling.`;
elements.results.setAttribute("aria-busy", "false");
usePolling("Service unavailable; polling retry scheduled");
}
async function fetchSnapshot(reason = "manual") {
if (!state.activeFilter) {
return;
}
if (state.requestController) {
state.requestController.abort();
}
elements.results.setAttribute("aria-busy", "true");
if (reason !== "poll") {
setStatus("reconnecting", "Loading a fresh snapshot", "Snapshot request");
}
const response = await readJson(buildBrowseUrl(state.activeFilter));
if (response.contractVersion !== CONTRACT_VERSION
|| !Array.isArray(response.items)
|| !validCursor(response.streamCursor)) {
throw new Error("The service returned an invalid browser snapshot.");
}
const outcome = state.projection.replace(
response.items,
response.streamCursor,
Boolean(response.nextCursor),
);
if (outcome !== "applied") {
throw new Error("The bounded browser snapshot could not be applied safely.");
}
scheduleRender();
markSuccess(`${response.items.length} public session${response.items.length === 1 ? "" : "s"} in the fresh snapshot.`);
const transport = chooseSnapshotTransport(Boolean(response.nextCursor), state.pollingOnly);
if (transport !== "stream") {
if (transport === "pollingOnly") {
usePolling("Polling only selected");
return;
}
usePolling("More sessions exist than this bounded view; polling keeps it authoritative");
return;
}
connectStream(response.streamCursor);
}
function handleStreamEvent(serialized, expectedKind) {
let event;
try {
event = JSON.parse(serialized);
} catch {
usePolling("Invalid stream data; polling fallback active");
return;
}
if (event.kind !== expectedKind) {
usePolling("Unexpected stream event; polling fallback active");
return;
}
const outcome = state.projection.apply(event);
if (outcome === "invalid" || outcome === "overflow" || outcome === "refresh") {
fetchSnapshot("stream-recovery").catch(showServiceError);
return;
}
if (outcome === "reset") {
setStatus("reset", "Cursor reset; refreshing snapshot", "Cursor reset");
fetchSnapshot("reset").catch(showServiceError);
return;
}
if (outcome === "keepalive") {
markSuccess("Live connection healthy; no listing changes.");
return;
}
state.streamFailures = 0;
scheduleRender();
markSuccess(expectedKind === "sessionUpsert"
? "Applied a live session add or update."
: "Applied a live session removal.");
}
function connectStream(cursor, corrupt = false) {
if (state.pollingTimer) {
clearTimeout(state.pollingTimer);
state.pollingTimer = null;
}
if (state.source) {
state.source.close();
}
const selectedCursor = corrupt
? `${cursor.slice(0, -1)}${cursor.endsWith("a") ? "b" : "a"}`
: cursor;
setStatus(corrupt ? "reset" : "reconnecting",
corrupt ? "Testing cursor reset" : "Connecting live updates",
corrupt ? "Cursor reset test" : "SSE reconnecting");
const source = new EventSource(buildStreamUrl(state.activeFilter, selectedCursor));
state.source = source;
source.addEventListener("open", () => {
if (state.source !== source) {
return;
}
state.streamFailures = 0;
setStatus("connected", "Live updates connected", "SSE live");
});
for (const [name, kind] of [
["session_upsert", "sessionUpsert"],
["session_remove", "sessionRemove"],
["reset", "reset"],
["keepalive", "keepalive"],
]) {
source.addEventListener(name, (message) => {
if (state.source === source) {
handleStreamEvent(message.data, kind);
}
});
}
source.addEventListener("error", () => {
if (state.source !== source) {
return;
}
state.streamFailures += 1;
if (state.streamFailures >= STREAM_FAILURE_LIMIT || source.readyState === EventSource.CLOSED) {
usePolling("Live stream unavailable; polling fallback active");
} else {
setStatus("reconnecting", "Live stream interrupted; reconnecting", "SSE reconnecting");
}
});
}
async function loadConfiguration() {
const configuration = await readJson("/diagnostics/config.json");
if (configuration.contractVersion !== CONTRACT_VERSION
|| !Array.isArray(configuration.scopes)
|| configuration.scopes.length < 1
|| !Number.isInteger(configuration.pollIntervalSeconds)
|| !Number.isInteger(configuration.maximumRenderedSessions)) {
throw new Error("The diagnostic configuration is invalid.");
}
state.configuration = configuration;
state.projection = new SessionProjection(configuration.maximumRenderedSessions);
populate(elements.game, [...new Set(configuration.scopes.map((scope) => scope.gameId))], "");
updateEnvironmentOptions();
state.activeFilter = currentFilter();
await fetchSnapshot("startup");
}
elements.game.addEventListener("change", updateEnvironmentOptions);
elements.environment.addEventListener("change", updateScopeOptions);
elements.form.addEventListener("submit", (event) => {
event.preventDefault();
try {
setFormError();
stopLiveWork();
state.pollingOnly = false;
state.activeFilter = currentFilter();
state.projection = new SessionProjection(state.configuration.maximumRenderedSessions);
fetchSnapshot("filter").catch(showServiceError);
} catch (error) {
setFormError(safeText(error.message, 200));
}
});
elements.reconnect.addEventListener("click", () => {
if (state.projection?.cursor) {
state.pollingOnly = false;
connectStream(state.projection.cursor);
}
});
elements.reset.addEventListener("click", () => {
if (state.projection?.cursor) {
state.pollingOnly = false;
connectStream(state.projection.cursor, true);
}
});
elements.poll.addEventListener("click", () => {
state.pollingOnly = true;
usePolling("Polling only selected deliberately");
fetchSnapshot("poll").catch(showServiceError);
});
window.addEventListener("pagehide", stopLiveWork, { once: true });
setInterval(updateAges, 5000);
loadConfiguration().catch(showServiceError);
}
if (typeof document !== "undefined") {
startDashboard();
}
@@ -0,0 +1,120 @@
<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<meta name="color-scheme" content="dark">
<title>Session diagnostics — Rendezvous</title>
<link rel="stylesheet" href="/diagnostics/styles.css">
<script type="module" src="/diagnostics/app.mjs"></script>
</head>
<body>
<a class="skip-link" href="#main-content">Skip to session results</a>
<header class="site-header">
<div>
<p class="eyebrow">Final Factory infrastructure</p>
<h1>Rendezvous session diagnostics</h1>
<p class="lede">A read-only view of public session discovery. Capacity is advisory; joining always revalidates current state.</p>
</div>
<div class="connection-state" aria-live="polite" aria-atomic="true">
<span class="status-dot" aria-hidden="true"></span>
<span id="stream-status">Loading configuration</span>
</div>
</header>
<main id="main-content" tabindex="-1">
<section class="filter-panel" aria-labelledby="filter-heading">
<div class="section-heading">
<div>
<p class="eyebrow">Public browser scope</p>
<h2 id="filter-heading">Choose a configured session view</h2>
</div>
<p class="boundary-note">This page has no join, publish, or operator authority.</p>
</div>
<form id="filters" novalidate>
<div class="filter-grid">
<label>
<span>Game</span>
<select id="game-filter" name="game" required></select>
</label>
<label>
<span>Environment</span>
<select id="environment-filter" name="environment" required></select>
</label>
<label>
<span>Protocol</span>
<select id="protocol-filter" name="protocol" required></select>
</label>
<label>
<span>Region</span>
<select id="region-filter" name="region" required></select>
</label>
<label>
<span>Availability</span>
<select id="capacity-filter" name="capacity">
<option value="open">Open slots only</option>
<option value="all">Include full sessions</option>
</select>
</label>
<label>
<span>Visibility</span>
<select id="visibility-filter" name="visibility" disabled>
<option value="public">Public only</option>
</select>
</label>
</div>
<p id="filter-error" class="form-error" role="alert" hidden></p>
<div class="button-row">
<button class="button primary" type="submit">Apply filters</button>
<button class="button" id="reconnect-button" type="button">Reconnect now</button>
<button class="button" id="reset-button" type="button">Test reset recovery</button>
<button class="button" id="poll-button" type="button">Use polling only</button>
</div>
</form>
</section>
<section class="summary-panel" aria-labelledby="summary-heading">
<h2 id="summary-heading" class="visually-hidden">Current diagnostic state</h2>
<dl class="summary-grid">
<div>
<dt>Sessions shown</dt>
<dd id="session-count">0</dd>
</div>
<div>
<dt>Transport</dt>
<dd id="transport-state">Starting</dd>
</div>
<div>
<dt>Last successful update</dt>
<dd><time id="last-update">Not yet</time></dd>
</div>
<div>
<dt>Projection</dt>
<dd id="projection-state">Waiting for snapshot</dd>
</div>
</dl>
</section>
<section class="results-panel" aria-labelledby="results-heading" aria-busy="true">
<div class="section-heading results-heading">
<div>
<p class="eyebrow">Bounded public projection</p>
<h2 id="results-heading">Available sessions</h2>
</div>
<p id="results-summary" role="status" aria-live="polite">Loading a fresh snapshot…</p>
</div>
<div id="session-list" class="session-grid" role="list"></div>
<div id="empty-state" class="empty-state" hidden>
<h3>No compatible public sessions</h3>
<p>The service answered successfully, but this configured filter currently has no listings.</p>
</div>
</section>
</main>
<footer>
<p>Read-only diagnostics · no credentials stored · no gameplay or administration</p>
</footer>
<noscript>This diagnostic requires JavaScript to consume the public snapshot and event stream.</noscript>
</body>
</html>
@@ -0,0 +1,505 @@
:root {
color-scheme: dark;
--bg: #071019;
--surface: #101c28;
--surface-raised: #172737;
--border: #375066;
--text: #f2f7fb;
--muted: #b6c8d6;
--accent: #55d7b5;
--accent-strong: #7ce9cc;
--accent-ink: #04231c;
--warning: #ffd479;
--danger: #ff9f9f;
--focus: #ffd479;
--radius: 0.75rem;
--space-1: 0.5rem;
--space-2: 0.75rem;
--space-3: 1rem;
--space-4: 1.5rem;
--space-5: 2rem;
--space-6: 3rem;
font-family: Inter, ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif;
font-size: 100%;
line-height: 1.55;
}
* {
box-sizing: border-box;
}
html {
min-width: 20rem;
background: var(--bg);
}
body {
min-height: 100vh;
margin: 0;
color: var(--text);
background:
radial-gradient(circle at 10% -10%, rgb(36 103 104 / 35%), transparent 32rem),
linear-gradient(180deg, #09141f 0%, var(--bg) 50%);
}
button,
select {
font: inherit;
}
button,
select,
a {
-webkit-tap-highlight-color: transparent;
}
:focus-visible {
outline: 0.2rem solid var(--focus);
outline-offset: 0.2rem;
}
.skip-link {
position: fixed;
z-index: 10;
top: var(--space-2);
left: var(--space-2);
padding: var(--space-2) var(--space-3);
color: #071019;
background: var(--focus);
border-radius: 0.4rem;
font-weight: 800;
transform: translateY(-200%);
}
.skip-link:focus {
transform: translateY(0);
}
.site-header,
main,
footer {
width: min(80rem, calc(100% - 2rem));
margin-inline: auto;
}
.site-header {
display: flex;
align-items: end;
justify-content: space-between;
gap: var(--space-5);
padding-block: var(--space-6) var(--space-5);
}
h1,
h2,
h3,
p {
margin-top: 0;
}
h1 {
max-width: 18ch;
margin-bottom: var(--space-2);
font-size: clamp(2rem, 6vw, 4.25rem);
line-height: 1.02;
letter-spacing: -0.045em;
}
h2 {
margin-bottom: var(--space-1);
font-size: clamp(1.35rem, 3vw, 2rem);
line-height: 1.2;
}
h3 {
margin-bottom: var(--space-1);
font-size: 1.1rem;
}
.eyebrow {
margin-bottom: var(--space-1);
color: var(--accent-strong);
font-size: 0.78rem;
font-weight: 800;
letter-spacing: 0.14em;
text-transform: uppercase;
}
.lede {
max-width: 65ch;
margin-bottom: 0;
color: var(--muted);
font-size: 1.05rem;
}
.connection-state {
display: inline-flex;
min-height: 2.75rem;
align-items: center;
gap: var(--space-2);
padding: var(--space-2) var(--space-3);
border: 1px solid var(--border);
border-radius: 999px;
background: rgb(16 28 40 / 88%);
color: var(--muted);
font-weight: 700;
white-space: nowrap;
}
.status-dot {
width: 0.7rem;
height: 0.7rem;
border: 2px solid currentColor;
border-radius: 50%;
background: currentColor;
}
body[data-stream-state="connected"] .connection-state {
color: var(--accent-strong);
}
body[data-stream-state="reconnecting"] .connection-state,
body[data-stream-state="polling"] .connection-state,
body[data-stream-state="reset"] .connection-state {
color: var(--warning);
}
body[data-stream-state="error"] .connection-state {
color: var(--danger);
}
main {
display: grid;
gap: var(--space-4);
}
.filter-panel,
.summary-panel,
.results-panel {
border: 1px solid var(--border);
border-radius: var(--radius);
background: rgb(16 28 40 / 94%);
box-shadow: 0 1rem 3rem rgb(0 0 0 / 18%);
}
.filter-panel,
.results-panel {
padding: clamp(1rem, 4vw, 2rem);
}
.section-heading {
display: flex;
align-items: start;
justify-content: space-between;
gap: var(--space-4);
margin-bottom: var(--space-4);
}
.boundary-note,
#results-summary {
max-width: 34rem;
margin-bottom: 0;
color: var(--muted);
}
.filter-grid {
display: grid;
grid-template-columns: repeat(3, minmax(0, 1fr));
gap: var(--space-3);
}
label {
display: grid;
gap: 0.35rem;
color: var(--muted);
font-size: 0.9rem;
font-weight: 750;
}
select {
width: 100%;
min-height: 2.75rem;
padding: 0.6rem 2.4rem 0.6rem 0.75rem;
border: 1px solid #587189;
border-radius: 0.45rem;
color: var(--text);
background: #0b1722;
}
select:disabled {
color: #9fb0bd;
border-style: dashed;
opacity: 1;
}
.button-row {
display: flex;
flex-wrap: wrap;
gap: var(--space-2);
margin-top: var(--space-4);
}
.button {
min-height: 2.75rem;
padding: 0.65rem 1rem;
border: 1px solid #6a8298;
border-radius: 0.45rem;
color: var(--text);
background: var(--surface-raised);
cursor: pointer;
font-weight: 800;
}
.button:hover {
border-color: var(--accent-strong);
background: #21384a;
}
.button:active {
transform: translateY(1px);
}
.button.primary {
color: var(--accent-ink);
border-color: var(--accent);
background: var(--accent);
}
.button.primary:hover {
background: var(--accent-strong);
}
.form-error {
margin: var(--space-3) 0 0;
color: var(--danger);
font-weight: 750;
}
.summary-panel {
padding: 0;
overflow: hidden;
}
.summary-grid {
display: grid;
grid-template-columns: repeat(4, minmax(0, 1fr));
margin: 0;
}
.summary-grid > div {
min-width: 0;
padding: var(--space-3) var(--space-4);
border-right: 1px solid var(--border);
}
.summary-grid > div:last-child {
border-right: 0;
}
.summary-grid dt {
color: var(--muted);
font-size: 0.78rem;
font-weight: 700;
}
.summary-grid dd {
margin: 0.25rem 0 0;
overflow-wrap: anywhere;
font-size: 1.05rem;
font-weight: 800;
}
.results-heading {
align-items: end;
}
.session-grid {
display: grid;
grid-template-columns: repeat(auto-fill, minmax(min(100%, 19rem), 1fr));
gap: var(--space-3);
}
.session-card {
min-width: 0;
padding: var(--space-3);
border: 1px solid #496177;
border-radius: 0.6rem;
background: #0b1722;
}
.card-heading {
display: flex;
align-items: start;
justify-content: space-between;
gap: var(--space-2);
}
.card-heading h3 {
overflow-wrap: anywhere;
}
.region-badge {
flex: 0 0 auto;
padding: 0.15rem 0.5rem;
border: 1px solid #597187;
border-radius: 999px;
color: var(--muted);
font-size: 0.75rem;
font-weight: 750;
}
.session-detail {
display: grid;
grid-template-columns: minmax(5rem, auto) 1fr;
gap: 0.25rem var(--space-2);
margin: var(--space-3) 0 0;
font-size: 0.9rem;
}
.session-detail dt {
color: var(--muted);
}
.session-detail dd {
min-width: 0;
margin: 0;
overflow-wrap: anywhere;
}
.capacity-row {
margin-top: var(--space-3);
}
.capacity-copy {
display: flex;
justify-content: space-between;
gap: var(--space-2);
margin-bottom: 0.35rem;
color: var(--muted);
font-size: 0.85rem;
}
meter {
width: 100%;
height: 0.65rem;
accent-color: var(--accent);
}
.metadata-list {
display: flex;
flex-wrap: wrap;
gap: 0.35rem;
margin: var(--space-3) 0 0;
padding: 0;
list-style: none;
}
.metadata-list li {
max-width: 100%;
padding: 0.2rem 0.45rem;
overflow-wrap: anywhere;
border-radius: 0.3rem;
color: #d6e4ed;
background: #1b2b39;
font-size: 0.78rem;
}
.updated-age {
margin: var(--space-3) 0 0;
color: var(--muted);
font-size: 0.78rem;
}
.empty-state {
padding: var(--space-6) var(--space-3);
text-align: center;
border: 1px dashed #587189;
border-radius: 0.6rem;
color: var(--muted);
}
.empty-state h3 {
color: var(--text);
}
footer {
padding-block: var(--space-5);
color: var(--muted);
font-size: 0.85rem;
text-align: center;
}
.visually-hidden {
position: absolute;
width: 1px;
height: 1px;
padding: 0;
overflow: hidden;
clip: rect(0, 0, 0, 0);
white-space: nowrap;
border: 0;
}
[hidden] {
display: none !important;
}
@media (max-width: 56rem) {
.site-header,
.section-heading {
align-items: start;
flex-direction: column;
}
.filter-grid,
.summary-grid {
grid-template-columns: repeat(2, minmax(0, 1fr));
}
.summary-grid > div:nth-child(2) {
border-right: 0;
}
.summary-grid > div:nth-child(-n + 2) {
border-bottom: 1px solid var(--border);
}
}
@media (max-width: 36rem) {
.site-header,
main,
footer {
width: min(100% - 1rem, 80rem);
}
.site-header {
padding-block: var(--space-5) var(--space-4);
}
.filter-grid,
.summary-grid {
grid-template-columns: 1fr;
}
.summary-grid > div {
border-right: 0;
border-bottom: 1px solid var(--border);
}
.summary-grid > div:last-child {
border-bottom: 0;
}
.button {
width: 100%;
}
}
@media (prefers-reduced-motion: reduce) {
*,
*::before,
*::after {
scroll-behavior: auto !important;
transition-duration: 0.01ms !important;
}
}
@@ -0,0 +1,111 @@
using System.Reflection;
using FinalFactory.Rendezvous.Contracts;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Options;
namespace FinalFactory.Rendezvous.Server.Diagnostics;
internal static class DiagnosticDashboardEndpoints
{
private const string ContentSecurityPolicy =
"default-src 'none'; base-uri 'none'; connect-src 'self'; "
+ "font-src 'self'; form-action 'none'; frame-ancestors 'none'; "
+ "img-src 'self'; manifest-src 'none'; object-src 'none'; "
+ "script-src 'self'; style-src 'self'";
private static readonly byte[] Index = ReadAsset("index.html");
private static readonly byte[] Script = ReadAsset("app.mjs");
private static readonly byte[] Styles = ReadAsset("styles.css");
public static IEndpointRouteBuilder MapDiagnosticDashboardEndpoints(
this IEndpointRouteBuilder endpoints)
{
RouteGroupBuilder dashboard = endpoints.MapGroup("/diagnostics")
.ExcludeFromDescription();
dashboard.MapGet("", ServeIndex);
dashboard.MapGet("/app.mjs", ServeScript);
dashboard.MapGet("/styles.css", ServeStyles);
dashboard.MapGet("/config.json", ServeConfiguration);
return endpoints;
}
private static IResult ServeIndex(
HttpContext context,
[FromServices] IOptions<DiagnosticDashboardOptions> configured) =>
ServeAsset(context, configured.Value, Index, "text/html; charset=utf-8");
private static IResult ServeScript(
HttpContext context,
[FromServices] IOptions<DiagnosticDashboardOptions> configured) =>
ServeAsset(context, configured.Value, Script, "text/javascript; charset=utf-8");
private static IResult ServeStyles(
HttpContext context,
[FromServices] IOptions<DiagnosticDashboardOptions> configured) =>
ServeAsset(context, configured.Value, Styles, "text/css; charset=utf-8");
private static IResult ServeConfiguration(
HttpContext context,
[FromServices] IOptions<DiagnosticDashboardOptions> configured)
{
DiagnosticDashboardOptions options = configured.Value;
if (!options.Enabled)
{
return Results.NotFound();
}
ApplySecurityHeaders(context.Response);
return Results.Json(new
{
contractVersion = ContractLimits.ContractVersion,
pollIntervalSeconds = options.PollIntervalSeconds,
maximumRenderedSessions = options.MaximumRenderedSessions,
scopes = options.Scopes.Select(static scope => new
{
gameId = scope.GameId,
environmentId = scope.EnvironmentId,
protocolVersions = scope.ProtocolVersions,
regions = scope.Regions,
visibility = "public",
}),
});
}
private static IResult ServeAsset(
HttpContext context,
DiagnosticDashboardOptions options,
byte[] content,
string contentType)
{
if (!options.Enabled)
{
return Results.NotFound();
}
ApplySecurityHeaders(context.Response);
return Results.Bytes(content, contentType);
}
private static void ApplySecurityHeaders(HttpResponse response)
{
response.Headers.CacheControl = "no-store";
response.Headers["Content-Security-Policy"] = ContentSecurityPolicy;
response.Headers["X-Content-Type-Options"] = "nosniff";
response.Headers["X-Frame-Options"] = "DENY";
response.Headers["Cross-Origin-Opener-Policy"] = "same-origin";
response.Headers["Cross-Origin-Resource-Policy"] = "same-origin";
response.Headers["Permissions-Policy"] =
"camera=(), geolocation=(), microphone=(), payment=(), usb=()";
response.Headers["Referrer-Policy"] = "no-referrer";
}
private static byte[] ReadAsset(string fileName)
{
Assembly assembly = typeof(DiagnosticDashboardEndpoints).Assembly;
string resourceName = $"FinalFactory.Rendezvous.Server.Diagnostics.Assets.{fileName}";
using Stream source = assembly.GetManifestResourceStream(resourceName)
?? throw new InvalidOperationException($"Missing embedded dashboard asset {fileName}.");
using MemoryStream destination = new();
source.CopyTo(destination);
return destination.ToArray();
}
}
@@ -0,0 +1,81 @@
using FinalFactory.Rendezvous.Contracts;
namespace FinalFactory.Rendezvous.Server.Diagnostics;
internal sealed record DiagnosticDashboardOptions
{
public const string SectionName = "Rendezvous:Diagnostics";
public bool Enabled { get; init; }
public int PollIntervalSeconds { get; init; } = 10;
public int MaximumRenderedSessions { get; init; } = 100;
public DiagnosticDashboardScope[] Scopes { get; init; } = [];
public IReadOnlyList<string> Validate()
{
List<string> errors = [];
if (PollIntervalSeconds is < 5 or > 60)
{
errors.Add($"{SectionName}:PollIntervalSeconds must be between 5 and 60.");
}
if (MaximumRenderedSessions is < 10 or > 500)
{
errors.Add($"{SectionName}:MaximumRenderedSessions must be between 10 and 500.");
}
if (!Enabled)
{
return errors;
}
if (Scopes is null || Scopes.Length is < 1 or > 32)
{
errors.Add($"{SectionName}:Scopes must contain between 1 and 32 allow-listed scopes when enabled.");
return errors;
}
HashSet<string> identities = new(StringComparer.Ordinal);
foreach (DiagnosticDashboardScope? scope in Scopes)
{
if (scope is null)
{
errors.Add($"{SectionName}:Scopes cannot contain null entries.");
continue;
}
string gameId = scope.GameId ?? string.Empty;
string environmentId = scope.EnvironmentId ?? string.Empty;
uint[] protocolVersions = scope.ProtocolVersions ?? [];
string[] regions = scope.Regions ?? [];
if (!GameId.TryParse(gameId, out _)
|| !EnvironmentId.TryParse(environmentId, out _))
{
errors.Add($"{SectionName}:Scopes contains an invalid game or environment identifier.");
}
if (protocolVersions.Length is < 1 or > 16
|| protocolVersions.Any(static version => version == 0)
|| protocolVersions.Distinct().Count() != protocolVersions.Length)
{
errors.Add($"{SectionName}:Scopes protocol versions must contain 1-16 unique positive values.");
}
if (regions.Length is < 1 or > 16
|| regions.Any(static region => !RegionId.TryParse(region ?? string.Empty, out _))
|| regions.Distinct(StringComparer.Ordinal).Count() != regions.Length)
{
errors.Add($"{SectionName}:Scopes regions must contain 1-16 unique valid identifiers.");
}
string identity = $"{gameId}\n{environmentId}";
if (!identities.Add(identity))
{
errors.Add($"{SectionName}:Scopes contains a duplicate game/environment pair.");
}
}
return errors;
}
}
internal sealed record DiagnosticDashboardScope
{
public string GameId { get; init; } = string.Empty;
public string EnvironmentId { get; init; } = string.Empty;
public uint[] ProtocolVersions { get; init; } = [];
public string[] Regions { get; init; } = [];
}
@@ -53,6 +53,8 @@
<Compile Include="ConnectionOutcomes/ConnectionOutcomeService.cs" /> <Compile Include="ConnectionOutcomes/ConnectionOutcomeService.cs" />
<Compile Include="Deployment/DeploymentOptions.cs" /> <Compile Include="Deployment/DeploymentOptions.cs" />
<Compile Include="Deployment/GracefulDrainService.cs" /> <Compile Include="Deployment/GracefulDrainService.cs" />
<Compile Include="Diagnostics/DiagnosticDashboardEndpoints.cs" />
<Compile Include="Diagnostics/DiagnosticDashboardOptions.cs" />
<Compile Include="Http/ContractEndpoints.cs" /> <Compile Include="Http/ContractEndpoints.cs" />
<Compile Include="Http/RendezvousExceptionHandler.cs" /> <Compile Include="Http/RendezvousExceptionHandler.cs" />
<Compile Include="JoinAttempts/JoinAttemptCursorCodec.cs" /> <Compile Include="JoinAttempts/JoinAttemptCursorCodec.cs" />
@@ -60,6 +62,7 @@
<Compile Include="Observability/AuditOptions.cs" /> <Compile Include="Observability/AuditOptions.cs" />
<Compile Include="Observability/AuditTrail.cs" /> <Compile Include="Observability/AuditTrail.cs" />
<Compile Include="Observability/HealthEndpoints.cs" /> <Compile Include="Observability/HealthEndpoints.cs" />
<Compile Include="Observability/PrometheusMetricsEndpoint.cs" />
<Compile Include="Observability/RendezvousReadiness.cs" /> <Compile Include="Observability/RendezvousReadiness.cs" />
<Compile Include="Observability/RendezvousTelemetry.cs" /> <Compile Include="Observability/RendezvousTelemetry.cs" />
<Compile Include="Observability/TelemetryMiddleware.cs" /> <Compile Include="Observability/TelemetryMiddleware.cs" />
@@ -88,4 +91,15 @@
<Compile Include="Transport/UdpMediatorOptions.cs" /> <Compile Include="Transport/UdpMediatorOptions.cs" />
<Compile Include="Transport/UdpMediatorService.cs" /> <Compile Include="Transport/UdpMediatorService.cs" />
</ItemGroup> </ItemGroup>
<ItemGroup>
<EmbeddedResource Include="Diagnostics/Assets/app.mjs">
<LogicalName>FinalFactory.Rendezvous.Server.Diagnostics.Assets.app.mjs</LogicalName>
</EmbeddedResource>
<EmbeddedResource Include="Diagnostics/Assets/index.html">
<LogicalName>FinalFactory.Rendezvous.Server.Diagnostics.Assets.index.html</LogicalName>
</EmbeddedResource>
<EmbeddedResource Include="Diagnostics/Assets/styles.css">
<LogicalName>FinalFactory.Rendezvous.Server.Diagnostics.Assets.styles.css</LogicalName>
</EmbeddedResource>
</ItemGroup>
</Project> </Project>
@@ -0,0 +1,148 @@
using System.Security.Cryptography;
using System.Text;
using FinalFactory.Rendezvous.Server.Provisioning;
namespace FinalFactory.Rendezvous.Server.Observability;
internal sealed record PrometheusMetricsOptions
{
public const string SectionName = "Rendezvous:Metrics";
public bool Enabled { get; init; }
public string BearerTokenSecretReference { get; init; } = string.Empty;
public IReadOnlyList<string> Validate()
{
if (!Enabled)
{
return [];
}
string reference = BearerTokenSecretReference ?? string.Empty;
bool environmentReference = reference.StartsWith("env:", StringComparison.Ordinal)
&& reference.Length > "env:".Length;
bool absoluteFileReference = reference.StartsWith("file:", StringComparison.Ordinal)
&& Path.IsPathFullyQualified(reference["file:".Length..]);
return reference.Length is < 5 or > 512
|| !(environmentReference || absoluteFileReference)
? [$"{SectionName}:BearerTokenSecretReference must be a bounded env: or absolute file: secret reference when metrics are enabled."]
: [];
}
}
internal sealed class MetricsAccessCredential : IDisposable
{
private byte[]? _token;
private MetricsAccessCredential(byte[] token) => _token = token;
public static bool TryCreate(
PrometheusMetricsOptions options,
ISecretProvider secrets,
out MetricsAccessCredential? credential)
{
credential = null;
if (!options.Enabled
|| !secrets.TryGetSecret(options.BearerTokenSecretReference, out SecretMaterial? material)
|| material is null)
{
return false;
}
using (material)
{
byte[] bytes = material.CopyBytes();
int length = bytes.Length;
while (length > 0 && bytes[length - 1] is (byte)'\r' or (byte)'\n')
{
length--;
}
bool valid = length is >= 32 and <= 128
&& bytes.AsSpan(0, length).IndexOfAnyExceptInRange((byte)0x21, (byte)0x7e) < 0;
if (!valid)
{
CryptographicOperations.ZeroMemory(bytes);
return false;
}
byte[] token = bytes.AsSpan(0, length).ToArray();
CryptographicOperations.ZeroMemory(bytes);
credential = new(token);
return true;
}
}
public bool Authorizes(HttpRequest request)
{
byte[]? expected = _token;
string authorization = request.Headers.Authorization.ToString();
const string prefix = "Bearer ";
if (expected is null
|| !authorization.StartsWith(prefix, StringComparison.Ordinal)
|| authorization.Length - prefix.Length is < 32 or > 128)
{
return false;
}
byte[] supplied = Encoding.UTF8.GetBytes(authorization[prefix.Length..]);
try
{
return supplied.Length == expected.Length
&& CryptographicOperations.FixedTimeEquals(supplied, expected);
}
finally
{
CryptographicOperations.ZeroMemory(supplied);
}
}
public void Dispose()
{
byte[]? token = Interlocked.Exchange(ref _token, null);
if (token is not null)
{
CryptographicOperations.ZeroMemory(token);
}
}
public override string ToString() => "[MetricsAccessCredential: REDACTED]";
}
internal static class PrometheusMetricsEndpoint
{
public static IEndpointRouteBuilder MapPrometheusMetricsEndpoint(
this IEndpointRouteBuilder endpoints,
PrometheusMetricsOptions options,
MetricsAccessCredential? credential)
{
endpoints.MapGet("/metrics", (HttpContext context, RendezvousTelemetry telemetry) =>
Export(context, telemetry, options, credential))
.WithName("MetricsScrape")
.ExcludeFromDescription();
return endpoints;
}
private static IResult Export(
HttpContext context,
RendezvousTelemetry telemetry,
PrometheusMetricsOptions options,
MetricsAccessCredential? credential)
{
if (!options.Enabled)
{
return Results.NotFound();
}
if (credential is null || !credential.Authorizes(context.Request))
{
telemetry.RecordMetricsScrape("rejected");
return Results.NotFound();
}
telemetry.RecordMetricsScrape("accepted");
context.Response.Headers.CacheControl = "no-store";
context.Response.Headers["X-Content-Type-Options"] = "nosniff";
return Results.Text(
telemetry.RenderPrometheus(),
"text/plain; version=0.0.4; charset=utf-8",
Encoding.UTF8);
}
}
@@ -1,5 +1,10 @@
using System.Collections.Concurrent;
using System.Diagnostics; using System.Diagnostics;
using System.Diagnostics.Metrics; using System.Diagnostics.Metrics;
using System.Globalization;
using System.Text;
using FinalFactory.Rendezvous.Server.Browser;
using FinalFactory.Rendezvous.Server.Provisioning;
using FinalFactory.Rendezvous.Server.State; using FinalFactory.Rendezvous.Server.State;
namespace FinalFactory.Rendezvous.Server.Observability; namespace FinalFactory.Rendezvous.Server.Observability;
@@ -10,6 +15,10 @@ internal sealed class RendezvousTelemetry : IDisposable
public const string ActivitySourceName = "FinalFactory.Rendezvous.Server"; public const string ActivitySourceName = "FinalFactory.Rendezvous.Server";
private readonly InMemoryEphemeralRendezvousStore _store; private readonly InMemoryEphemeralRendezvousStore _store;
private readonly SessionChangeJournal? _sessionChanges;
private readonly ProvisioningRuntime? _provisioning;
private readonly ConcurrentDictionary<MetricSeriesKey, long> _prometheusCounters = new();
private readonly ConcurrentDictionary<MetricSeriesKey, PrometheusHistogram> _prometheusHistograms = new();
private readonly Meter _meter = new(MeterName, "1.0.0"); private readonly Meter _meter = new(MeterName, "1.0.0");
private readonly ActivitySource _activities = new(ActivitySourceName, "1.0.0"); private readonly ActivitySource _activities = new(ActivitySourceName, "1.0.0");
private readonly Counter<long> _httpRequests; private readonly Counter<long> _httpRequests;
@@ -22,9 +31,14 @@ internal sealed class RendezvousTelemetry : IDisposable
private readonly Counter<long> _operatorAuthentication; private readonly Counter<long> _operatorAuthentication;
private readonly Histogram<double> _pairingLatency; private readonly Histogram<double> _pairingLatency;
public RendezvousTelemetry(InMemoryEphemeralRendezvousStore store) public RendezvousTelemetry(
InMemoryEphemeralRendezvousStore store,
SessionChangeJournal? sessionChanges = null,
ProvisioningRuntime? provisioning = null)
{ {
_store = store; _store = store;
_sessionChanges = sessionChanges;
_provisioning = provisioning;
_httpRequests = _meter.CreateCounter<long>("rendezvous.http.requests"); _httpRequests = _meter.CreateCounter<long>("rendezvous.http.requests");
_httpDuration = _meter.CreateHistogram<double>( _httpDuration = _meter.CreateHistogram<double>(
"rendezvous.http.duration", "rendezvous.http.duration",
@@ -75,9 +89,21 @@ internal sealed class RendezvousTelemetry : IDisposable
}; };
_httpRequests.Add(1, tags); _httpRequests.Add(1, tags);
_httpDuration.Record(elapsedMilliseconds, tags); _httpDuration.Record(elapsedMilliseconds, tags);
IncrementPrometheus(
"rendezvous_http_requests_total",
Labels(("operation", operation), ("status_code", statusCode.ToString(CultureInfo.InvariantCulture))));
ObservePrometheus(
"rendezvous_http_duration_milliseconds",
elapsedMilliseconds,
Labels(("operation", operation), ("status_code", statusCode.ToString(CultureInfo.InvariantCulture))));
} }
public void RecordUdp(string operation, string result, double elapsedMilliseconds) public void RecordUdp(
string operation,
string result,
double elapsedMilliseconds,
int receivedBytes = 0,
int responseBudgetBytes = 0)
{ {
TagList tags = new() TagList tags = new()
{ {
@@ -86,41 +112,315 @@ internal sealed class RendezvousTelemetry : IDisposable
}; };
_udpResults.Add(1, tags); _udpResults.Add(1, tags);
_udpDuration.Record(elapsedMilliseconds, tags); _udpDuration.Record(elapsedMilliseconds, tags);
IncrementPrometheus(
"rendezvous_udp_results_total",
Labels(("operation", operation), ("result", result)));
ObservePrometheus(
"rendezvous_udp_duration_milliseconds",
elapsedMilliseconds,
Labels(("operation", operation), ("result", result)));
AddPrometheus(
"rendezvous_udp_received_bytes_total",
Math.Max(0, receivedBytes),
Labels(("operation", operation)));
AddPrometheus(
"rendezvous_udp_response_budget_bytes_total",
Math.Max(0, responseBudgetBytes),
Labels(("operation", operation)));
} }
public void RecordLimiterDrop(string transport, string partition) => public void RecordLimiterDrop(string transport, string partition)
{
_limiterDrops.Add(1, new TagList _limiterDrops.Add(1, new TagList
{ {
{ "transport", transport }, { "transport", transport },
{ "partition", partition }, { "partition", partition },
}); });
IncrementPrometheus(
"rendezvous_limiter_drops_total",
Labels(("transport", transport), ("partition", partition)));
}
public void RecordAudit(string action, string result) => public void RecordAudit(string action, string result)
{
_auditEvents.Add(1, new TagList _auditEvents.Add(1, new TagList
{ {
{ "action", action }, { "action", action },
{ "result", result }, { "result", result },
}); });
IncrementPrometheus(
"rendezvous_audit_events_total",
Labels(("action", action), ("result", result)));
}
public void RecordConnectionOutcome(string outcome, string elapsedBucket) => public void RecordConnectionOutcome(string outcome, string elapsedBucket)
{
_connectionOutcomes.Add(1, new TagList _connectionOutcomes.Add(1, new TagList
{ {
{ "outcome", outcome }, { "outcome", outcome },
{ "elapsed_bucket", elapsedBucket }, { "elapsed_bucket", elapsedBucket },
}); });
IncrementPrometheus(
"rendezvous_connection_outcomes_total",
Labels(("outcome", outcome), ("elapsed_bucket", elapsedBucket)));
}
public void RecordOperatorAuthentication(string result) => public void RecordOperatorAuthentication(string result)
{
_operatorAuthentication.Add(1, new TagList _operatorAuthentication.Add(1, new TagList
{ {
{ "result", result }, { "result", result },
}); });
IncrementPrometheus(
"rendezvous_operator_authentication_total",
Labels(("result", result)));
}
public void RecordPairingLatency(double elapsedMilliseconds) => public void RecordPairingLatency(double elapsedMilliseconds)
{
_pairingLatency.Record(elapsedMilliseconds); _pairingLatency.Record(elapsedMilliseconds);
ObservePrometheus("rendezvous_pairing_latency_milliseconds", elapsedMilliseconds, string.Empty);
}
public void RecordMetricsScrape(string result) => IncrementPrometheus(
"rendezvous_metrics_scrapes_total",
Labels(("result", result)));
public string RenderPrometheus()
{
StringBuilder output = new(16 * 1024);
foreach (IGrouping<string, KeyValuePair<MetricSeriesKey, long>> family in _prometheusCounters
.OrderBy(static item => item.Key.Name, StringComparer.Ordinal)
.ThenBy(static item => item.Key.Labels, StringComparer.Ordinal)
.GroupBy(static item => item.Key.Name, StringComparer.Ordinal))
{
output.Append("# TYPE ").Append(family.Key).Append(" counter\n");
foreach (KeyValuePair<MetricSeriesKey, long> series in family)
{
AppendValue(output, series.Key.Name, series.Key.Labels, series.Value);
}
}
foreach (IGrouping<string, KeyValuePair<MetricSeriesKey, PrometheusHistogram>> family in
_prometheusHistograms
.OrderBy(static item => item.Key.Name, StringComparer.Ordinal)
.ThenBy(static item => item.Key.Labels, StringComparer.Ordinal)
.GroupBy(static item => item.Key.Name, StringComparer.Ordinal))
{
output.Append("# TYPE ").Append(family.Key).Append(" histogram\n");
foreach (KeyValuePair<MetricSeriesKey, PrometheusHistogram> series in family)
{
series.Value.Append(output, series.Key.Name, series.Key.Labels);
}
}
EphemeralStoreSnapshot store = _store.GetMetricsSnapshot();
AppendGauge(output, "rendezvous_store_active_listings", store.ActiveListings);
AppendGauge(output, "rendezvous_store_fresh_presence_bindings", store.FreshPresenceBindings);
AppendGauge(
output,
"rendezvous_store_awaiting_presence_listings",
Math.Max(0, store.ActiveListings - store.FreshPresenceBindings));
AppendGauge(output, "rendezvous_store_active_leases", store.ActiveListings);
AppendGauge(output, "rendezvous_store_active_attempts", store.ActiveJoinAttempts);
AppendGauge(output, "rendezvous_queue_depth", store.ActiveJoinAttempts);
AppendGauge(output, "rendezvous_store_replay_markers", store.ReplayMarkers);
AppendGauge(output, "rendezvous_store_available", store.IsAvailable ? 1 : 0);
AppendGauge(output, "rendezvous_store_draining", store.IsDraining ? 1 : 0);
AppendCounter(output, "rendezvous_store_expiry_churn_total", store.ExpiryChurn);
if (_sessionChanges is not null)
{
AppendGauge(output, "rendezvous_browser_sse_subscribers", _sessionChanges.SubscriberCount);
AppendGauge(output, "rendezvous_browser_sse_tenants", _sessionChanges.SubscribedTenantCount);
AppendGauge(output, "rendezvous_browser_replay_entries", _sessionChanges.ReplayCount);
}
AppendProcessMetrics(output);
AppendSigningKeyMetrics(output);
return output.ToString();
}
private void AppendSigningKeyMetrics(StringBuilder output)
{
if (_provisioning is null)
{
return;
}
DateTimeOffset now = DateTimeOffset.UtcNow;
SigningKeyStatus[] statuses = _provisioning.SigningKeys.GetStatuses(now).ToArray();
output.Append("# TYPE rendezvous_signing_keys gauge\n");
foreach (IGrouping<string, SigningKeyStatus> state in statuses.GroupBy(
static status => status.Status,
StringComparer.Ordinal))
{
AppendValue(
output,
"rendezvous_signing_keys",
Labels(("state", state.Key)),
state.Count());
}
double seconds = statuses
.Where(static status => status.Status == "signing")
.Select(status => Math.Max(0, (status.SignUntil - now).TotalSeconds))
.DefaultIfEmpty(0)
.Min();
AppendGauge(output, "rendezvous_signing_key_sign_seconds_remaining", seconds);
}
private static void AppendProcessMetrics(StringBuilder output)
{
using Process process = Process.GetCurrentProcess();
process.Refresh();
AppendCounter(output, "process_cpu_seconds_total", process.TotalProcessorTime.TotalSeconds);
AppendGauge(output, "process_resident_memory_bytes", process.WorkingSet64);
AppendGauge(output, "process_virtual_memory_bytes", process.VirtualMemorySize64);
AppendGauge(output, "process_threads", process.Threads.Count);
try
{
AppendGauge(
output,
"process_open_file_descriptors",
Directory.EnumerateFileSystemEntries("/proc/self/fd").Count());
}
catch (Exception exception) when (exception is IOException
or UnauthorizedAccessException)
{
}
AppendGauge(output, "dotnet_gc_heap_size_bytes", GC.GetTotalMemory(forceFullCollection: false));
output.Append("# TYPE dotnet_gc_collections_total counter\n");
AppendValue(output, "dotnet_gc_collections_total", Labels(("generation", "0")), GC.CollectionCount(0));
AppendValue(output, "dotnet_gc_collections_total", Labels(("generation", "1")), GC.CollectionCount(1));
AppendValue(output, "dotnet_gc_collections_total", Labels(("generation", "2")), GC.CollectionCount(2));
AppendGauge(output, "dotnet_thread_pool_threads", ThreadPool.ThreadCount);
ThreadPool.GetAvailableThreads(out int workerThreads, out int completionThreads);
AppendGauge(output, "dotnet_thread_pool_available_worker_threads", workerThreads);
AppendGauge(output, "dotnet_thread_pool_available_completion_threads", completionThreads);
}
private void IncrementPrometheus(string name, string labels) =>
_prometheusCounters.AddOrUpdate(new(name, labels), 1, static (_, current) => current + 1);
private void AddPrometheus(string name, long value, string labels)
{
if (value <= 0)
{
return;
}
_prometheusCounters.AddOrUpdate(new(name, labels), value, (_, current) => current + value);
}
private void ObservePrometheus(string name, double value, string labels) =>
_prometheusHistograms.GetOrAdd(new(name, labels), static _ => new()).Observe(value);
private static string Labels(params (string Name, string Value)[] labels)
{
if (labels.Length == 0)
{
return string.Empty;
}
return "{" + string.Join(',', labels.Select(static label =>
$"{label.Name}=\"{EscapeLabel(NormalizeLabel(label.Value))}\"")) + "}";
}
private static string NormalizeLabel(string value)
{
if (string.IsNullOrWhiteSpace(value) || value.Length > 64)
{
return "other";
}
return value.All(static character => char.IsAsciiLetterOrDigit(character)
|| character is '-' or '_' or '.')
? value
: "other";
}
private static string EscapeLabel(string value) => value
.Replace("\\", "\\\\", StringComparison.Ordinal)
.Replace("\"", "\\\"", StringComparison.Ordinal)
.Replace("\n", "\\n", StringComparison.Ordinal);
private static void AppendCounter(
StringBuilder output,
string name,
double value,
string labels = "")
{
output.Append("# TYPE ").Append(name).Append(" counter\n");
AppendValue(output, name, labels, value);
}
private static void AppendGauge(
StringBuilder output,
string name,
double value,
string labels = "")
{
output.Append("# TYPE ").Append(name).Append(" gauge\n");
AppendValue(output, name, labels, value);
}
private static void AppendValue(StringBuilder output, string name, string labels, double value) =>
output.Append(name)
.Append(labels)
.Append(' ')
.Append(value.ToString("R", CultureInfo.InvariantCulture))
.Append('\n');
public void Dispose() public void Dispose()
{ {
_activities.Dispose(); _activities.Dispose();
_meter.Dispose(); _meter.Dispose();
} }
private readonly record struct MetricSeriesKey(string Name, string Labels);
private sealed class PrometheusHistogram
{
private static readonly double[] Bounds = [1, 5, 10, 25, 50, 100, 250, 500, 1000, 5000];
private readonly object _gate = new();
private readonly long[] _buckets = new long[Bounds.Length];
private long _count;
private double _sum;
public void Observe(double value)
{
if (!double.IsFinite(value) || value < 0)
{
return;
}
lock (_gate)
{
_count++;
_sum += value;
for (int index = 0; index < Bounds.Length; index++)
{
if (value <= Bounds[index])
{
_buckets[index]++;
}
}
}
}
public void Append(StringBuilder output, string name, string labels)
{
lock (_gate)
{
for (int index = 0; index < Bounds.Length; index++)
{
AppendValue(
output,
name + "_bucket",
AddLabel(labels, "le", Bounds[index].ToString("R", CultureInfo.InvariantCulture)),
_buckets[index]);
}
AppendValue(output, name + "_bucket", AddLabel(labels, "le", "+Inf"), _count);
AppendValue(output, name + "_sum", labels, _sum);
AppendValue(output, name + "_count", labels, _count);
}
}
private static string AddLabel(string labels, string name, string value) =>
string.IsNullOrEmpty(labels)
? $"{{{name}=\"{value}\"}}"
: labels[..^1] + $",{name}=\"{value}\"}}";
}
} }
@@ -4,6 +4,7 @@ using FinalFactory.Rendezvous.Server.Abuse;
using FinalFactory.Rendezvous.Server.Browser; using FinalFactory.Rendezvous.Server.Browser;
using FinalFactory.Rendezvous.Server.ConnectionOutcomes; using FinalFactory.Rendezvous.Server.ConnectionOutcomes;
using FinalFactory.Rendezvous.Server.Deployment; using FinalFactory.Rendezvous.Server.Deployment;
using FinalFactory.Rendezvous.Server.Diagnostics;
using FinalFactory.Rendezvous.Server.Http; using FinalFactory.Rendezvous.Server.Http;
using FinalFactory.Rendezvous.Server.JoinAttempts; using FinalFactory.Rendezvous.Server.JoinAttempts;
using FinalFactory.Rendezvous.Server.Observability; using FinalFactory.Rendezvous.Server.Observability;
@@ -251,6 +252,37 @@ if (!builder.Environment.IsDevelopment() && !isOpenApiGeneration)
} }
builder.Services.AddSingleton(Microsoft.Extensions.Options.Options.Create(deploymentOptions)); builder.Services.AddSingleton(Microsoft.Extensions.Options.Options.Create(deploymentOptions));
DiagnosticDashboardOptions diagnosticDashboardOptions = builder.Configuration
.GetSection(DiagnosticDashboardOptions.SectionName)
.Get<DiagnosticDashboardOptions>() ?? new DiagnosticDashboardOptions();
IReadOnlyList<string> diagnosticErrors = diagnosticDashboardOptions.Validate();
if (diagnosticErrors.Count > 0)
{
throw new DeploymentConfigurationException(diagnosticErrors);
}
builder.Services.AddSingleton(
Microsoft.Extensions.Options.Options.Create(diagnosticDashboardOptions));
PrometheusMetricsOptions metricsOptions = builder.Configuration
.GetSection(PrometheusMetricsOptions.SectionName)
.Get<PrometheusMetricsOptions>() ?? new PrometheusMetricsOptions();
IReadOnlyList<string> metricsErrors = metricsOptions.Validate();
if (metricsErrors.Count > 0)
{
throw new DeploymentConfigurationException(metricsErrors);
}
builder.Services.AddSingleton(Microsoft.Extensions.Options.Options.Create(metricsOptions));
MetricsAccessCredential? metricsCredential = null;
if (metricsOptions.Enabled && !isOpenApiGeneration)
{
EnvironmentSecretProvider metricsSecrets = new();
if (!MetricsAccessCredential.TryCreate(metricsOptions, metricsSecrets, out metricsCredential)
|| metricsCredential is null)
{
throw new DeploymentConfigurationException(
[$"{PrometheusMetricsOptions.SectionName}:BearerTokenSecretReference did not resolve to 32-128 visible ASCII bytes."]);
}
builder.Services.AddSingleton(metricsCredential);
}
builder.Services.Configure<HostOptions>(options => builder.Services.Configure<HostOptions>(options =>
options.ShutdownTimeout = TimeSpan.FromSeconds(deploymentOptions.DrainDeadlineSeconds + 10)); options.ShutdownTimeout = TimeSpan.FromSeconds(deploymentOptions.DrainDeadlineSeconds + 10));
@@ -349,6 +381,8 @@ app.MapOpenApi();
app.MapRendezvousContractEndpoints(); app.MapRendezvousContractEndpoints();
app.MapOperatorEndpoints(); app.MapOperatorEndpoints();
app.MapRendezvousHealthEndpoints(); app.MapRendezvousHealthEndpoints();
app.MapDiagnosticDashboardEndpoints();
app.MapPrometheusMetricsEndpoint(metricsOptions, metricsCredential);
await app.RunAsync(); await app.RunAsync();
@@ -82,7 +82,9 @@ internal sealed class NatMediationProcessor(
telemetry?.RecordUdp( telemetry?.RecordUdp(
"frozen", "frozen",
result.ToString(), result.ToString(),
Stopwatch.GetElapsedTime(started).TotalMilliseconds); Stopwatch.GetElapsedTime(started).TotalMilliseconds,
encoded.Length,
result == NatMediationResult.Introduced ? 2 * ContractLimits.UdpDatagramMaxBytes : 0);
return result; return result;
} }
@@ -157,7 +159,7 @@ internal sealed class NatMediationProcessor(
observedPublicEndpoint, observedPublicEndpoint,
token, token,
introductionSink, introductionSink,
cancellationToken); cancellationToken: cancellationToken);
} }
internal NatMediationResult ProcessRequestAfterIngress( internal NatMediationResult ProcessRequestAfterIngress(
@@ -165,6 +167,7 @@ internal sealed class NatMediationProcessor(
IPEndPoint observedPublicEndpoint, IPEndPoint observedPublicEndpoint,
string token, string token,
INatIntroductionSink introductionSink, INatIntroductionSink introductionSink,
int receivedBytes = 0,
CancellationToken cancellationToken = default) CancellationToken cancellationToken = default)
{ {
long started = Stopwatch.GetTimestamp(); long started = Stopwatch.GetTimestamp();
@@ -178,7 +181,9 @@ internal sealed class NatMediationProcessor(
telemetry?.RecordUdp( telemetry?.RecordUdp(
"litenet", "litenet",
result.ToString(), result.ToString(),
Stopwatch.GetElapsedTime(started).TotalMilliseconds); Stopwatch.GetElapsedTime(started).TotalMilliseconds,
receivedBytes,
result == NatMediationResult.Introduced ? 2 * ContractLimits.UdpDatagramMaxBytes : 0);
return result; return result;
} }
@@ -201,7 +201,7 @@ internal sealed class UdpMediatorService : BackgroundService
&& token is not null) && token is not null)
{ {
_ = processor.ProcessRequestAfterIngress( _ = processor.ProcessRequestAfterIngress(
claimedLocalEndpoint, endPoint, token, sink); claimedLocalEndpoint, endPoint, token, sink, length);
} }
// Every inbound packet is consumed here. NatPunchModule is used only for outbound introductions. // Every inbound packet is consumed here. NatPunchModule is used only for outbound introductions.
@@ -0,0 +1,135 @@
import assert from "node:assert/strict";
import { readFile } from "node:fs/promises";
import test from "node:test";
import {
SessionProjection,
buildBrowseUrl,
buildStreamUrl,
chooseSnapshotTransport,
safeText,
} from "../../src/FinalFactory.Rendezvous.Server/Diagnostics/Assets/app.mjs";
const cursor = "rvs1.test-cursor";
function session(id, name = `Host ${id}`) {
return {
contractVersion: 1,
listingId: `00000000-0000-0000-0000-${String(id).padStart(12, "0")}`,
gameId: "space-game",
environmentId: "smoke",
regionId: "local",
protocolVersion: 1,
buildVersion: "1.0.0",
displayName: name,
visibility: "public",
publisherTrustMode: "managedDedicated",
capacity: { currentPlayers: 1, maximumPlayers: 8 },
metadata: { mode: "online-coop" },
};
}
function event(kind, values = {}) {
return {
contractVersion: 1,
kind,
cursor: values.cursor ?? cursor,
session: values.session ?? null,
listingId: values.listingId ?? null,
};
}
test("snapshot and ordered deltas match the final public projection", () => {
const projection = new SessionProjection(10);
assert.equal(projection.replace([session(1)], cursor), "applied");
assert.equal(projection.apply(event("sessionUpsert", {
cursor: `${cursor}-2`,
session: session(2),
})), "applied");
assert.equal(projection.apply(event("sessionUpsert", {
cursor: `${cursor}-3`,
session: session(1, "Updated host"),
})), "applied");
assert.equal(projection.apply(event("sessionRemove", {
cursor: `${cursor}-4`,
listingId: session(2).listingId,
})), "applied");
assert.deepEqual(
projection.sessions().map((entry) => entry.session.displayName),
["Updated host"],
);
assert.equal(projection.cursor, `${cursor}-4`);
});
test("reset, malformed events, and a partial snapshot fail closed", () => {
const projection = new SessionProjection(2);
assert.equal(projection.replace([session(1)], cursor, true), "applied");
assert.equal(projection.apply(event("sessionRemove", {
listingId: session(1).listingId,
})), "refresh");
assert.equal(projection.apply(event("reset")), "reset");
assert.equal(projection.apply({ kind: "sessionUpsert" }), "invalid");
assert.equal(projection.replace([session(1), session(2), session(3)], cursor), "overflow");
});
test("bursts coalesce by listing identity and memory stays bounded", () => {
const projection = new SessionProjection(2);
assert.equal(projection.replace([], cursor), "applied");
for (let index = 0; index < 1_000; index += 1) {
assert.equal(projection.apply(event("sessionUpsert", {
cursor: `${cursor}-${index}`,
session: session(1, `Host ${index}`),
})), "applied");
}
assert.equal(projection.sessions().length, 1);
assert.equal(projection.sessions()[0].session.displayName, "Host 999");
assert.equal(projection.apply(event("sessionUpsert", { session: session(2) })), "applied");
assert.equal(projection.apply(event("sessionUpsert", { session: session(3) })), "overflow");
assert.equal(projection.sessions().length, 2);
});
test("requests are fixed same-origin paths with an exact configured filter", () => {
const filter = {
gameId: "space-game",
environmentId: "smoke",
protocolVersion: 1,
regionId: "local",
excludeFull: true,
};
const browse = buildBrowseUrl(filter);
const stream = buildStreamUrl(filter, cursor);
assert.match(browse, /^\/v1\/sessions\?/u);
assert.match(stream, /^\/v1\/sessions\/stream\?/u);
assert.match(browse, /gameId=space-game/u);
assert.match(browse, /environmentId=smoke/u);
assert.match(stream, /streamCursor=rvs1.test-cursor/u);
assert.doesNotMatch(browse, /https?:/u);
assert.doesNotMatch(stream, /https?:/u);
});
test("snapshot transport preserves deliberate polling and bounds partial snapshots", () => {
assert.equal(chooseSnapshotTransport(false, false), "stream");
assert.equal(chooseSnapshotTransport(true, false), "boundedPolling");
assert.equal(chooseSnapshotTransport(false, true), "pollingOnly");
assert.equal(chooseSnapshotTransport(true, true), "pollingOnly");
});
test("hostile display data remains literal text and no unsafe DOM sink exists", async () => {
const payload = `<img src=x onerror=alert(1)><style>body{display:none}</style><a href=//evil>go</a>`;
assert.equal(safeText(payload, 200), payload);
const source = await readFile(new URL(
"../../src/FinalFactory.Rendezvous.Server/Diagnostics/Assets/app.mjs",
import.meta.url,
), "utf8");
for (const forbidden of [
"innerHTML",
"outerHTML",
"insertAdjacentHTML",
"document.write",
"eval(",
"new Function",
"window.location",
]) {
assert.equal(source.includes(forbidden), false, `unsafe browser sink: ${forbidden}`);
}
});
@@ -147,7 +147,12 @@ public sealed class RendezvousClientIntegrationTests
{ {
await using ClientTestHost host = await ClientTestHost.StartAsync(); await using ClientTestHost host = await ClientTestHost.StartAsync();
RendezvousPublisherClient publisher = new(host.HttpClient); RendezvousPublisherClient publisher = new(host.HttpClient);
RendezvousSessionBrowserClient browser = new(host.HttpClient); RendezvousSessionBrowserClient browser = new(
host.HttpClient,
new RendezvousClientOptions
{
RequestTimeout = TimeSpan.FromSeconds(15),
});
PublishedSession session = AssertSuccess(await publisher.RegisterAsync( PublishedSession session = AssertSuccess(await publisher.RegisterAsync(
CreateRegistration(200), CreateRegistration(200),
host.PublisherCredential)); host.PublisherCredential));
@@ -156,7 +161,7 @@ public sealed class RendezvousClientIntegrationTests
BrowseSessionsResponse snapshot = AssertSuccess(await browser.BrowseAsync(request)); BrowseSessionsResponse snapshot = AssertSuccess(await browser.BrowseAsync(request));
Assert.False(string.IsNullOrWhiteSpace(snapshot.StreamCursor)); Assert.False(string.IsNullOrWhiteSpace(snapshot.StreamCursor));
using CancellationTokenSource timeout = new(TimeSpan.FromSeconds(10)); using CancellationTokenSource timeout = new(TimeSpan.FromSeconds(30));
await using (IAsyncEnumerator<RendezvousClientResult<SessionStreamEvent>> invalid = browser await using (IAsyncEnumerator<RendezvousClientResult<SessionStreamEvent>> invalid = browser
.StreamAsync(request, CorruptCursor(snapshot.StreamCursor), timeout.Token) .StreamAsync(request, CorruptCursor(snapshot.StreamCursor), timeout.Token)
.GetAsyncEnumerator(timeout.Token)) .GetAsyncEnumerator(timeout.Token))
@@ -0,0 +1,205 @@
using System.Net;
using System.Net.Sockets;
using System.Text.Json;
using FinalFactory.Rendezvous.Server.Diagnostics;
using Microsoft.AspNetCore.Builder;
using Microsoft.AspNetCore.Hosting;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Options;
namespace FinalFactory.Rendezvous.Tests.Diagnostics;
public sealed class DiagnosticDashboardTests
{
[Fact]
public void ConfigurationRequiresBoundedUniqueAllowListedScopes()
{
Assert.Empty(ValidOptions().Validate());
Assert.Empty(new DiagnosticDashboardOptions().Validate());
DiagnosticDashboardOptions invalid = ValidOptions() with
{
PollIntervalSeconds = 1,
MaximumRenderedSessions = 501,
Scopes =
[
new DiagnosticDashboardScope
{
GameId = "INVALID",
EnvironmentId = "smoke",
ProtocolVersions = [0, 0],
Regions = ["INVALID", "INVALID"],
},
new DiagnosticDashboardScope
{
GameId = "INVALID",
EnvironmentId = "smoke",
ProtocolVersions = [1],
Regions = ["local"],
},
],
};
IReadOnlyList<string> errors = invalid.Validate();
Assert.Contains(errors, error => error.Contains("PollIntervalSeconds", StringComparison.Ordinal));
Assert.Contains(errors, error => error.Contains("MaximumRenderedSessions", StringComparison.Ordinal));
Assert.Contains(errors, error => error.Contains("invalid game", StringComparison.Ordinal));
Assert.Contains(errors, error => error.Contains("protocol versions", StringComparison.Ordinal));
Assert.Contains(errors, error => error.Contains("regions", StringComparison.Ordinal));
Assert.Contains(errors, error => error.Contains("duplicate", StringComparison.Ordinal));
DiagnosticDashboardOptions nullBound = ValidOptions() with
{
Scopes = null!,
};
Assert.Contains(
nullBound.Validate(),
error => error.Contains("Scopes must contain", StringComparison.Ordinal));
DiagnosticDashboardOptions nullScope = ValidOptions() with
{
Scopes = [null!],
};
Assert.Contains(
nullScope.Validate(),
error => error.Contains("null entries", StringComparison.Ordinal));
DiagnosticDashboardOptions nullCollections = ValidOptions() with
{
Scopes =
[
new DiagnosticDashboardScope
{
GameId = null!,
EnvironmentId = null!,
ProtocolVersions = null!,
Regions = null!,
},
],
};
Assert.True(nullCollections.Validate().Count >= 3);
}
[Fact]
public async Task DisabledDashboardHasNoPublicAssetOrConfigurationSurface()
{
await using DashboardHost host = await DashboardHost.StartAsync(new());
Assert.Equal(HttpStatusCode.NotFound, (await host.Client.GetAsync("diagnostics/")).StatusCode);
Assert.Equal(HttpStatusCode.NotFound, (await host.Client.GetAsync("diagnostics/app.mjs")).StatusCode);
Assert.Equal(HttpStatusCode.NotFound, (await host.Client.GetAsync("diagnostics/config.json")).StatusCode);
}
[Fact]
public async Task EnabledDashboardServesOnlyHardenedSameOriginReadOnlyAssets()
{
await using DashboardHost host = await DashboardHost.StartAsync(ValidOptions());
using HttpRequestMessage request = new(HttpMethod.Get, "diagnostics/");
request.Headers.Add("Origin", "https://hostile.example");
using HttpResponseMessage response = await host.Client.SendAsync(request);
string html = await response.Content.ReadAsStringAsync();
Assert.True(response.IsSuccessStatusCode, html);
Assert.Equal("text/html", response.Content.Headers.ContentType?.MediaType);
Assert.Equal("DENY", Header(response, "X-Frame-Options"));
Assert.Equal("nosniff", Header(response, "X-Content-Type-Options"));
Assert.Equal("no-referrer", Header(response, "Referrer-Policy"));
Assert.Contains("default-src 'none'", Header(response, "Content-Security-Policy"), StringComparison.Ordinal);
Assert.Contains("connect-src 'self'", Header(response, "Content-Security-Policy"), StringComparison.Ordinal);
Assert.Contains("frame-ancestors 'none'", Header(response, "Content-Security-Policy"), StringComparison.Ordinal);
Assert.False(response.Headers.Contains("Access-Control-Allow-Origin"));
Assert.Contains("<html lang=\"en\">", html, StringComparison.Ordinal);
Assert.Contains("href=\"#main-content\"", html, StringComparison.Ordinal);
Assert.Contains("aria-live=\"polite\"", html, StringComparison.Ordinal);
Assert.Contains("type=\"module\" src=\"/diagnostics/app.mjs\"", html, StringComparison.Ordinal);
Assert.DoesNotContain("<script src=\"http", html, StringComparison.OrdinalIgnoreCase);
Assert.DoesNotContain("/v1/operator", html, StringComparison.OrdinalIgnoreCase);
using HttpResponseMessage scriptResponse = await host.Client.GetAsync("diagnostics/app.mjs");
string script = await scriptResponse.Content.ReadAsStringAsync();
Assert.Equal("text/javascript", scriptResponse.Content.Headers.ContentType?.MediaType);
Assert.DoesNotContain("/v1/operator", script, StringComparison.OrdinalIgnoreCase);
Assert.DoesNotContain("Authorization", script, StringComparison.OrdinalIgnoreCase);
Assert.DoesNotContain("innerHTML", script, StringComparison.Ordinal);
}
[Fact]
public async Task BrowserConfigurationContainsOnlyTheExplicitPublicAllowList()
{
await using DashboardHost host = await DashboardHost.StartAsync(ValidOptions());
using HttpResponseMessage response = await host.Client.GetAsync("diagnostics/config.json");
using JsonDocument document = JsonDocument.Parse(await response.Content.ReadAsStringAsync());
JsonElement root = document.RootElement;
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
Assert.Equal(1, root.GetProperty("contractVersion").GetInt32());
Assert.Equal(10, root.GetProperty("pollIntervalSeconds").GetInt32());
JsonElement scope = Assert.Single(root.GetProperty("scopes").EnumerateArray());
Assert.Equal("space-game", scope.GetProperty("gameId").GetString());
Assert.Equal("smoke", scope.GetProperty("environmentId").GetString());
Assert.Equal("public", scope.GetProperty("visibility").GetString());
string json = root.GetRawText();
Assert.DoesNotContain("http", json, StringComparison.OrdinalIgnoreCase);
Assert.DoesNotContain("credential", json, StringComparison.OrdinalIgnoreCase);
Assert.DoesNotContain("token", json, StringComparison.OrdinalIgnoreCase);
Assert.DoesNotContain("endpoint", json, StringComparison.OrdinalIgnoreCase);
}
private static string Header(HttpResponseMessage response, string name) =>
Assert.Single(response.Headers.GetValues(name));
private static DiagnosticDashboardOptions ValidOptions() => new()
{
Enabled = true,
PollIntervalSeconds = 10,
MaximumRenderedSessions = 100,
Scopes =
[
new DiagnosticDashboardScope
{
GameId = "space-game",
EnvironmentId = "smoke",
ProtocolVersions = [1, 2],
Regions = ["local"],
},
],
};
private sealed class DashboardHost : IAsyncDisposable
{
private readonly WebApplication _application;
private DashboardHost(WebApplication application, HttpClient client)
{
_application = application;
Client = client;
}
public HttpClient Client { get; }
public static async Task<DashboardHost> StartAsync(DiagnosticDashboardOptions options)
{
int port = ReserveTcpPort();
string address = $"http://127.0.0.1:{port}";
WebApplicationBuilder builder = WebApplication.CreateBuilder();
builder.WebHost.UseUrls(address);
builder.Services.AddSingleton(Options.Create(options));
WebApplication application = builder.Build();
application.MapDiagnosticDashboardEndpoints();
await application.StartAsync();
return new(application, new HttpClient { BaseAddress = new Uri(address) });
}
public async ValueTask DisposeAsync()
{
Client.Dispose();
await _application.StopAsync();
await _application.DisposeAsync();
}
private static int ReserveTcpPort()
{
using TcpListener listener = new(IPAddress.Loopback, 0);
listener.Start();
return ((IPEndPoint)listener.LocalEndpoint).Port;
}
}
}
@@ -61,6 +61,10 @@ public sealed partial class DocumentationContractTests
Assert.Contains("metadata.st_mode & 0o077", helper, StringComparison.Ordinal); Assert.Contains("metadata.st_mode & 0o077", helper, StringComparison.Ordinal);
Assert.Contains("metadata.st_nlink != 1", helper, StringComparison.Ordinal); Assert.Contains("metadata.st_nlink != 1", helper, StringComparison.Ordinal);
Assert.Contains("mint-local-publisher-credential.sh", smoke, StringComparison.Ordinal); Assert.Contains("mint-local-publisher-credential.sh", smoke, StringComparison.Ordinal);
Assert.Contains("select(.event == \"host.ready\" and .status == \"ready\")", smoke, StringComparison.Ordinal);
Assert.Contains("hasListingId", smoke, StringComparison.Ordinal);
Assert.Contains("jq -s -e", smoke, StringComparison.Ordinal);
Assert.DoesNotContain("jq -c .", smoke, StringComparison.Ordinal);
Assert.DoesNotContain("hexkey:", smoke, StringComparison.Ordinal); Assert.DoesNotContain("hexkey:", smoke, StringComparison.Ordinal);
Assert.DoesNotContain("openssl dgst", smoke, StringComparison.Ordinal); Assert.DoesNotContain("openssl dgst", smoke, StringComparison.Ordinal);
} }
@@ -0,0 +1,165 @@
using System.Net;
using System.Net.Sockets;
using System.Text;
using FinalFactory.Rendezvous.Server.Browser;
using FinalFactory.Rendezvous.Server.Observability;
using FinalFactory.Rendezvous.Server.Provisioning;
using FinalFactory.Rendezvous.Server.State;
using FinalFactory.Rendezvous.Tests.State;
using Microsoft.AspNetCore.Builder;
using Microsoft.AspNetCore.Hosting;
using Microsoft.Extensions.DependencyInjection;
namespace FinalFactory.Rendezvous.Tests.Observability;
[Collection(RendezvousTelemetryIsolation.Name)]
public sealed class PrometheusMetricsTests
{
private const string Token = "0123456789abcdef0123456789abcdef";
[Fact]
public void EnabledExporterRequiresAnExternalBoundedSecretReference()
{
Assert.Empty(new PrometheusMetricsOptions().Validate());
Assert.Empty(new PrometheusMetricsOptions
{
Enabled = true,
BearerTokenSecretReference = "file:/run/secrets/rendezvous-metrics-token",
}.Validate());
Assert.NotEmpty(new PrometheusMetricsOptions { Enabled = true }.Validate());
Assert.NotEmpty(new PrometheusMetricsOptions
{
Enabled = true,
BearerTokenSecretReference = "literal-secret",
}.Validate());
Assert.NotEmpty(new PrometheusMetricsOptions
{
Enabled = true,
BearerTokenSecretReference = "file:relative-token",
}.Validate());
Assert.NotEmpty(new PrometheusMetricsOptions
{
Enabled = true,
BearerTokenSecretReference = null!,
}.Validate());
}
[Fact]
public async Task ExporterIsDisabledByDefaultAndConcealsRejectedAuthentication()
{
await using MetricsHost disabled = await MetricsHost.StartAsync(
new PrometheusMetricsOptions(),
credential: null);
Assert.Equal(HttpStatusCode.NotFound, (await disabled.Client.GetAsync("metrics")).StatusCode);
PrometheusMetricsOptions options = EnabledOptions();
using MetricsAccessCredential credential = Credential(options);
await using MetricsHost enabled = await MetricsHost.StartAsync(options, credential);
Assert.Equal(HttpStatusCode.NotFound, (await enabled.Client.GetAsync("metrics")).StatusCode);
using HttpRequestMessage wrong = new(HttpMethod.Get, "metrics");
wrong.Headers.Authorization = new("Bearer", new string('x', 32));
Assert.Equal(HttpStatusCode.NotFound, (await enabled.Client.SendAsync(wrong)).StatusCode);
}
[Fact]
public async Task AuthenticatedExporterReturnsBoundedPrivacySafePrometheusFamilies()
{
PrometheusMetricsOptions options = EnabledOptions();
using MetricsAccessCredential credential = Credential(options);
await using MetricsHost host = await MetricsHost.StartAsync(options, credential);
host.Telemetry.RecordHttp("BrowseSessions", 200, 3.5);
host.Telemetry.RecordHttp("listing-id/canary", 500, 4.5);
host.Telemetry.RecordUdp("frozen", "Introduced", 1.25, 128, 2400);
host.Telemetry.RecordLimiterDrop("udp", "rate-or-concurrency");
host.Telemetry.RecordConnectionOutcome("Connected", "UnderOneSecond");
host.Telemetry.RecordPairingLatency(12.5);
using HttpRequestMessage request = new(HttpMethod.Get, "metrics");
request.Headers.Authorization = new("Bearer", Token);
using HttpResponseMessage response = await host.Client.SendAsync(request);
string body = await response.Content.ReadAsStringAsync();
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
Assert.Equal("text/plain", response.Content.Headers.ContentType?.MediaType);
Assert.Equal("no-store", response.Headers.CacheControl?.ToString());
Assert.Contains("rendezvous_http_requests_total{operation=\"BrowseSessions\",status_code=\"200\"} 1", body, StringComparison.Ordinal);
Assert.Contains("rendezvous_http_duration_milliseconds_bucket", body, StringComparison.Ordinal);
Assert.Contains("rendezvous_udp_received_bytes_total{operation=\"frozen\"} 128", body, StringComparison.Ordinal);
Assert.Contains("rendezvous_udp_response_budget_bytes_total{operation=\"frozen\"} 2400", body, StringComparison.Ordinal);
Assert.Contains("rendezvous_store_active_listings", body, StringComparison.Ordinal);
Assert.Contains("rendezvous_browser_sse_subscribers", body, StringComparison.Ordinal);
Assert.Contains("process_resident_memory_bytes", body, StringComparison.Ordinal);
Assert.Contains("dotnet_gc_heap_size_bytes", body, StringComparison.Ordinal);
Assert.Contains("operation=\"other\"", body, StringComparison.Ordinal);
Assert.DoesNotContain("listing-id", body, StringComparison.OrdinalIgnoreCase);
Assert.DoesNotContain(Token, body, StringComparison.Ordinal);
}
private static PrometheusMetricsOptions EnabledOptions() => new()
{
Enabled = true,
BearerTokenSecretReference = "file:/metrics-token",
};
private static MetricsAccessCredential Credential(PrometheusMetricsOptions options)
{
using DictionarySecretProvider secrets = new(new Dictionary<string, byte[]>
{
[options.BearerTokenSecretReference] = Encoding.ASCII.GetBytes(Token + "\n"),
});
Assert.True(MetricsAccessCredential.TryCreate(options, secrets, out MetricsAccessCredential? credential));
return Assert.IsType<MetricsAccessCredential>(credential);
}
private sealed class MetricsHost : IAsyncDisposable
{
private readonly WebApplication _application;
private MetricsHost(
WebApplication application,
HttpClient client,
RendezvousTelemetry telemetry)
{
_application = application;
Client = client;
Telemetry = telemetry;
}
public HttpClient Client { get; }
public RendezvousTelemetry Telemetry { get; }
public static async Task<MetricsHost> StartAsync(
PrometheusMetricsOptions options,
MetricsAccessCredential? credential)
{
ManualRendezvousClock clock = new();
SessionChangeJournal changes = new(new SessionChangeJournalOptions());
InMemoryEphemeralRendezvousStore store = new(new(), clock, clock, changes);
RendezvousTelemetry telemetry = new(store, changes);
int port = ReserveTcpPort();
string address = $"http://127.0.0.1:{port}";
WebApplicationBuilder builder = WebApplication.CreateBuilder();
builder.WebHost.UseUrls(address);
builder.Services.AddSingleton(telemetry);
WebApplication application = builder.Build();
application.MapPrometheusMetricsEndpoint(options, credential);
await application.StartAsync();
return new(application, new HttpClient { BaseAddress = new Uri(address) }, telemetry);
}
public async ValueTask DisposeAsync()
{
Client.Dispose();
await _application.StopAsync();
await _application.DisposeAsync();
// This test supplies an existing singleton instance, so the host does not own it.
Telemetry.Dispose();
}
private static int ReserveTcpPort()
{
using TcpListener listener = new(IPAddress.Loopback, 0);
listener.Start();
return ((IPEndPoint)listener.LocalEndpoint).Port;
}
}
}
@@ -149,6 +149,68 @@ public sealed class ReleaseCompatibilityTests
Assert.Contains("<PackageReference Remove=\"FinalFactory.Rendezvous.Contracts\" />", realConsumerGate, StringComparison.Ordinal); Assert.Contains("<PackageReference Remove=\"FinalFactory.Rendezvous.Contracts\" />", realConsumerGate, StringComparison.Ordinal);
} }
[Fact]
public void AutomationProvidesNet8RuntimeForProcessVerification()
{
string root = FindRepositoryRoot();
string qualityWorkflow = File.ReadAllText(Path.Combine(root, ".gitea/workflows/ci.yml"));
Assert.Equal(2, qualityWorkflow.Split("8.0.128", StringSplitOptions.None).Length - 1);
string releaseWorkflow = File.ReadAllText(Path.Combine(root, ".gitea/workflows/release.yml"));
Assert.Contains("8.0.128", releaseWorkflow, StringComparison.Ordinal);
string releaseBuilder = File.ReadAllText(Path.Combine(root, "eng/release-builder.Dockerfile"));
Assert.Contains("runtime:8.0.28-noble@sha256:", releaseBuilder, StringComparison.Ordinal);
Assert.Contains("Microsoft.NETCore.App/8.0.28", releaseBuilder, StringComparison.Ordinal);
}
[Fact]
public void AutomationUsesPerRunContainerPorts()
{
string root = FindRepositoryRoot();
string compose = File.ReadAllText(Path.Combine(root, "deploy/compose/compose.yaml"));
Assert.Contains("${RENDEZVOUS_HTTP_HOST_PORT:-8080}:8080/tcp", compose, StringComparison.Ordinal);
Assert.Contains("${RENDEZVOUS_UDP_HOST_PORT:-9050}:9050/udp", compose, StringComparison.Ordinal);
Assert.Contains("${RENDEZVOUS_CONFIG_SOURCE:-./appsettings.Production.json}", compose, StringComparison.Ordinal);
Assert.Contains("${RENDEZVOUS_SECRET_SOURCE:-./secrets/signing-key}", compose, StringComparison.Ordinal);
foreach (string workflowName in new[] { "ci.yml", "release.yml" })
{
string workflow = File.ReadAllText(Path.Combine(root, ".gitea/workflows", workflowName));
Assert.Contains("GITHUB_RUN_ID", workflow, StringComparison.Ordinal);
Assert.Contains("--network \"container:${container_id}\"", workflow, StringComparison.Ordinal);
Assert.Contains("publisher_credential", workflow, StringComparison.Ordinal);
Assert.Contains("RENDEZVOUS_PUBLISHER_CREDENTIAL", workflow, StringComparison.Ordinal);
Assert.Contains("for attempt in {1..180}", workflow, StringComparison.Ordinal);
Assert.Contains("sleep 1", workflow, StringComparison.Ordinal);
Assert.Contains("http://127.0.0.1:8080/", workflow, StringComparison.Ordinal);
Assert.Contains("127.0.0.1:9050", workflow, StringComparison.Ordinal);
Assert.Contains("RENDEZVOUS_HTTP_HOST_PORT", workflow, StringComparison.Ordinal);
Assert.Contains("RENDEZVOUS_UDP_HOST_PORT", workflow, StringComparison.Ordinal);
Assert.Contains("RENDEZVOUS_SMOKE_HTTP_URL", workflow, StringComparison.Ordinal);
Assert.Contains("RENDEZVOUS_SMOKE_UDP_ENDPOINT", workflow, StringComparison.Ordinal);
Assert.Contains("runner_workspace_source", workflow, StringComparison.Ordinal);
Assert.Contains("RENDEZVOUS_CONFIG_SOURCE", workflow, StringComparison.Ordinal);
Assert.Contains("RENDEZVOUS_SECRET_SOURCE", workflow, StringComparison.Ordinal);
}
}
[Fact]
public void ContainerGateUsesValidInspectTemplates()
{
string root = FindRepositoryRoot();
string workflow = File.ReadAllText(Path.Combine(root, ".gitea/workflows/ci.yml"));
Assert.Contains(
"eq .Destination \"/app/appsettings.Production.json\"",
workflow,
StringComparison.Ordinal);
Assert.Contains(
"eq .Destination \"/run/secrets/rendezvous-signing-key\"",
workflow,
StringComparison.Ordinal);
Assert.DoesNotContain("eq .Destination \\\"", workflow, StringComparison.Ordinal);
}
[Fact] [Fact]
public void ServerSourceManifestIsCompleteSortedAndDeterministic() public void ServerSourceManifestIsCompleteSortedAndDeterministic()
{ {
@@ -0,0 +1,89 @@
import assert from "node:assert/strict";
import { readFile } from "node:fs/promises";
import test from "node:test";
const root = new URL("../../", import.meta.url);
const dashboardPath = new URL(
"deploy/observability/grafana/dashboards/rendezvous-overview.json",
root,
);
test("dashboard is provisionable, broad, and uses privacy-safe bounded series", async () => {
const dashboard = JSON.parse(await readFile(dashboardPath, "utf8"));
assert.equal(dashboard.uid, "rendezvous-overview");
assert.equal(dashboard.editable, false);
assert.ok(dashboard.panels.length >= 20);
assert.equal(new Set(dashboard.panels.map((panel) => panel.id)).size, dashboard.panels.length);
assert.equal(new Set(dashboard.panels.map((panel) => panel.title)).size, dashboard.panels.length);
const expressions = dashboard.panels
.flatMap((panel) => panel.targets ?? [])
.map((target) => target.expr ?? "")
.join("\n");
for (const metric of [
"rendezvous_http_requests_total",
"rendezvous_http_duration_milliseconds_bucket",
"rendezvous_udp_results_total",
"rendezvous_udp_received_bytes_total",
"rendezvous_limiter_drops_total",
"rendezvous_connection_outcomes_total",
"rendezvous_browser_sse_subscribers",
"rendezvous_store_active_listings",
"rendezvous_store_active_attempts",
"rendezvous_signing_keys",
"process_resident_memory_bytes",
"process_open_file_descriptors",
"dotnet_gc_heap_size_bytes",
]) {
assert.match(expressions, new RegExp(`\\b${metric}\\b`));
}
assert.doesNotMatch(
expressions,
/listing_?id|session_?id|player|subject|token|capability|endpoint|address|metadata|credential/i,
);
for (const panel of dashboard.panels) {
assert.ok(panel.title?.trim());
assert.ok(panel.gridPos?.w > 0 && panel.gridPos?.h > 0);
for (const target of panel.targets ?? []) {
assert.equal(target.editorMode, "code");
assert.ok(target.expr?.trim());
}
}
});
test("Compose overlay pins hardened services and keeps telemetry private", async () => {
const compose = await readFile(new URL("deploy/observability/compose.yaml", root), "utf8");
assert.match(compose, /prom\/prometheus:v3\.13\.1@sha256:[a-f0-9]{64}/);
assert.match(compose, /grafana\/grafana:13\.1\.0@sha256:[a-f0-9]{64}/);
assert.match(compose, /Rendezvous__Metrics__Enabled: "true"/);
assert.match(compose, /rendezvous-metrics-token:\/run\/secrets\/rendezvous-metrics-token:ro/);
assert.match(compose, /GF_AUTH_ANONYMOUS_ENABLED: "false"/);
assert.match(compose, /GF_PLUGINS_PREINSTALL_DISABLED: "true"/);
assert.match(compose, /"127\.0\.0\.1:3000:3000\/tcp"/);
assert.doesNotMatch(compose, /9090:9090/);
assert.ok((compose.match(/read_only: true/g) ?? []).length >= 2);
assert.ok((compose.match(/no-new-privileges:true/g) ?? []).length >= 2);
});
test("Prometheus and Grafana provisioning use server-side authenticated access", async () => {
const prometheus = await readFile(
new URL("deploy/observability/prometheus/prometheus.yml", root),
"utf8",
);
const datasource = await readFile(
new URL("deploy/observability/grafana/provisioning/datasources/prometheus.yaml", root),
"utf8",
);
const provider = await readFile(
new URL("deploy/observability/grafana/provisioning/dashboards/rendezvous.yaml", root),
"utf8",
);
assert.match(prometheus, /bearer_token_file: \/run\/secrets\/rendezvous-metrics-token/);
assert.match(prometheus, /- rendezvous:8080/);
assert.match(datasource, /uid: rendezvous-prometheus/);
assert.match(datasource, /access: proxy/);
assert.match(datasource, /url: http:\/\/prometheus:9090/);
assert.match(provider, /allowUiUpdates: false/);
assert.match(provider, /path: \/var\/lib\/grafana\/dashboards/);
});