diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index 86c6ad3..e80f9fa 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -45,6 +45,12 @@ jobs: - name: Verify formatting and analyzers run: dotnet format Rendezvous.slnx --verify-no-changes --no-restore + - name: Test dependency-free diagnostic dashboard + run: ./scripts/test-diagnostic-dashboard.sh + + - name: Test observability dashboard provisioning + run: ./scripts/test-observability-assets.sh + - name: Build run: dotnet build Rendezvous.slnx --configuration Release --no-restore diff --git a/README.md b/README.md index b361ca7..6596c24 100644 --- a/README.md +++ b/README.md @@ -84,10 +84,10 @@ directory leases, authenticated join attempts, LiteNetLib mediator, caller-owned SDK coordination, typed connection outcomes, thin public-SDK diagnostic client, deterministic NAT topology harness, hostile-input controls, observability/operator surface, secure single-active Linux deployment, and -numeric capacity/resilience gates, and reproducible signed release pipeline are -implemented. Consumer pilots and final production-readiness gates remain in progress; -participating games must not treat the current repository as a finished production -service until those gates land. +numeric capacity/resilience gates, reproducible signed release pipeline, consumer +pilot integrations, and production-readiness decision framework are implemented. +Deployment-specific external canaries and optional roadmap follow-ups remain; +participating games must not treat a checkout alone as production approval. The ratified v1 boundaries, trust decisions, privacy rules, safety budgets, and threat model are indexed in [the architecture documentation](docs/architecture/README.md). @@ -100,6 +100,8 @@ defined in [hostile-input and overload protection](docs/security/abuse-protectio Health semantics, bounded telemetry, alerting, audit privacy, and the authenticated operator controls are defined in the [observability and operator runbook](docs/operations/observability-and-operator-runbook.md). +The optional public session diagnostic and the private provisioned Grafana stack +are described in [diagnostic dashboards](docs/operations/diagnostic-dashboards.md). Concrete detect/contain/recover/verify procedures are in the [incident and change runbooks](docs/operations/incident-runbooks.md). The pinned non-root container, production topology, graceful drain, Linux diff --git a/deploy/compose/appsettings.Production.json b/deploy/compose/appsettings.Production.json index 2c0dc0e..c8219f7 100644 --- a/deploy/compose/appsettings.Production.json +++ b/deploy/compose/appsettings.Production.json @@ -14,6 +14,29 @@ "ListenAddress": "0.0.0.0", "Port": 9050 }, + "Diagnostics": { + "Enabled": false, + "PollIntervalSeconds": 10, + "MaximumRenderedSessions": 100, + "Scopes": [ + { + "GameId": "space-game", + "EnvironmentId": "smoke", + "ProtocolVersions": [1, 2], + "Regions": ["local"] + }, + { + "GameId": "unscouted", + "EnvironmentId": "smoke", + "ProtocolVersions": [1], + "Regions": ["local"] + } + ] + }, + "Metrics": { + "Enabled": false, + "BearerTokenSecretReference": "file:/run/secrets/rendezvous-metrics-token" + }, "AbuseProtection": { "TrustedProxyAddresses": ["127.0.0.1"], "OperatorAllowedAddresses": ["127.0.0.1"] diff --git a/deploy/compose/compose.yaml b/deploy/compose/compose.yaml index be81af3..10947e0 100644 --- a/deploy/compose/compose.yaml +++ b/deploy/compose/compose.yaml @@ -27,6 +27,8 @@ services: environment: ASPNETCORE_ENVIRONMENT: Production ASPNETCORE_HTTP_PORTS: "8080" + Rendezvous__Diagnostics__Enabled: "${RENDEZVOUS_DIAGNOSTICS_ENABLED:-false}" + Rendezvous__Metrics__Enabled: "${RENDEZVOUS_METRICS_ENABLED:-false}" volumes: - ./appsettings.Production.json:/app/appsettings.Production.json:ro - ./secrets/signing-key:/run/secrets/rendezvous-signing-key:ro diff --git a/deploy/observability/compose.yaml b/deploy/observability/compose.yaml new file mode 100644 index 0000000..4e741cb --- /dev/null +++ b/deploy/observability/compose.yaml @@ -0,0 +1,71 @@ +services: + rendezvous: + environment: + Rendezvous__Metrics__Enabled: "true" + volumes: + - ../observability/secrets/rendezvous-metrics-token:/run/secrets/rendezvous-metrics-token:ro + + prometheus: + image: prom/prometheus:v3.13.1@sha256:3c42b892cf723fa54d2f262c37a0e1f80aa8c8ddb1da7b9b0df9455a35a7f893 + user: "65534:65534" + read_only: true + init: true + cap_drop: + - ALL + security_opt: + - no-new-privileges:true + pids_limit: 128 + mem_limit: 512m + cpus: 1.0 + restart: unless-stopped + command: + - --config.file=/etc/prometheus/prometheus.yml + - --storage.tsdb.path=/prometheus + - --storage.tsdb.retention.time=15d + volumes: + - ../observability/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro + - ../observability/secrets/rendezvous-metrics-token:/run/secrets/rendezvous-metrics-token:ro + - prometheus-data:/prometheus + depends_on: + - rendezvous + + grafana: + image: grafana/grafana:13.1.0@sha256:121a7a9ece6dc10b969f1f96eed64b4f07dfac0d0b8abc070f7cb83bbde86f63 + user: "472:472" + read_only: true + init: true + cap_drop: + - ALL + security_opt: + - no-new-privileges:true + pids_limit: 128 + mem_limit: 512m + cpus: 1.0 + restart: unless-stopped + environment: + GF_ANALYTICS_REPORTING_ENABLED: "false" + GF_ANALYTICS_CHECK_FOR_UPDATES: "false" + GF_PLUGINS_PREINSTALL_DISABLED: "true" + GF_PLUGINS_PREINSTALL_AUTO_UPDATE: "false" + GF_SECURITY_ADMIN_USER: "rendezvous-admin" + GF_SECURITY_ADMIN_PASSWORD__FILE: /run/secrets/grafana-admin-password + GF_USERS_ALLOW_SIGN_UP: "false" + GF_AUTH_ANONYMOUS_ENABLED: "false" + GF_SERVER_DOMAIN: localhost + GF_SERVER_ROOT_URL: http://localhost:3000 + tmpfs: + - /tmp:rw,noexec,nosuid,nodev,size=32m,uid=472,gid=472,mode=0700 + volumes: + - ../observability/grafana/provisioning/datasources:/etc/grafana/provisioning/datasources:ro + - ../observability/grafana/provisioning/dashboards:/etc/grafana/provisioning/dashboards:ro + - ../observability/grafana/dashboards:/var/lib/grafana/dashboards:ro + - ../observability/secrets/grafana-admin-password:/run/secrets/grafana-admin-password:ro + - grafana-data:/var/lib/grafana + ports: + - "127.0.0.1:3000:3000/tcp" + depends_on: + - prometheus + +volumes: + prometheus-data: + grafana-data: diff --git a/deploy/observability/grafana/dashboards/rendezvous-overview.json b/deploy/observability/grafana/dashboards/rendezvous-overview.json new file mode 100644 index 0000000..d2a0918 --- /dev/null +++ b/deploy/observability/grafana/dashboards/rendezvous-overview.json @@ -0,0 +1,292 @@ +{ + "annotations": {"list": []}, + "description": "Privacy-safe operational view of the single-active Rendezvous service. Capacity percentages use the approved 25,000 listing and 10,000 active-attempt launch envelope.", + "editable": false, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "liveNow": false, + "panels": [ + { + "id": 1, + "title": "Service", + "description": "Prometheus can authenticate to and scrape the Rendezvous process.", + "type": "stat", + "datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"}, + "gridPos": {"h": 4, "w": 4, "x": 0, "y": 0}, + "fieldConfig": {"defaults": {"mappings": [{"options": {"0": {"color": "red", "text": "DOWN"}, "1": {"color": "green", "text": "UP"}}, "type": "value"}], "thresholds": {"mode": "absolute", "steps": [{"color": "red", "value": null}, {"color": "green", "value": 1}]}}, "overrides": []}, + "options": {"colorMode": "background", "graphMode": "none", "justifyMode": "center", "orientation": "auto", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "textMode": "auto", "wideLayout": true}, + "targets": [{"editorMode": "code", "expr": "up{job=\"rendezvous\"}", "legendFormat": "Rendezvous", "range": true, "refId": "A"}] + }, + { + "id": 2, + "title": "Store", + "type": "stat", + "datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"}, + "gridPos": {"h": 4, "w": 4, "x": 4, "y": 0}, + "fieldConfig": {"defaults": {"mappings": [{"options": {"0": {"color": "red", "text": "UNAVAILABLE"}, "1": {"color": "green", "text": "AVAILABLE"}}, "type": "value"}], "thresholds": {"mode": "absolute", "steps": [{"color": "red", "value": null}, {"color": "green", "value": 1}]}}, "overrides": []}, + "options": {"colorMode": "background", "graphMode": "none", "justifyMode": "center", "orientation": "auto", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "textMode": "auto", "wideLayout": true}, + "targets": [{"editorMode": "code", "expr": "rendezvous_store_available", "legendFormat": "Store", "range": true, "refId": "A"}] + }, + { + "id": 3, + "title": "Drain", + "type": "stat", + "datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"}, + "gridPos": {"h": 4, "w": 4, "x": 8, "y": 0}, + "fieldConfig": {"defaults": {"mappings": [{"options": {"0": {"color": "green", "text": "ACCEPTING"}, "1": {"color": "orange", "text": "DRAINING"}}, "type": "value"}], "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}, {"color": "orange", "value": 1}]}}, "overrides": []}, + "options": {"colorMode": "background", "graphMode": "none", "justifyMode": "center", "orientation": "auto", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "textMode": "auto", "wideLayout": true}, + "targets": [{"editorMode": "code", "expr": "rendezvous_store_draining", "legendFormat": "Drain", "range": true, "refId": "A"}] + }, + { + "id": 4, + "title": "Listings", + "description": "Percentage of the approved 25,000-listing single-process envelope.", + "type": "gauge", + "datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"}, + "gridPos": {"h": 4, "w": 6, "x": 12, "y": 0}, + "fieldConfig": {"defaults": {"max": 100, "min": 0, "unit": "percent", "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}, {"color": "orange", "value": 70}, {"color": "red", "value": 90}]}}, "overrides": []}, + "options": {"minVizHeight": 75, "minVizWidth": 75, "orientation": "auto", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "showThresholdLabels": false, "showThresholdMarkers": true, "sizing": "auto"}, + "targets": [{"editorMode": "code", "expr": "100 * rendezvous_store_active_listings / 25000", "legendFormat": "Listings", "range": true, "refId": "A"}] + }, + { + "id": 5, + "title": "Join attempts", + "description": "Percentage of the approved 10,000 active-attempt single-process envelope.", + "type": "gauge", + "datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"}, + "gridPos": {"h": 4, "w": 6, "x": 18, "y": 0}, + "fieldConfig": {"defaults": {"max": 100, "min": 0, "unit": "percent", "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}, {"color": "orange", "value": 70}, {"color": "red", "value": 90}]}}, "overrides": []}, + "options": {"minVizHeight": 75, "minVizWidth": 75, "orientation": "auto", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "showThresholdLabels": false, "showThresholdMarkers": true, "sizing": "auto"}, + "targets": [{"editorMode": "code", "expr": "100 * rendezvous_store_active_attempts / 10000", "legendFormat": "Attempts", "range": true, "refId": "A"}] + }, + { + "id": 6, + "title": "HTTP request rate by operation", + "description": "Registration, renewal, browse, and join issuance appear as bounded operation names.", + "type": "timeseries", + "datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"}, + "gridPos": {"h": 8, "w": 8, "x": 0, "y": 4}, + "fieldConfig": {"defaults": {"unit": "reqps"}, "overrides": []}, + "options": {"legend": {"calcs": ["lastNotNull"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}}, + "targets": [{"editorMode": "code", "expr": "sum by (operation) (rate(rendezvous_http_requests_total[5m]))", "legendFormat": "{{operation}}", "range": true, "refId": "A"}] + }, + { + "id": 7, + "title": "HTTP p95 latency", + "type": "timeseries", + "datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"}, + "gridPos": {"h": 8, "w": 8, "x": 8, "y": 4}, + "fieldConfig": {"defaults": {"unit": "ms", "custom": {"thresholdsStyle": {"mode": "line"}}, "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}, {"color": "red", "value": 200}]}}, "overrides": []}, + "options": {"legend": {"calcs": ["lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}}, + "targets": [{"editorMode": "code", "expr": "histogram_quantile(0.95, sum by (le, operation) (rate(rendezvous_http_duration_milliseconds_bucket[5m])))", "legendFormat": "{{operation}}", "range": true, "refId": "A"}] + }, + { + "id": 8, + "title": "HTTP error and shedding rate", + "type": "timeseries", + "datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"}, + "gridPos": {"h": 8, "w": 8, "x": 16, "y": 4}, + "fieldConfig": {"defaults": {"unit": "reqps"}, "overrides": []}, + "options": {"legend": {"calcs": ["lastNotNull"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}}, + "targets": [{"editorMode": "code", "expr": "sum by (status_code) (rate(rendezvous_http_requests_total{status_code=~\"4..|5..\"}[5m]))", "legendFormat": "HTTP {{status_code}}", "range": true, "refId": "A"}] + }, + { + "id": 9, + "title": "Browser live-update load", + "description": "Active public SSE clients, tenant scopes, and bounded replay entries.", + "type": "timeseries", + "datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"}, + "gridPos": {"h": 7, "w": 8, "x": 0, "y": 12}, + "fieldConfig": {"defaults": {"unit": "short"}, "overrides": []}, + "options": {"legend": {"calcs": ["lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}}, + "targets": [ + {"editorMode": "code", "expr": "rendezvous_browser_sse_subscribers", "legendFormat": "Subscribers", "range": true, "refId": "A"}, + {"editorMode": "code", "expr": "rendezvous_browser_sse_tenants", "legendFormat": "Tenant scopes", "range": true, "refId": "B"}, + {"editorMode": "code", "expr": "rendezvous_browser_replay_entries", "legendFormat": "Replay entries", "range": true, "refId": "C"} + ] + }, + { + "id": 10, + "title": "UDP mediation results", + "type": "timeseries", + "datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"}, + "gridPos": {"h": 7, "w": 8, "x": 8, "y": 12}, + "fieldConfig": {"defaults": {"unit": "pps"}, "overrides": []}, + "options": {"legend": {"calcs": ["lastNotNull"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}}, + "targets": [{"editorMode": "code", "expr": "sum by (operation, result) (rate(rendezvous_udp_results_total[5m]))", "legendFormat": "{{operation}} · {{result}}", "range": true, "refId": "A"}] + }, + { + "id": 11, + "title": "UDP p95 processing latency", + "type": "timeseries", + "datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"}, + "gridPos": {"h": 7, "w": 8, "x": 16, "y": 12}, + "fieldConfig": {"defaults": {"unit": "ms", "custom": {"thresholdsStyle": {"mode": "line"}}, "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}, {"color": "red", "value": 100}]}}, "overrides": []}, + "options": {"legend": {"calcs": ["lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}}, + "targets": [{"editorMode": "code", "expr": "histogram_quantile(0.95, sum by (le, operation) (rate(rendezvous_udp_duration_milliseconds_bucket[5m])))", "legendFormat": "{{operation}}", "range": true, "refId": "A"}] + }, + { + "id": 12, + "title": "UDP ingress and admitted response budget", + "description": "Traffic bytes observed by the process and the conservative maximum response budget admitted for successful introductions; this is not actual egress.", + "type": "timeseries", + "datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"}, + "gridPos": {"h": 7, "w": 8, "x": 0, "y": 19}, + "fieldConfig": {"defaults": {"unit": "Bps"}, "overrides": []}, + "options": {"legend": {"calcs": ["lastNotNull"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}}, + "targets": [ + {"editorMode": "code", "expr": "sum by (operation) (rate(rendezvous_udp_received_bytes_total[5m]))", "legendFormat": "Ingress · {{operation}}", "range": true, "refId": "A"}, + {"editorMode": "code", "expr": "sum by (operation) (rate(rendezvous_udp_response_budget_bytes_total[5m]))", "legendFormat": "Response budget · {{operation}}", "range": true, "refId": "B"} + ] + }, + { + "id": 13, + "title": "Admission-control drops", + "type": "timeseries", + "datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"}, + "gridPos": {"h": 7, "w": 8, "x": 8, "y": 19}, + "fieldConfig": {"defaults": {"unit": "ops"}, "overrides": []}, + "options": {"legend": {"calcs": ["lastNotNull"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}}, + "targets": [{"editorMode": "code", "expr": "sum by (transport, partition) (rate(rendezvous_limiter_drops_total[5m]))", "legendFormat": "{{transport}} · {{partition}}", "range": true, "refId": "A"}] + }, + { + "id": 14, + "title": "Direct-connect outcomes", + "type": "timeseries", + "datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"}, + "gridPos": {"h": 7, "w": 8, "x": 16, "y": 19}, + "fieldConfig": {"defaults": {"unit": "ops"}, "overrides": []}, + "options": {"legend": {"calcs": ["lastNotNull"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}}, + "targets": [{"editorMode": "code", "expr": "sum by (outcome, elapsed_bucket) (rate(rendezvous_connection_outcomes_total[5m]))", "legendFormat": "{{outcome}} · {{elapsed_bucket}}", "range": true, "refId": "A"}] + }, + { + "id": 15, + "title": "Pairing p95 latency", + "type": "timeseries", + "datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"}, + "gridPos": {"h": 7, "w": 8, "x": 0, "y": 26}, + "fieldConfig": {"defaults": {"unit": "ms", "custom": {"thresholdsStyle": {"mode": "line"}}, "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}, {"color": "red", "value": 100}]}}, "overrides": []}, + "options": {"legend": {"calcs": ["lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}}, + "targets": [{"editorMode": "code", "expr": "histogram_quantile(0.95, rate(rendezvous_pairing_latency_milliseconds_bucket[5m]))", "legendFormat": "Pairing p95", "range": true, "refId": "A"}] + }, + { + "id": 16, + "title": "Presence and expiry state", + "type": "timeseries", + "datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"}, + "gridPos": {"h": 7, "w": 8, "x": 8, "y": 26}, + "fieldConfig": {"defaults": {"unit": "short"}, "overrides": []}, + "options": {"legend": {"calcs": ["lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}}, + "targets": [ + {"editorMode": "code", "expr": "rendezvous_store_fresh_presence_bindings", "legendFormat": "Fresh presence", "range": true, "refId": "A"}, + {"editorMode": "code", "expr": "rendezvous_store_awaiting_presence_listings", "legendFormat": "Awaiting presence", "range": true, "refId": "B"}, + {"editorMode": "code", "expr": "rate(rendezvous_store_expiry_churn_total[5m])", "legendFormat": "Expiry churn/s", "range": true, "refId": "C"} + ] + }, + { + "id": 17, + "title": "Signing key state", + "type": "timeseries", + "datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"}, + "gridPos": {"h": 7, "w": 8, "x": 16, "y": 26}, + "fieldConfig": {"defaults": {"unit": "short"}, "overrides": []}, + "options": {"legend": {"calcs": ["lastNotNull"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}}, + "targets": [{"editorMode": "code", "expr": "rendezvous_signing_keys", "legendFormat": "{{state}}", "range": true, "refId": "A"}] + }, + { + "id": 18, + "title": "Minimum signing window", + "description": "Page below seven days; escalate below 24 hours.", + "type": "stat", + "datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"}, + "gridPos": {"h": 5, "w": 6, "x": 0, "y": 33}, + "fieldConfig": {"defaults": {"unit": "s", "thresholds": {"mode": "absolute", "steps": [{"color": "red", "value": null}, {"color": "orange", "value": 86400}, {"color": "green", "value": 604800}]}}, "overrides": []}, + "options": {"colorMode": "background", "graphMode": "area", "justifyMode": "center", "orientation": "auto", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "textMode": "auto", "wideLayout": true}, + "targets": [{"editorMode": "code", "expr": "rendezvous_signing_key_sign_seconds_remaining", "legendFormat": "Signing window", "range": true, "refId": "A"}] + }, + { + "id": 19, + "title": "Operator authentication", + "type": "timeseries", + "datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"}, + "gridPos": {"h": 5, "w": 6, "x": 6, "y": 33}, + "fieldConfig": {"defaults": {"unit": "ops"}, "overrides": []}, + "options": {"legend": {"calcs": ["lastNotNull"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}}, + "targets": [{"editorMode": "code", "expr": "sum by (result) (rate(rendezvous_operator_authentication_total[5m]))", "legendFormat": "{{result}}", "range": true, "refId": "A"}] + }, + { + "id": 20, + "title": "Privileged audit outcomes", + "type": "timeseries", + "datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"}, + "gridPos": {"h": 5, "w": 6, "x": 12, "y": 33}, + "fieldConfig": {"defaults": {"unit": "ops"}, "overrides": []}, + "options": {"legend": {"calcs": ["lastNotNull"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}}, + "targets": [{"editorMode": "code", "expr": "sum by (action, result) (rate(rendezvous_audit_events_total[5m]))", "legendFormat": "{{action}} · {{result}}", "range": true, "refId": "A"}] + }, + { + "id": 21, + "title": "Metrics access", + "type": "timeseries", + "datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"}, + "gridPos": {"h": 5, "w": 6, "x": 18, "y": 33}, + "fieldConfig": {"defaults": {"unit": "ops"}, "overrides": []}, + "options": {"legend": {"calcs": ["lastNotNull"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}}, + "targets": [{"editorMode": "code", "expr": "sum by (result) (rate(rendezvous_metrics_scrapes_total[5m]))", "legendFormat": "{{result}}", "range": true, "refId": "A"}] + }, + { + "id": 22, + "title": "Process memory", + "type": "timeseries", + "datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"}, + "gridPos": {"h": 7, "w": 8, "x": 0, "y": 38}, + "fieldConfig": {"defaults": {"unit": "bytes", "custom": {"thresholdsStyle": {"mode": "line"}}, "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}, {"color": "orange", "value": 1073741824}, {"color": "red", "value": 1610612736}]}}, "overrides": []}, + "options": {"legend": {"calcs": ["lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}}, + "targets": [ + {"editorMode": "code", "expr": "process_resident_memory_bytes", "legendFormat": "Resident", "range": true, "refId": "A"}, + {"editorMode": "code", "expr": "dotnet_gc_heap_size_bytes", "legendFormat": "Managed heap", "range": true, "refId": "B"} + ] + }, + { + "id": 23, + "title": "Process CPU and threads", + "type": "timeseries", + "datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"}, + "gridPos": {"h": 7, "w": 8, "x": 8, "y": 38}, + "fieldConfig": {"defaults": {"unit": "short"}, "overrides": [{"matcher": {"id": "byName", "options": "CPU cores"}, "properties": [{"id": "unit", "value": "cores"}]}]}, + "options": {"legend": {"calcs": ["lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}}, + "targets": [ + {"editorMode": "code", "expr": "rate(process_cpu_seconds_total[5m])", "legendFormat": "CPU cores", "range": true, "refId": "A"}, + {"editorMode": "code", "expr": "process_threads", "legendFormat": "Process threads", "range": true, "refId": "B"}, + {"editorMode": "code", "expr": "dotnet_thread_pool_threads", "legendFormat": "Thread-pool threads", "range": true, "refId": "C"} + ] + }, + { + "id": 24, + "title": "Descriptor and GC pressure", + "type": "timeseries", + "datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"}, + "gridPos": {"h": 7, "w": 8, "x": 16, "y": 38}, + "fieldConfig": {"defaults": {"unit": "short"}, "overrides": []}, + "options": {"legend": {"calcs": ["lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}}, + "targets": [ + {"editorMode": "code", "expr": "process_open_file_descriptors", "legendFormat": "Open descriptors", "range": true, "refId": "A"}, + {"editorMode": "code", "expr": "sum(rate(dotnet_gc_collections_total[5m]))", "legendFormat": "GC collections/s", "range": true, "refId": "B"}, + {"editorMode": "code", "expr": "dotnet_thread_pool_available_worker_threads", "legendFormat": "Available workers", "range": true, "refId": "C"} + ] + } + ], + "refresh": "15s", + "schemaVersion": 41, + "tags": ["rendezvous", "operations", "privacy-safe"], + "templating": {"list": []}, + "time": {"from": "now-1h", "to": "now"}, + "timepicker": {}, + "timezone": "browser", + "title": "Rendezvous operational overview", + "uid": "rendezvous-overview", + "version": 1, + "weekStart": "" +} diff --git a/deploy/observability/grafana/provisioning/dashboards/rendezvous.yaml b/deploy/observability/grafana/provisioning/dashboards/rendezvous.yaml new file mode 100644 index 0000000..fa9fecd --- /dev/null +++ b/deploy/observability/grafana/provisioning/dashboards/rendezvous.yaml @@ -0,0 +1,13 @@ +apiVersion: 1 + +providers: + - name: Rendezvous + orgId: 1 + folder: Rendezvous + type: file + disableDeletion: true + allowUiUpdates: false + updateIntervalSeconds: 30 + options: + path: /var/lib/grafana/dashboards + foldersFromFilesStructure: false diff --git a/deploy/observability/grafana/provisioning/datasources/prometheus.yaml b/deploy/observability/grafana/provisioning/datasources/prometheus.yaml new file mode 100644 index 0000000..cd754f1 --- /dev/null +++ b/deploy/observability/grafana/provisioning/datasources/prometheus.yaml @@ -0,0 +1,17 @@ +apiVersion: 1 + +deleteDatasources: + - name: Rendezvous Prometheus + orgId: 1 + +datasources: + - name: Rendezvous Prometheus + uid: rendezvous-prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + editable: false + jsonData: + httpMethod: POST + timeInterval: 15s diff --git a/deploy/observability/prometheus/prometheus.yml b/deploy/observability/prometheus/prometheus.yml new file mode 100644 index 0000000..8a1c44f --- /dev/null +++ b/deploy/observability/prometheus/prometheus.yml @@ -0,0 +1,14 @@ +global: + scrape_interval: 15s + evaluation_interval: 15s + external_labels: + service: rendezvous + +scrape_configs: + - job_name: rendezvous + scheme: http + metrics_path: /metrics + bearer_token_file: /run/secrets/rendezvous-metrics-token + static_configs: + - targets: + - rendezvous:8080 diff --git a/deploy/observability/secrets/.gitignore b/deploy/observability/secrets/.gitignore new file mode 100644 index 0000000..d6b7ef3 --- /dev/null +++ b/deploy/observability/secrets/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore diff --git a/docs/operations/diagnostic-dashboards.md b/docs/operations/diagnostic-dashboards.md new file mode 100644 index 0000000..4382cf3 --- /dev/null +++ b/docs/operations/diagnostic-dashboards.md @@ -0,0 +1,123 @@ +# Diagnostic dashboards + +Tracking: #27 + +Rendezvous provides two deliberately separate, optional views. The public +session diagnostic helps a player or integration operator understand safe +session-list state using only the public browse contract. The private Grafana +dashboard exposes aggregate operational health through authenticated metrics. +Neither view grants operator privileges or exposes player identity, endpoints, +credentials, capabilities, or raw session metadata beyond the explicitly +allowlisted public browse fields. + +## Public read-only session diagnostic + +The static diagnostic is disabled by default. Enable it only for approved +game/environment scopes and keep the allowlist narrow: + +```json +"Diagnostics": { + "Enabled": true, + "PollIntervalSeconds": 10, + "MaximumRenderedSessions": 100, + "Scopes": [ + { + "GameId": "space-game", + "EnvironmentId": "smoke", + "ProtocolVersions": [1, 2], + "Regions": ["local"] + } + ] +} +``` + +Open `/diagnostics` on the same origin as Rendezvous. The page cannot choose a +different backend, request private visibility, join a session, or call the +operator surface. It renders a bounded snapshot, then applies ordered SSE +updates. A replay reset, corrupt cursor, incomplete snapshot, transport failure, +or deliberate reconnect returns to a fresh authoritative snapshot and bounded +polling. Apply filter changes explicitly; **Reset filters** restores the +configured defaults, while **Reconnect now** tests recovery without changing +the selection. + +The page uses semantic HTML, labelled controls, visible keyboard focus, status +text in addition to color, a reduced-motion mode, and a 320-pixel reflow. It +creates untrusted content with `textContent` only. The endpoint sets a restrictive +same-origin content-security policy, denies framing, disables MIME sniffing and +browser capabilities, and marks every asset/config response `no-store`. + +This is a diagnostics convenience, not a game browser, management console, or +availability monitor. Disable it independently by setting `Enabled` to `false`; +all diagnostic paths then return `404` without affecting game traffic, metrics, +or health endpoints. + +## Private Prometheus and Grafana view + +The observability overlay pins Prometheus 3.13.1 and Grafana 13.1.0 by immutable +multi-platform image digest. Prometheus is not published to the host. Grafana is +bound to host loopback, disables anonymous access and sign-up, and reads its +administrator password from a file. The service and Prometheus share only the +metrics bearer-token file. All three secrets remain ignored by Git. + +Create independent random secrets, then start the base service and overlay: + +```bash +install -d -m 0700 deploy/compose/secrets deploy/observability/secrets +umask 077 +openssl rand -out deploy/compose/secrets/signing-key 32 +openssl rand -hex 32 >deploy/observability/secrets/rendezvous-metrics-token +openssl rand -base64 36 >deploy/observability/secrets/grafana-admin-password +export RENDEZVOUS_UID="$(id -u)" +export RENDEZVOUS_GID="$(id -g)" +test "$RENDEZVOUS_UID" -ne 0 +docker compose \ + -f deploy/compose/compose.yaml \ + -f deploy/observability/compose.yaml \ + up --build --detach +``` + +Visit `http://127.0.0.1:3000`, sign in as `rendezvous-admin`, and open the +**Rendezvous operational overview** folder/dashboard. The provisioned panels +cover scrape/store/drain health, listing and join capacity, HTTP volume/errors +and p95, browse/SSE load, lease and join operations, UDP results/latency/bytes, +admission drops, connection outcomes, pairing latency, presence/expiry state, +signing windows, security/audit results, and process/GC/descriptor pressure. +Capacity gauges use the approved single-process envelope of 25,000 listings and +10,000 active attempts, with 70% warning and 90% critical thresholds. The UDP +response series is a conservative admitted maximum, not observed egress. + +Validate merged configuration and checked-in dashboard structure before every +rollout: + +```bash +RENDEZVOUS_UID="$(id -u)" RENDEZVOUS_GID="$(id -g)" \ + docker compose \ + -f deploy/compose/compose.yaml \ + -f deploy/observability/compose.yaml \ + config --quiet +./scripts/test-observability-assets.sh +``` + +For a real deployment, keep Grafana on a private authenticated management +network instead of host loopback, replace the local admin login with the +organization's supported identity boundary, enforce TLS at the edge, and set +retention to the approved operational period. Do not make Prometheus public. +Provisioning is read-only so local UI edits cannot silently drift from source. + +## Verify, rotate, and disable + +After startup, verify the dashboard shows `UP`, store `AVAILABLE`, a nonzero +signing window, and changing request/UDP panels during a smoke run. Confirm an +unauthenticated `/metrics` request returns `404`, the bearer-authenticated +collector target is healthy, Prometheus is not bound on a host port, Grafana is +not anonymously accessible, and dashboard query labels contain no identifiers. + +Rotate metrics access by writing a new 32-128 character token to the secret file +with private permissions and restarting Rendezvous and Prometheus together. +Rotate the Grafana administrator password through the same protected secret +workflow. Delete both secret files after a disposable local run. + +To disable aggregate observability independently, stop/remove the overlay and +set `Rendezvous:Metrics:Enabled` to `false`; `/metrics` returns `404` and the +core service continues. To disable only the public session diagnostic, leave the +overlay running and set `Rendezvous:Diagnostics:Enabled` to `false`. diff --git a/docs/operations/observability-and-operator-runbook.md b/docs/operations/observability-and-operator-runbook.md index 7a657d8..65c9634 100644 --- a/docs/operations/observability-and-operator-runbook.md +++ b/docs/operations/observability-and-operator-runbook.md @@ -2,9 +2,10 @@ This runbook defines the production signals and privileged controls for the Rendezvous service. The service emits `System.Diagnostics.Metrics` instruments -from the `FinalFactory.Rendezvous` meter and distributed-tracing activities from -`FinalFactory.Rendezvous.Server`. Connect those sources to the deployment's -OpenTelemetry or equivalent collector. Do not add identifiers to metric labels. +from the `FinalFactory.Rendezvous` meter, distributed-tracing activities from +`FinalFactory.Rendezvous.Server`, and an optional bearer-protected Prometheus +endpoint. Connect only a private collector network. Do not add identifiers to +metric labels. Concrete detect/contain/recover/verify procedures for abuse, key compromise, targeted revocation, restart, rollback, saturation, privacy incidents, and @@ -38,6 +39,25 @@ dependency upgrades are in the [incident and change runbooks](incident-runbooks. | `rendezvous.store.expiry_churn` | Cumulative natural expiry activity | none | | `rendezvous.store.available` | Store health (`1` available, `0` unavailable) | none | +The Prometheus exporter additionally exposes active/fresh/awaiting store state, +drain state, SSE subscriber/tenant/replay gauges, bounded HTTP and UDP +histograms, UDP ingress and conservative admitted-response budgets, signing-key +state/window gauges, and process/.NET pressure. Its only labels are the fixed +operation, status, result, transport, partition, action, outcome, elapsed-bucket, +key-state, and GC-generation dimensions. Unknown or unsafe values normalize to +`other`; identifiers, metadata, addresses, endpoints, tokens, and capabilities +are never labels. + +The exporter is disabled by default. Enabling `Rendezvous:Metrics:Enabled` +requires `BearerTokenSecretReference` to be an external `env:` or absolute +`file:` secret containing 32-128 visible ASCII bytes. Unauthorized requests get +the same `404` as a disabled endpoint, and accepted responses are `no-store`. +Expose `/metrics` only to the private collector network, rotate its token as a +deployment secret, and never place the token in a URL, Compose environment +value, dashboard, log, or issue. The checked-in Prometheus/Grafana provisioning +and its verification procedure are in +[diagnostic dashboards](diagnostic-dashboards.md). + HTTP responses include `X-Rendezvous-Correlation-ID`. It is a generated trace ID or random value, never a caller-supplied session or player identifier. UDP and HTTP activities contain operation-level data only. Logs and traces must not add diff --git a/scripts/test-diagnostic-dashboard.sh b/scripts/test-diagnostic-dashboard.sh new file mode 100755 index 0000000..d888b9e --- /dev/null +++ b/scripts/test-diagnostic-dashboard.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +set -euo pipefail + +root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$root" +node --test tests/Diagnostics/*.test.mjs diff --git a/scripts/test-observability-assets.sh b/scripts/test-observability-assets.sh new file mode 100755 index 0000000..944f17e --- /dev/null +++ b/scripts/test-observability-assets.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +set -euo pipefail + +node --test tests/Observability/observability-assets.test.mjs diff --git a/src/FinalFactory.Rendezvous.Server/Browser/SessionChangeJournal.cs b/src/FinalFactory.Rendezvous.Server/Browser/SessionChangeJournal.cs index a80e841..5a145a2 100644 --- a/src/FinalFactory.Rendezvous.Server/Browser/SessionChangeJournal.cs +++ b/src/FinalFactory.Rendezvous.Server/Browser/SessionChangeJournal.cs @@ -143,6 +143,39 @@ internal sealed class SessionChangeJournal public SessionChangeJournalOptions Options => _options; + public int ReplayCount + { + get + { + lock (_gate) + { + return _changes.Count; + } + } + } + + public int SubscriberCount + { + get + { + lock (_gate) + { + return _subscribers; + } + } + } + + public int SubscribedTenantCount + { + get + { + lock (_gate) + { + return _subscribersByTenant.Count; + } + } + } + public long CurrentRevision { get diff --git a/src/FinalFactory.Rendezvous.Server/Diagnostics/Assets/app.mjs b/src/FinalFactory.Rendezvous.Server/Diagnostics/Assets/app.mjs new file mode 100644 index 0000000..f01b6e5 --- /dev/null +++ b/src/FinalFactory.Rendezvous.Server/Diagnostics/Assets/app.mjs @@ -0,0 +1,615 @@ +const CONTRACT_VERSION = 1; +const BROWSER_PAGE_LIMIT = 100; +const REQUEST_TIMEOUT_MS = 10_000; +const STREAM_FAILURE_LIMIT = 3; + +export class SessionProjection { + constructor(maximumSessions = 100) { + if (!Number.isInteger(maximumSessions) || maximumSessions < 1 || maximumSessions > 500) { + throw new RangeError("maximumSessions must be between 1 and 500"); + } + this.maximumSessions = maximumSessions; + this.entries = new Map(); + this.cursor = ""; + this.hasMore = false; + } + + replace(items, cursor, hasMore = false, updatedAt = Date.now()) { + if (!Array.isArray(items) || items.length > this.maximumSessions) { + return "overflow"; + } + const next = new Map(); + for (const session of items) { + if (!isPublicSession(session) || next.has(session.listingId)) { + return "invalid"; + } + next.set(session.listingId, { session, updatedAt }); + } + this.entries = next; + this.cursor = validCursor(cursor) ? cursor : ""; + this.hasMore = Boolean(hasMore); + return "applied"; + } + + apply(event, updatedAt = Date.now()) { + if (!event || event.contractVersion !== CONTRACT_VERSION || !validCursor(event.cursor)) { + return "invalid"; + } + if (event.kind === "sessionUpsert" && isPublicSession(event.session) && event.listingId == null) { + if (!this.entries.has(event.session.listingId) + && this.entries.size >= this.maximumSessions) { + return "overflow"; + } + this.entries.set(event.session.listingId, { session: event.session, updatedAt }); + this.cursor = event.cursor; + return "applied"; + } + if (event.kind === "sessionRemove" + && typeof event.listingId === "string" + && event.listingId.length > 0 + && event.session == null) { + this.entries.delete(event.listingId); + this.cursor = event.cursor; + return this.hasMore ? "refresh" : "applied"; + } + if (event.kind === "keepalive" && event.session == null && event.listingId == null) { + this.cursor = event.cursor; + return "keepalive"; + } + if (event.kind === "reset" && event.session == null && event.listingId == null) { + this.cursor = event.cursor; + return "reset"; + } + return "invalid"; + } + + sessions() { + return [...this.entries.values()] + .sort((left, right) => left.session.listingId.localeCompare(right.session.listingId)); + } +} + +export function safeText(value, maximumLength = 160) { + const text = typeof value === "string" ? value : String(value ?? ""); + const visible = text.replace(/[\u0000-\u001f\u007f]/gu, "�"); + return visible.length <= maximumLength + ? visible + : `${visible.slice(0, Math.max(0, maximumLength - 1))}…`; +} + +export function buildBrowseUrl(filter) { + validateFilter(filter); + const query = new URLSearchParams({ + contractVersion: String(CONTRACT_VERSION), + gameId: filter.gameId, + environmentId: filter.environmentId, + protocolVersion: String(filter.protocolVersion), + regionId: filter.regionId, + pageSize: String(Math.min(BROWSER_PAGE_LIMIT, filter.pageSize ?? BROWSER_PAGE_LIMIT)), + excludeFull: String(Boolean(filter.excludeFull)), + }); + return `/v1/sessions?${query}`; +} + +export function buildStreamUrl(filter, cursor) { + validateFilter(filter); + if (!validCursor(cursor)) { + throw new TypeError("A bounded stream cursor is required"); + } + const query = new URLSearchParams({ + contractVersion: String(CONTRACT_VERSION), + gameId: filter.gameId, + environmentId: filter.environmentId, + protocolVersion: String(filter.protocolVersion), + regionId: filter.regionId, + excludeFull: String(Boolean(filter.excludeFull)), + streamCursor: cursor, + }); + return `/v1/sessions/stream?${query}`; +} + +export function chooseSnapshotTransport(hasMore, pollingOnly) { + if (pollingOnly) { + return "pollingOnly"; + } + return hasMore ? "boundedPolling" : "stream"; +} + +function validCursor(value) { + return typeof value === "string" + && value.length > 0 + && value.length <= 1024 + && /^[\x21-\x7e]+$/u.test(value); +} + +function isPublicSession(session) { + return session != null + && session.contractVersion === CONTRACT_VERSION + && typeof session.listingId === "string" + && session.listingId.length > 0 + && typeof session.gameId === "string" + && typeof session.environmentId === "string" + && typeof session.regionId === "string" + && Number.isInteger(session.protocolVersion) + && session.protocolVersion > 0 + && typeof session.buildVersion === "string" + && typeof session.displayName === "string" + && session.visibility === "public" + && session.capacity != null + && Number.isInteger(session.capacity.currentPlayers) + && Number.isInteger(session.capacity.maximumPlayers) + && session.capacity.currentPlayers >= 0 + && session.capacity.maximumPlayers >= 1 + && session.capacity.currentPlayers <= session.capacity.maximumPlayers + && session.metadata != null + && typeof session.metadata === "object" + && !Array.isArray(session.metadata); +} + +function validateFilter(filter) { + if (!filter + || typeof filter.gameId !== "string" + || typeof filter.environmentId !== "string" + || typeof filter.regionId !== "string" + || !Number.isInteger(filter.protocolVersion) + || filter.protocolVersion <= 0) { + throw new TypeError("The selected diagnostic filter is invalid"); + } +} + +function startDashboard() { + const elements = { + form: document.querySelector("#filters"), + game: document.querySelector("#game-filter"), + environment: document.querySelector("#environment-filter"), + protocol: document.querySelector("#protocol-filter"), + region: document.querySelector("#region-filter"), + capacity: document.querySelector("#capacity-filter"), + error: document.querySelector("#filter-error"), + reconnect: document.querySelector("#reconnect-button"), + reset: document.querySelector("#reset-button"), + poll: document.querySelector("#poll-button"), + streamStatus: document.querySelector("#stream-status"), + transportState: document.querySelector("#transport-state"), + projectionState: document.querySelector("#projection-state"), + lastUpdate: document.querySelector("#last-update"), + sessionCount: document.querySelector("#session-count"), + results: document.querySelector(".results-panel"), + resultsSummary: document.querySelector("#results-summary"), + list: document.querySelector("#session-list"), + empty: document.querySelector("#empty-state"), + }; + + const state = { + configuration: null, + projection: null, + activeFilter: null, + source: null, + pollingTimer: null, + requestController: null, + streamFailures: 0, + pollingOnly: false, + renderPending: false, + lastSuccess: 0, + }; + + function setStatus(kind, message, transport = message) { + document.body.dataset.streamState = kind; + elements.streamStatus.textContent = message; + elements.transportState.textContent = transport; + } + + function setFormError(message = "") { + elements.error.textContent = message; + elements.error.hidden = message.length === 0; + } + + function populate(select, values, previous) { + select.replaceChildren(); + for (const value of values) { + const option = document.createElement("option"); + option.value = String(value); + option.textContent = safeText(value, 80); + select.append(option); + } + if (values.some((value) => String(value) === previous)) { + select.value = previous; + } + } + + function selectedScope() { + return state.configuration?.scopes.find((scope) => + scope.gameId === elements.game.value + && scope.environmentId === elements.environment.value) ?? null; + } + + function updateEnvironmentOptions() { + const prior = elements.environment.value; + const environments = state.configuration.scopes + .filter((scope) => scope.gameId === elements.game.value) + .map((scope) => scope.environmentId); + populate(elements.environment, environments, prior); + updateScopeOptions(); + } + + function updateScopeOptions() { + const scope = selectedScope(); + populate(elements.protocol, scope?.protocolVersions ?? [], elements.protocol.value); + populate(elements.region, scope?.regions ?? [], elements.region.value); + } + + function currentFilter() { + const scope = selectedScope(); + const protocolVersion = Number(elements.protocol.value); + if (!scope + || !scope.protocolVersions.includes(protocolVersion) + || !scope.regions.includes(elements.region.value)) { + throw new TypeError("Choose one of the configured game, environment, protocol, and region combinations."); + } + return { + gameId: scope.gameId, + environmentId: scope.environmentId, + protocolVersion, + regionId: elements.region.value, + excludeFull: elements.capacity.value === "open", + pageSize: Math.min(BROWSER_PAGE_LIMIT, state.configuration.maximumRenderedSessions), + }; + } + + function stopLiveWork() { + if (state.source) { + state.source.close(); + state.source = null; + } + if (state.pollingTimer) { + clearTimeout(state.pollingTimer); + state.pollingTimer = null; + } + if (state.requestController) { + state.requestController.abort(); + state.requestController = null; + } + } + + function markSuccess(message) { + state.lastSuccess = Date.now(); + elements.lastUpdate.dateTime = new Date(state.lastSuccess).toISOString(); + elements.lastUpdate.textContent = "Just now"; + elements.resultsSummary.textContent = message; + } + + function scheduleRender() { + if (state.renderPending) { + return; + } + state.renderPending = true; + requestAnimationFrame(() => { + state.renderPending = false; + renderProjection(); + }); + } + + function appendDetail(list, term, description) { + const dt = document.createElement("dt"); + dt.textContent = term; + const dd = document.createElement("dd"); + dd.textContent = safeText(description, 160); + list.append(dt, dd); + } + + function sessionCard(entry) { + const session = entry.session; + const article = document.createElement("article"); + article.className = "session-card"; + article.setAttribute("role", "listitem"); + article.dataset.updatedAt = String(entry.updatedAt); + + const heading = document.createElement("div"); + heading.className = "card-heading"; + const title = document.createElement("h3"); + title.textContent = safeText(session.displayName, 120); + const region = document.createElement("span"); + region.className = "region-badge"; + region.textContent = safeText(session.regionId, 48); + heading.append(title, region); + + const details = document.createElement("dl"); + details.className = "session-detail"; + appendDetail(details, "Build", session.buildVersion); + appendDetail(details, "Protocol", session.protocolVersion); + appendDetail(details, "Visibility", "Public"); + appendDetail(details, "Presence", "Recently verified"); + + const capacity = document.createElement("div"); + capacity.className = "capacity-row"; + const capacityCopy = document.createElement("div"); + capacityCopy.className = "capacity-copy"; + const capacityLabel = document.createElement("span"); + capacityLabel.textContent = "Advisory capacity"; + const capacityValue = document.createElement("span"); + capacityValue.textContent = `${session.capacity.currentPlayers} / ${session.capacity.maximumPlayers}`; + capacityCopy.append(capacityLabel, capacityValue); + const meter = document.createElement("meter"); + meter.min = 0; + meter.max = session.capacity.maximumPlayers; + meter.value = session.capacity.currentPlayers; + meter.setAttribute("aria-label", `Advisory capacity ${capacityValue.textContent}`); + capacity.append(capacityCopy, meter); + + const metadata = document.createElement("ul"); + metadata.className = "metadata-list"; + for (const [key, value] of Object.entries(session.metadata).slice(0, 16)) { + const item = document.createElement("li"); + item.textContent = `${safeText(key, 48)}: ${safeText(value, 96)}`; + metadata.append(item); + } + if (metadata.childElementCount === 0) { + const item = document.createElement("li"); + item.textContent = "No public metadata"; + metadata.append(item); + } + + const updated = document.createElement("p"); + updated.className = "updated-age"; + updated.textContent = "Updated just now"; + article.append(heading, details, capacity, metadata, updated); + return article; + } + + function renderProjection() { + const entries = state.projection?.sessions() ?? []; + const fragment = document.createDocumentFragment(); + for (const entry of entries) { + fragment.append(sessionCard(entry)); + } + elements.list.replaceChildren(fragment); + elements.sessionCount.textContent = String(entries.length); + elements.empty.hidden = entries.length !== 0; + elements.projectionState.textContent = state.projection?.hasMore + ? "Bounded snapshot; polling" + : "Snapshot plus ordered deltas"; + elements.results.setAttribute("aria-busy", "false"); + } + + function updateAges() { + const now = Date.now(); + if (state.lastSuccess > 0) { + const age = Math.max(0, Math.floor((now - state.lastSuccess) / 1000)); + elements.lastUpdate.textContent = age < 5 ? "Just now" : `${age} seconds ago`; + } + for (const card of elements.list.querySelectorAll(".session-card")) { + const age = Math.max(0, Math.floor((now - Number(card.dataset.updatedAt)) / 1000)); + const copy = card.querySelector(".updated-age"); + copy.textContent = age < 5 ? "Updated just now" : `Updated ${age} seconds ago`; + } + } + + async function readJson(url) { + const controller = new AbortController(); + state.requestController = controller; + const timeout = setTimeout(() => controller.abort(), REQUEST_TIMEOUT_MS); + try { + const response = await fetch(url, { + cache: "no-store", + credentials: "same-origin", + headers: { Accept: "application/json" }, + signal: controller.signal, + }); + if (!response.ok) { + throw new Error(`The service returned HTTP ${response.status}.`); + } + return await response.json(); + } finally { + clearTimeout(timeout); + if (state.requestController === controller) { + state.requestController = null; + } + } + } + + function schedulePolling() { + if (state.pollingTimer) { + clearTimeout(state.pollingTimer); + } + state.pollingTimer = setTimeout( + () => fetchSnapshot("poll").catch(showServiceError), + state.configuration.pollIntervalSeconds * 1000, + ); + } + + function usePolling(message = "Polling fallback active") { + if (state.source) { + state.source.close(); + state.source = null; + } + setStatus("polling", message, "Polling fallback"); + schedulePolling(); + } + + function showServiceError(error) { + const message = error?.name === "AbortError" + ? "The service did not answer within the request deadline." + : safeText(error?.message || "The service is unavailable.", 200); + setStatus("error", "Service unavailable", "Unavailable"); + elements.resultsSummary.textContent = `${message} Retrying with bounded polling.`; + elements.results.setAttribute("aria-busy", "false"); + usePolling("Service unavailable; polling retry scheduled"); + } + + async function fetchSnapshot(reason = "manual") { + if (!state.activeFilter) { + return; + } + if (state.requestController) { + state.requestController.abort(); + } + elements.results.setAttribute("aria-busy", "true"); + if (reason !== "poll") { + setStatus("reconnecting", "Loading a fresh snapshot", "Snapshot request"); + } + const response = await readJson(buildBrowseUrl(state.activeFilter)); + if (response.contractVersion !== CONTRACT_VERSION + || !Array.isArray(response.items) + || !validCursor(response.streamCursor)) { + throw new Error("The service returned an invalid browser snapshot."); + } + const outcome = state.projection.replace( + response.items, + response.streamCursor, + Boolean(response.nextCursor), + ); + if (outcome !== "applied") { + throw new Error("The bounded browser snapshot could not be applied safely."); + } + scheduleRender(); + markSuccess(`${response.items.length} public session${response.items.length === 1 ? "" : "s"} in the fresh snapshot.`); + const transport = chooseSnapshotTransport(Boolean(response.nextCursor), state.pollingOnly); + if (transport !== "stream") { + if (transport === "pollingOnly") { + usePolling("Polling only selected"); + return; + } + usePolling("More sessions exist than this bounded view; polling keeps it authoritative"); + return; + } + connectStream(response.streamCursor); + } + + function handleStreamEvent(serialized, expectedKind) { + let event; + try { + event = JSON.parse(serialized); + } catch { + usePolling("Invalid stream data; polling fallback active"); + return; + } + if (event.kind !== expectedKind) { + usePolling("Unexpected stream event; polling fallback active"); + return; + } + const outcome = state.projection.apply(event); + if (outcome === "invalid" || outcome === "overflow" || outcome === "refresh") { + fetchSnapshot("stream-recovery").catch(showServiceError); + return; + } + if (outcome === "reset") { + setStatus("reset", "Cursor reset; refreshing snapshot", "Cursor reset"); + fetchSnapshot("reset").catch(showServiceError); + return; + } + if (outcome === "keepalive") { + markSuccess("Live connection healthy; no listing changes."); + return; + } + state.streamFailures = 0; + scheduleRender(); + markSuccess(expectedKind === "sessionUpsert" + ? "Applied a live session add or update." + : "Applied a live session removal."); + } + + function connectStream(cursor, corrupt = false) { + if (state.pollingTimer) { + clearTimeout(state.pollingTimer); + state.pollingTimer = null; + } + if (state.source) { + state.source.close(); + } + const selectedCursor = corrupt + ? `${cursor.slice(0, -1)}${cursor.endsWith("a") ? "b" : "a"}` + : cursor; + setStatus(corrupt ? "reset" : "reconnecting", + corrupt ? "Testing cursor reset" : "Connecting live updates", + corrupt ? "Cursor reset test" : "SSE reconnecting"); + const source = new EventSource(buildStreamUrl(state.activeFilter, selectedCursor)); + state.source = source; + source.addEventListener("open", () => { + if (state.source !== source) { + return; + } + state.streamFailures = 0; + setStatus("connected", "Live updates connected", "SSE live"); + }); + for (const [name, kind] of [ + ["session_upsert", "sessionUpsert"], + ["session_remove", "sessionRemove"], + ["reset", "reset"], + ["keepalive", "keepalive"], + ]) { + source.addEventListener(name, (message) => { + if (state.source === source) { + handleStreamEvent(message.data, kind); + } + }); + } + source.addEventListener("error", () => { + if (state.source !== source) { + return; + } + state.streamFailures += 1; + if (state.streamFailures >= STREAM_FAILURE_LIMIT || source.readyState === EventSource.CLOSED) { + usePolling("Live stream unavailable; polling fallback active"); + } else { + setStatus("reconnecting", "Live stream interrupted; reconnecting", "SSE reconnecting"); + } + }); + } + + async function loadConfiguration() { + const configuration = await readJson("/diagnostics/config.json"); + if (configuration.contractVersion !== CONTRACT_VERSION + || !Array.isArray(configuration.scopes) + || configuration.scopes.length < 1 + || !Number.isInteger(configuration.pollIntervalSeconds) + || !Number.isInteger(configuration.maximumRenderedSessions)) { + throw new Error("The diagnostic configuration is invalid."); + } + state.configuration = configuration; + state.projection = new SessionProjection(configuration.maximumRenderedSessions); + populate(elements.game, [...new Set(configuration.scopes.map((scope) => scope.gameId))], ""); + updateEnvironmentOptions(); + state.activeFilter = currentFilter(); + await fetchSnapshot("startup"); + } + + elements.game.addEventListener("change", updateEnvironmentOptions); + elements.environment.addEventListener("change", updateScopeOptions); + elements.form.addEventListener("submit", (event) => { + event.preventDefault(); + try { + setFormError(); + stopLiveWork(); + state.pollingOnly = false; + state.activeFilter = currentFilter(); + state.projection = new SessionProjection(state.configuration.maximumRenderedSessions); + fetchSnapshot("filter").catch(showServiceError); + } catch (error) { + setFormError(safeText(error.message, 200)); + } + }); + elements.reconnect.addEventListener("click", () => { + if (state.projection?.cursor) { + state.pollingOnly = false; + connectStream(state.projection.cursor); + } + }); + elements.reset.addEventListener("click", () => { + if (state.projection?.cursor) { + state.pollingOnly = false; + connectStream(state.projection.cursor, true); + } + }); + elements.poll.addEventListener("click", () => { + state.pollingOnly = true; + usePolling("Polling only selected deliberately"); + fetchSnapshot("poll").catch(showServiceError); + }); + window.addEventListener("pagehide", stopLiveWork, { once: true }); + setInterval(updateAges, 5000); + loadConfiguration().catch(showServiceError); +} + +if (typeof document !== "undefined") { + startDashboard(); +} diff --git a/src/FinalFactory.Rendezvous.Server/Diagnostics/Assets/index.html b/src/FinalFactory.Rendezvous.Server/Diagnostics/Assets/index.html new file mode 100644 index 0000000..411e886 --- /dev/null +++ b/src/FinalFactory.Rendezvous.Server/Diagnostics/Assets/index.html @@ -0,0 +1,120 @@ + + + + + + + Session diagnostics — Rendezvous + + + + + + + +
+
+
+
+

Public browser scope

+

Choose a configured session view

+
+

This page has no join, publish, or operator authority.

+
+ +
+
+ + + + + + +
+ +
+ + + + +
+
+
+ +
+

Current diagnostic state

+
+
+
Sessions shown
+
0
+
+
+
Transport
+
Starting
+
+
+
Last successful update
+
+
+
+
Projection
+
Waiting for snapshot
+
+
+
+ +
+
+
+

Bounded public projection

+

Available sessions

+
+

Loading a fresh snapshot…

+
+
+ +
+
+ + + + + diff --git a/src/FinalFactory.Rendezvous.Server/Diagnostics/Assets/styles.css b/src/FinalFactory.Rendezvous.Server/Diagnostics/Assets/styles.css new file mode 100644 index 0000000..e396d5c --- /dev/null +++ b/src/FinalFactory.Rendezvous.Server/Diagnostics/Assets/styles.css @@ -0,0 +1,505 @@ +:root { + color-scheme: dark; + --bg: #071019; + --surface: #101c28; + --surface-raised: #172737; + --border: #375066; + --text: #f2f7fb; + --muted: #b6c8d6; + --accent: #55d7b5; + --accent-strong: #7ce9cc; + --accent-ink: #04231c; + --warning: #ffd479; + --danger: #ff9f9f; + --focus: #ffd479; + --radius: 0.75rem; + --space-1: 0.5rem; + --space-2: 0.75rem; + --space-3: 1rem; + --space-4: 1.5rem; + --space-5: 2rem; + --space-6: 3rem; + font-family: Inter, ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif; + font-size: 100%; + line-height: 1.55; +} + +* { + box-sizing: border-box; +} + +html { + min-width: 20rem; + background: var(--bg); +} + +body { + min-height: 100vh; + margin: 0; + color: var(--text); + background: + radial-gradient(circle at 10% -10%, rgb(36 103 104 / 35%), transparent 32rem), + linear-gradient(180deg, #09141f 0%, var(--bg) 50%); +} + +button, +select { + font: inherit; +} + +button, +select, +a { + -webkit-tap-highlight-color: transparent; +} + +:focus-visible { + outline: 0.2rem solid var(--focus); + outline-offset: 0.2rem; +} + +.skip-link { + position: fixed; + z-index: 10; + top: var(--space-2); + left: var(--space-2); + padding: var(--space-2) var(--space-3); + color: #071019; + background: var(--focus); + border-radius: 0.4rem; + font-weight: 800; + transform: translateY(-200%); +} + +.skip-link:focus { + transform: translateY(0); +} + +.site-header, +main, +footer { + width: min(80rem, calc(100% - 2rem)); + margin-inline: auto; +} + +.site-header { + display: flex; + align-items: end; + justify-content: space-between; + gap: var(--space-5); + padding-block: var(--space-6) var(--space-5); +} + +h1, +h2, +h3, +p { + margin-top: 0; +} + +h1 { + max-width: 18ch; + margin-bottom: var(--space-2); + font-size: clamp(2rem, 6vw, 4.25rem); + line-height: 1.02; + letter-spacing: -0.045em; +} + +h2 { + margin-bottom: var(--space-1); + font-size: clamp(1.35rem, 3vw, 2rem); + line-height: 1.2; +} + +h3 { + margin-bottom: var(--space-1); + font-size: 1.1rem; +} + +.eyebrow { + margin-bottom: var(--space-1); + color: var(--accent-strong); + font-size: 0.78rem; + font-weight: 800; + letter-spacing: 0.14em; + text-transform: uppercase; +} + +.lede { + max-width: 65ch; + margin-bottom: 0; + color: var(--muted); + font-size: 1.05rem; +} + +.connection-state { + display: inline-flex; + min-height: 2.75rem; + align-items: center; + gap: var(--space-2); + padding: var(--space-2) var(--space-3); + border: 1px solid var(--border); + border-radius: 999px; + background: rgb(16 28 40 / 88%); + color: var(--muted); + font-weight: 700; + white-space: nowrap; +} + +.status-dot { + width: 0.7rem; + height: 0.7rem; + border: 2px solid currentColor; + border-radius: 50%; + background: currentColor; +} + +body[data-stream-state="connected"] .connection-state { + color: var(--accent-strong); +} + +body[data-stream-state="reconnecting"] .connection-state, +body[data-stream-state="polling"] .connection-state, +body[data-stream-state="reset"] .connection-state { + color: var(--warning); +} + +body[data-stream-state="error"] .connection-state { + color: var(--danger); +} + +main { + display: grid; + gap: var(--space-4); +} + +.filter-panel, +.summary-panel, +.results-panel { + border: 1px solid var(--border); + border-radius: var(--radius); + background: rgb(16 28 40 / 94%); + box-shadow: 0 1rem 3rem rgb(0 0 0 / 18%); +} + +.filter-panel, +.results-panel { + padding: clamp(1rem, 4vw, 2rem); +} + +.section-heading { + display: flex; + align-items: start; + justify-content: space-between; + gap: var(--space-4); + margin-bottom: var(--space-4); +} + +.boundary-note, +#results-summary { + max-width: 34rem; + margin-bottom: 0; + color: var(--muted); +} + +.filter-grid { + display: grid; + grid-template-columns: repeat(3, minmax(0, 1fr)); + gap: var(--space-3); +} + +label { + display: grid; + gap: 0.35rem; + color: var(--muted); + font-size: 0.9rem; + font-weight: 750; +} + +select { + width: 100%; + min-height: 2.75rem; + padding: 0.6rem 2.4rem 0.6rem 0.75rem; + border: 1px solid #587189; + border-radius: 0.45rem; + color: var(--text); + background: #0b1722; +} + +select:disabled { + color: #9fb0bd; + border-style: dashed; + opacity: 1; +} + +.button-row { + display: flex; + flex-wrap: wrap; + gap: var(--space-2); + margin-top: var(--space-4); +} + +.button { + min-height: 2.75rem; + padding: 0.65rem 1rem; + border: 1px solid #6a8298; + border-radius: 0.45rem; + color: var(--text); + background: var(--surface-raised); + cursor: pointer; + font-weight: 800; +} + +.button:hover { + border-color: var(--accent-strong); + background: #21384a; +} + +.button:active { + transform: translateY(1px); +} + +.button.primary { + color: var(--accent-ink); + border-color: var(--accent); + background: var(--accent); +} + +.button.primary:hover { + background: var(--accent-strong); +} + +.form-error { + margin: var(--space-3) 0 0; + color: var(--danger); + font-weight: 750; +} + +.summary-panel { + padding: 0; + overflow: hidden; +} + +.summary-grid { + display: grid; + grid-template-columns: repeat(4, minmax(0, 1fr)); + margin: 0; +} + +.summary-grid > div { + min-width: 0; + padding: var(--space-3) var(--space-4); + border-right: 1px solid var(--border); +} + +.summary-grid > div:last-child { + border-right: 0; +} + +.summary-grid dt { + color: var(--muted); + font-size: 0.78rem; + font-weight: 700; +} + +.summary-grid dd { + margin: 0.25rem 0 0; + overflow-wrap: anywhere; + font-size: 1.05rem; + font-weight: 800; +} + +.results-heading { + align-items: end; +} + +.session-grid { + display: grid; + grid-template-columns: repeat(auto-fill, minmax(min(100%, 19rem), 1fr)); + gap: var(--space-3); +} + +.session-card { + min-width: 0; + padding: var(--space-3); + border: 1px solid #496177; + border-radius: 0.6rem; + background: #0b1722; +} + +.card-heading { + display: flex; + align-items: start; + justify-content: space-between; + gap: var(--space-2); +} + +.card-heading h3 { + overflow-wrap: anywhere; +} + +.region-badge { + flex: 0 0 auto; + padding: 0.15rem 0.5rem; + border: 1px solid #597187; + border-radius: 999px; + color: var(--muted); + font-size: 0.75rem; + font-weight: 750; +} + +.session-detail { + display: grid; + grid-template-columns: minmax(5rem, auto) 1fr; + gap: 0.25rem var(--space-2); + margin: var(--space-3) 0 0; + font-size: 0.9rem; +} + +.session-detail dt { + color: var(--muted); +} + +.session-detail dd { + min-width: 0; + margin: 0; + overflow-wrap: anywhere; +} + +.capacity-row { + margin-top: var(--space-3); +} + +.capacity-copy { + display: flex; + justify-content: space-between; + gap: var(--space-2); + margin-bottom: 0.35rem; + color: var(--muted); + font-size: 0.85rem; +} + +meter { + width: 100%; + height: 0.65rem; + accent-color: var(--accent); +} + +.metadata-list { + display: flex; + flex-wrap: wrap; + gap: 0.35rem; + margin: var(--space-3) 0 0; + padding: 0; + list-style: none; +} + +.metadata-list li { + max-width: 100%; + padding: 0.2rem 0.45rem; + overflow-wrap: anywhere; + border-radius: 0.3rem; + color: #d6e4ed; + background: #1b2b39; + font-size: 0.78rem; +} + +.updated-age { + margin: var(--space-3) 0 0; + color: var(--muted); + font-size: 0.78rem; +} + +.empty-state { + padding: var(--space-6) var(--space-3); + text-align: center; + border: 1px dashed #587189; + border-radius: 0.6rem; + color: var(--muted); +} + +.empty-state h3 { + color: var(--text); +} + +footer { + padding-block: var(--space-5); + color: var(--muted); + font-size: 0.85rem; + text-align: center; +} + +.visually-hidden { + position: absolute; + width: 1px; + height: 1px; + padding: 0; + overflow: hidden; + clip: rect(0, 0, 0, 0); + white-space: nowrap; + border: 0; +} + +[hidden] { + display: none !important; +} + +@media (max-width: 56rem) { + .site-header, + .section-heading { + align-items: start; + flex-direction: column; + } + + .filter-grid, + .summary-grid { + grid-template-columns: repeat(2, minmax(0, 1fr)); + } + + .summary-grid > div:nth-child(2) { + border-right: 0; + } + + .summary-grid > div:nth-child(-n + 2) { + border-bottom: 1px solid var(--border); + } +} + +@media (max-width: 36rem) { + .site-header, + main, + footer { + width: min(100% - 1rem, 80rem); + } + + .site-header { + padding-block: var(--space-5) var(--space-4); + } + + .filter-grid, + .summary-grid { + grid-template-columns: 1fr; + } + + .summary-grid > div { + border-right: 0; + border-bottom: 1px solid var(--border); + } + + .summary-grid > div:last-child { + border-bottom: 0; + } + + .button { + width: 100%; + } +} + +@media (prefers-reduced-motion: reduce) { + *, + *::before, + *::after { + scroll-behavior: auto !important; + transition-duration: 0.01ms !important; + } +} diff --git a/src/FinalFactory.Rendezvous.Server/Diagnostics/DiagnosticDashboardEndpoints.cs b/src/FinalFactory.Rendezvous.Server/Diagnostics/DiagnosticDashboardEndpoints.cs new file mode 100644 index 0000000..7c21f29 --- /dev/null +++ b/src/FinalFactory.Rendezvous.Server/Diagnostics/DiagnosticDashboardEndpoints.cs @@ -0,0 +1,111 @@ +using System.Reflection; +using FinalFactory.Rendezvous.Contracts; +using Microsoft.AspNetCore.Mvc; +using Microsoft.Extensions.Options; + +namespace FinalFactory.Rendezvous.Server.Diagnostics; + +internal static class DiagnosticDashboardEndpoints +{ + private const string ContentSecurityPolicy = + "default-src 'none'; base-uri 'none'; connect-src 'self'; " + + "font-src 'self'; form-action 'none'; frame-ancestors 'none'; " + + "img-src 'self'; manifest-src 'none'; object-src 'none'; " + + "script-src 'self'; style-src 'self'"; + private static readonly byte[] Index = ReadAsset("index.html"); + private static readonly byte[] Script = ReadAsset("app.mjs"); + private static readonly byte[] Styles = ReadAsset("styles.css"); + + public static IEndpointRouteBuilder MapDiagnosticDashboardEndpoints( + this IEndpointRouteBuilder endpoints) + { + RouteGroupBuilder dashboard = endpoints.MapGroup("/diagnostics") + .ExcludeFromDescription(); + dashboard.MapGet("", ServeIndex); + dashboard.MapGet("/app.mjs", ServeScript); + dashboard.MapGet("/styles.css", ServeStyles); + dashboard.MapGet("/config.json", ServeConfiguration); + return endpoints; + } + + private static IResult ServeIndex( + HttpContext context, + [FromServices] IOptions configured) => + ServeAsset(context, configured.Value, Index, "text/html; charset=utf-8"); + + private static IResult ServeScript( + HttpContext context, + [FromServices] IOptions configured) => + ServeAsset(context, configured.Value, Script, "text/javascript; charset=utf-8"); + + private static IResult ServeStyles( + HttpContext context, + [FromServices] IOptions configured) => + ServeAsset(context, configured.Value, Styles, "text/css; charset=utf-8"); + + private static IResult ServeConfiguration( + HttpContext context, + [FromServices] IOptions configured) + { + DiagnosticDashboardOptions options = configured.Value; + if (!options.Enabled) + { + return Results.NotFound(); + } + + ApplySecurityHeaders(context.Response); + return Results.Json(new + { + contractVersion = ContractLimits.ContractVersion, + pollIntervalSeconds = options.PollIntervalSeconds, + maximumRenderedSessions = options.MaximumRenderedSessions, + scopes = options.Scopes.Select(static scope => new + { + gameId = scope.GameId, + environmentId = scope.EnvironmentId, + protocolVersions = scope.ProtocolVersions, + regions = scope.Regions, + visibility = "public", + }), + }); + } + + private static IResult ServeAsset( + HttpContext context, + DiagnosticDashboardOptions options, + byte[] content, + string contentType) + { + if (!options.Enabled) + { + return Results.NotFound(); + } + + ApplySecurityHeaders(context.Response); + return Results.Bytes(content, contentType); + } + + private static void ApplySecurityHeaders(HttpResponse response) + { + response.Headers.CacheControl = "no-store"; + response.Headers["Content-Security-Policy"] = ContentSecurityPolicy; + response.Headers["X-Content-Type-Options"] = "nosniff"; + response.Headers["X-Frame-Options"] = "DENY"; + response.Headers["Cross-Origin-Opener-Policy"] = "same-origin"; + response.Headers["Cross-Origin-Resource-Policy"] = "same-origin"; + response.Headers["Permissions-Policy"] = + "camera=(), geolocation=(), microphone=(), payment=(), usb=()"; + response.Headers["Referrer-Policy"] = "no-referrer"; + } + + private static byte[] ReadAsset(string fileName) + { + Assembly assembly = typeof(DiagnosticDashboardEndpoints).Assembly; + string resourceName = $"FinalFactory.Rendezvous.Server.Diagnostics.Assets.{fileName}"; + using Stream source = assembly.GetManifestResourceStream(resourceName) + ?? throw new InvalidOperationException($"Missing embedded dashboard asset {fileName}."); + using MemoryStream destination = new(); + source.CopyTo(destination); + return destination.ToArray(); + } +} diff --git a/src/FinalFactory.Rendezvous.Server/Diagnostics/DiagnosticDashboardOptions.cs b/src/FinalFactory.Rendezvous.Server/Diagnostics/DiagnosticDashboardOptions.cs new file mode 100644 index 0000000..7bc244b --- /dev/null +++ b/src/FinalFactory.Rendezvous.Server/Diagnostics/DiagnosticDashboardOptions.cs @@ -0,0 +1,81 @@ +using FinalFactory.Rendezvous.Contracts; + +namespace FinalFactory.Rendezvous.Server.Diagnostics; + +internal sealed record DiagnosticDashboardOptions +{ + public const string SectionName = "Rendezvous:Diagnostics"; + + public bool Enabled { get; init; } + public int PollIntervalSeconds { get; init; } = 10; + public int MaximumRenderedSessions { get; init; } = 100; + public DiagnosticDashboardScope[] Scopes { get; init; } = []; + + public IReadOnlyList Validate() + { + List errors = []; + if (PollIntervalSeconds is < 5 or > 60) + { + errors.Add($"{SectionName}:PollIntervalSeconds must be between 5 and 60."); + } + if (MaximumRenderedSessions is < 10 or > 500) + { + errors.Add($"{SectionName}:MaximumRenderedSessions must be between 10 and 500."); + } + if (!Enabled) + { + return errors; + } + if (Scopes is null || Scopes.Length is < 1 or > 32) + { + errors.Add($"{SectionName}:Scopes must contain between 1 and 32 allow-listed scopes when enabled."); + return errors; + } + + HashSet identities = new(StringComparer.Ordinal); + foreach (DiagnosticDashboardScope? scope in Scopes) + { + if (scope is null) + { + errors.Add($"{SectionName}:Scopes cannot contain null entries."); + continue; + } + string gameId = scope.GameId ?? string.Empty; + string environmentId = scope.EnvironmentId ?? string.Empty; + uint[] protocolVersions = scope.ProtocolVersions ?? []; + string[] regions = scope.Regions ?? []; + if (!GameId.TryParse(gameId, out _) + || !EnvironmentId.TryParse(environmentId, out _)) + { + errors.Add($"{SectionName}:Scopes contains an invalid game or environment identifier."); + } + if (protocolVersions.Length is < 1 or > 16 + || protocolVersions.Any(static version => version == 0) + || protocolVersions.Distinct().Count() != protocolVersions.Length) + { + errors.Add($"{SectionName}:Scopes protocol versions must contain 1-16 unique positive values."); + } + if (regions.Length is < 1 or > 16 + || regions.Any(static region => !RegionId.TryParse(region ?? string.Empty, out _)) + || regions.Distinct(StringComparer.Ordinal).Count() != regions.Length) + { + errors.Add($"{SectionName}:Scopes regions must contain 1-16 unique valid identifiers."); + } + + string identity = $"{gameId}\n{environmentId}"; + if (!identities.Add(identity)) + { + errors.Add($"{SectionName}:Scopes contains a duplicate game/environment pair."); + } + } + return errors; + } +} + +internal sealed record DiagnosticDashboardScope +{ + public string GameId { get; init; } = string.Empty; + public string EnvironmentId { get; init; } = string.Empty; + public uint[] ProtocolVersions { get; init; } = []; + public string[] Regions { get; init; } = []; +} diff --git a/src/FinalFactory.Rendezvous.Server/FinalFactory.Rendezvous.Server.csproj b/src/FinalFactory.Rendezvous.Server/FinalFactory.Rendezvous.Server.csproj index ddb7ca0..71b563b 100644 --- a/src/FinalFactory.Rendezvous.Server/FinalFactory.Rendezvous.Server.csproj +++ b/src/FinalFactory.Rendezvous.Server/FinalFactory.Rendezvous.Server.csproj @@ -53,6 +53,8 @@ + + @@ -60,6 +62,7 @@ + @@ -88,4 +91,15 @@ + + + FinalFactory.Rendezvous.Server.Diagnostics.Assets.app.mjs + + + FinalFactory.Rendezvous.Server.Diagnostics.Assets.index.html + + + FinalFactory.Rendezvous.Server.Diagnostics.Assets.styles.css + + diff --git a/src/FinalFactory.Rendezvous.Server/Observability/PrometheusMetricsEndpoint.cs b/src/FinalFactory.Rendezvous.Server/Observability/PrometheusMetricsEndpoint.cs new file mode 100644 index 0000000..aa891d5 --- /dev/null +++ b/src/FinalFactory.Rendezvous.Server/Observability/PrometheusMetricsEndpoint.cs @@ -0,0 +1,148 @@ +using System.Security.Cryptography; +using System.Text; +using FinalFactory.Rendezvous.Server.Provisioning; + +namespace FinalFactory.Rendezvous.Server.Observability; + +internal sealed record PrometheusMetricsOptions +{ + public const string SectionName = "Rendezvous:Metrics"; + + public bool Enabled { get; init; } + public string BearerTokenSecretReference { get; init; } = string.Empty; + + public IReadOnlyList Validate() + { + if (!Enabled) + { + return []; + } + string reference = BearerTokenSecretReference ?? string.Empty; + bool environmentReference = reference.StartsWith("env:", StringComparison.Ordinal) + && reference.Length > "env:".Length; + bool absoluteFileReference = reference.StartsWith("file:", StringComparison.Ordinal) + && Path.IsPathFullyQualified(reference["file:".Length..]); + return reference.Length is < 5 or > 512 + || !(environmentReference || absoluteFileReference) + ? [$"{SectionName}:BearerTokenSecretReference must be a bounded env: or absolute file: secret reference when metrics are enabled."] + : []; + } +} + +internal sealed class MetricsAccessCredential : IDisposable +{ + private byte[]? _token; + + private MetricsAccessCredential(byte[] token) => _token = token; + + public static bool TryCreate( + PrometheusMetricsOptions options, + ISecretProvider secrets, + out MetricsAccessCredential? credential) + { + credential = null; + if (!options.Enabled + || !secrets.TryGetSecret(options.BearerTokenSecretReference, out SecretMaterial? material) + || material is null) + { + return false; + } + + using (material) + { + byte[] bytes = material.CopyBytes(); + int length = bytes.Length; + while (length > 0 && bytes[length - 1] is (byte)'\r' or (byte)'\n') + { + length--; + } + bool valid = length is >= 32 and <= 128 + && bytes.AsSpan(0, length).IndexOfAnyExceptInRange((byte)0x21, (byte)0x7e) < 0; + if (!valid) + { + CryptographicOperations.ZeroMemory(bytes); + return false; + } + + byte[] token = bytes.AsSpan(0, length).ToArray(); + CryptographicOperations.ZeroMemory(bytes); + credential = new(token); + return true; + } + } + + public bool Authorizes(HttpRequest request) + { + byte[]? expected = _token; + string authorization = request.Headers.Authorization.ToString(); + const string prefix = "Bearer "; + if (expected is null + || !authorization.StartsWith(prefix, StringComparison.Ordinal) + || authorization.Length - prefix.Length is < 32 or > 128) + { + return false; + } + + byte[] supplied = Encoding.UTF8.GetBytes(authorization[prefix.Length..]); + try + { + return supplied.Length == expected.Length + && CryptographicOperations.FixedTimeEquals(supplied, expected); + } + finally + { + CryptographicOperations.ZeroMemory(supplied); + } + } + + public void Dispose() + { + byte[]? token = Interlocked.Exchange(ref _token, null); + if (token is not null) + { + CryptographicOperations.ZeroMemory(token); + } + } + + public override string ToString() => "[MetricsAccessCredential: REDACTED]"; +} + +internal static class PrometheusMetricsEndpoint +{ + public static IEndpointRouteBuilder MapPrometheusMetricsEndpoint( + this IEndpointRouteBuilder endpoints, + PrometheusMetricsOptions options, + MetricsAccessCredential? credential) + { + endpoints.MapGet("/metrics", (HttpContext context, RendezvousTelemetry telemetry) => + Export(context, telemetry, options, credential)) + .WithName("MetricsScrape") + .ExcludeFromDescription(); + return endpoints; + } + + private static IResult Export( + HttpContext context, + RendezvousTelemetry telemetry, + PrometheusMetricsOptions options, + MetricsAccessCredential? credential) + { + if (!options.Enabled) + { + return Results.NotFound(); + } + if (credential is null || !credential.Authorizes(context.Request)) + { + telemetry.RecordMetricsScrape("rejected"); + return Results.NotFound(); + } + + telemetry.RecordMetricsScrape("accepted"); + context.Response.Headers.CacheControl = "no-store"; + context.Response.Headers["X-Content-Type-Options"] = "nosniff"; + return Results.Text( + telemetry.RenderPrometheus(), + "text/plain; version=0.0.4; charset=utf-8", + Encoding.UTF8); + } +} diff --git a/src/FinalFactory.Rendezvous.Server/Observability/RendezvousTelemetry.cs b/src/FinalFactory.Rendezvous.Server/Observability/RendezvousTelemetry.cs index 2eb2810..5cb99ec 100644 --- a/src/FinalFactory.Rendezvous.Server/Observability/RendezvousTelemetry.cs +++ b/src/FinalFactory.Rendezvous.Server/Observability/RendezvousTelemetry.cs @@ -1,5 +1,10 @@ +using System.Collections.Concurrent; using System.Diagnostics; using System.Diagnostics.Metrics; +using System.Globalization; +using System.Text; +using FinalFactory.Rendezvous.Server.Browser; +using FinalFactory.Rendezvous.Server.Provisioning; using FinalFactory.Rendezvous.Server.State; namespace FinalFactory.Rendezvous.Server.Observability; @@ -10,6 +15,10 @@ internal sealed class RendezvousTelemetry : IDisposable public const string ActivitySourceName = "FinalFactory.Rendezvous.Server"; private readonly InMemoryEphemeralRendezvousStore _store; + private readonly SessionChangeJournal? _sessionChanges; + private readonly ProvisioningRuntime? _provisioning; + private readonly ConcurrentDictionary _prometheusCounters = new(); + private readonly ConcurrentDictionary _prometheusHistograms = new(); private readonly Meter _meter = new(MeterName, "1.0.0"); private readonly ActivitySource _activities = new(ActivitySourceName, "1.0.0"); private readonly Counter _httpRequests; @@ -22,9 +31,14 @@ internal sealed class RendezvousTelemetry : IDisposable private readonly Counter _operatorAuthentication; private readonly Histogram _pairingLatency; - public RendezvousTelemetry(InMemoryEphemeralRendezvousStore store) + public RendezvousTelemetry( + InMemoryEphemeralRendezvousStore store, + SessionChangeJournal? sessionChanges = null, + ProvisioningRuntime? provisioning = null) { _store = store; + _sessionChanges = sessionChanges; + _provisioning = provisioning; _httpRequests = _meter.CreateCounter("rendezvous.http.requests"); _httpDuration = _meter.CreateHistogram( "rendezvous.http.duration", @@ -75,9 +89,21 @@ internal sealed class RendezvousTelemetry : IDisposable }; _httpRequests.Add(1, tags); _httpDuration.Record(elapsedMilliseconds, tags); + IncrementPrometheus( + "rendezvous_http_requests_total", + Labels(("operation", operation), ("status_code", statusCode.ToString(CultureInfo.InvariantCulture)))); + ObservePrometheus( + "rendezvous_http_duration_milliseconds", + elapsedMilliseconds, + Labels(("operation", operation), ("status_code", statusCode.ToString(CultureInfo.InvariantCulture)))); } - public void RecordUdp(string operation, string result, double elapsedMilliseconds) + public void RecordUdp( + string operation, + string result, + double elapsedMilliseconds, + int receivedBytes = 0, + int responseBudgetBytes = 0) { TagList tags = new() { @@ -86,41 +112,315 @@ internal sealed class RendezvousTelemetry : IDisposable }; _udpResults.Add(1, tags); _udpDuration.Record(elapsedMilliseconds, tags); + IncrementPrometheus( + "rendezvous_udp_results_total", + Labels(("operation", operation), ("result", result))); + ObservePrometheus( + "rendezvous_udp_duration_milliseconds", + elapsedMilliseconds, + Labels(("operation", operation), ("result", result))); + AddPrometheus( + "rendezvous_udp_received_bytes_total", + Math.Max(0, receivedBytes), + Labels(("operation", operation))); + AddPrometheus( + "rendezvous_udp_response_budget_bytes_total", + Math.Max(0, responseBudgetBytes), + Labels(("operation", operation))); } - public void RecordLimiterDrop(string transport, string partition) => + public void RecordLimiterDrop(string transport, string partition) + { _limiterDrops.Add(1, new TagList { { "transport", transport }, { "partition", partition }, }); + IncrementPrometheus( + "rendezvous_limiter_drops_total", + Labels(("transport", transport), ("partition", partition))); + } - public void RecordAudit(string action, string result) => + public void RecordAudit(string action, string result) + { _auditEvents.Add(1, new TagList { { "action", action }, { "result", result }, }); + IncrementPrometheus( + "rendezvous_audit_events_total", + Labels(("action", action), ("result", result))); + } - public void RecordConnectionOutcome(string outcome, string elapsedBucket) => + public void RecordConnectionOutcome(string outcome, string elapsedBucket) + { _connectionOutcomes.Add(1, new TagList { { "outcome", outcome }, { "elapsed_bucket", elapsedBucket }, }); + IncrementPrometheus( + "rendezvous_connection_outcomes_total", + Labels(("outcome", outcome), ("elapsed_bucket", elapsedBucket))); + } - public void RecordOperatorAuthentication(string result) => + public void RecordOperatorAuthentication(string result) + { _operatorAuthentication.Add(1, new TagList { { "result", result }, }); + IncrementPrometheus( + "rendezvous_operator_authentication_total", + Labels(("result", result))); + } - public void RecordPairingLatency(double elapsedMilliseconds) => + public void RecordPairingLatency(double elapsedMilliseconds) + { _pairingLatency.Record(elapsedMilliseconds); + ObservePrometheus("rendezvous_pairing_latency_milliseconds", elapsedMilliseconds, string.Empty); + } + + public void RecordMetricsScrape(string result) => IncrementPrometheus( + "rendezvous_metrics_scrapes_total", + Labels(("result", result))); + + public string RenderPrometheus() + { + StringBuilder output = new(16 * 1024); + foreach (IGrouping> family in _prometheusCounters + .OrderBy(static item => item.Key.Name, StringComparer.Ordinal) + .ThenBy(static item => item.Key.Labels, StringComparer.Ordinal) + .GroupBy(static item => item.Key.Name, StringComparer.Ordinal)) + { + output.Append("# TYPE ").Append(family.Key).Append(" counter\n"); + foreach (KeyValuePair series in family) + { + AppendValue(output, series.Key.Name, series.Key.Labels, series.Value); + } + } + foreach (IGrouping> family in + _prometheusHistograms + .OrderBy(static item => item.Key.Name, StringComparer.Ordinal) + .ThenBy(static item => item.Key.Labels, StringComparer.Ordinal) + .GroupBy(static item => item.Key.Name, StringComparer.Ordinal)) + { + output.Append("# TYPE ").Append(family.Key).Append(" histogram\n"); + foreach (KeyValuePair series in family) + { + series.Value.Append(output, series.Key.Name, series.Key.Labels); + } + } + + EphemeralStoreSnapshot store = _store.GetMetricsSnapshot(); + AppendGauge(output, "rendezvous_store_active_listings", store.ActiveListings); + AppendGauge(output, "rendezvous_store_fresh_presence_bindings", store.FreshPresenceBindings); + AppendGauge( + output, + "rendezvous_store_awaiting_presence_listings", + Math.Max(0, store.ActiveListings - store.FreshPresenceBindings)); + AppendGauge(output, "rendezvous_store_active_leases", store.ActiveListings); + AppendGauge(output, "rendezvous_store_active_attempts", store.ActiveJoinAttempts); + AppendGauge(output, "rendezvous_queue_depth", store.ActiveJoinAttempts); + AppendGauge(output, "rendezvous_store_replay_markers", store.ReplayMarkers); + AppendGauge(output, "rendezvous_store_available", store.IsAvailable ? 1 : 0); + AppendGauge(output, "rendezvous_store_draining", store.IsDraining ? 1 : 0); + AppendCounter(output, "rendezvous_store_expiry_churn_total", store.ExpiryChurn); + if (_sessionChanges is not null) + { + AppendGauge(output, "rendezvous_browser_sse_subscribers", _sessionChanges.SubscriberCount); + AppendGauge(output, "rendezvous_browser_sse_tenants", _sessionChanges.SubscribedTenantCount); + AppendGauge(output, "rendezvous_browser_replay_entries", _sessionChanges.ReplayCount); + } + AppendProcessMetrics(output); + AppendSigningKeyMetrics(output); + return output.ToString(); + } + + private void AppendSigningKeyMetrics(StringBuilder output) + { + if (_provisioning is null) + { + return; + } + DateTimeOffset now = DateTimeOffset.UtcNow; + SigningKeyStatus[] statuses = _provisioning.SigningKeys.GetStatuses(now).ToArray(); + output.Append("# TYPE rendezvous_signing_keys gauge\n"); + foreach (IGrouping state in statuses.GroupBy( + static status => status.Status, + StringComparer.Ordinal)) + { + AppendValue( + output, + "rendezvous_signing_keys", + Labels(("state", state.Key)), + state.Count()); + } + double seconds = statuses + .Where(static status => status.Status == "signing") + .Select(status => Math.Max(0, (status.SignUntil - now).TotalSeconds)) + .DefaultIfEmpty(0) + .Min(); + AppendGauge(output, "rendezvous_signing_key_sign_seconds_remaining", seconds); + } + + private static void AppendProcessMetrics(StringBuilder output) + { + using Process process = Process.GetCurrentProcess(); + process.Refresh(); + AppendCounter(output, "process_cpu_seconds_total", process.TotalProcessorTime.TotalSeconds); + AppendGauge(output, "process_resident_memory_bytes", process.WorkingSet64); + AppendGauge(output, "process_virtual_memory_bytes", process.VirtualMemorySize64); + AppendGauge(output, "process_threads", process.Threads.Count); + try + { + AppendGauge( + output, + "process_open_file_descriptors", + Directory.EnumerateFileSystemEntries("/proc/self/fd").Count()); + } + catch (Exception exception) when (exception is IOException + or UnauthorizedAccessException) + { + } + AppendGauge(output, "dotnet_gc_heap_size_bytes", GC.GetTotalMemory(forceFullCollection: false)); + output.Append("# TYPE dotnet_gc_collections_total counter\n"); + AppendValue(output, "dotnet_gc_collections_total", Labels(("generation", "0")), GC.CollectionCount(0)); + AppendValue(output, "dotnet_gc_collections_total", Labels(("generation", "1")), GC.CollectionCount(1)); + AppendValue(output, "dotnet_gc_collections_total", Labels(("generation", "2")), GC.CollectionCount(2)); + AppendGauge(output, "dotnet_thread_pool_threads", ThreadPool.ThreadCount); + ThreadPool.GetAvailableThreads(out int workerThreads, out int completionThreads); + AppendGauge(output, "dotnet_thread_pool_available_worker_threads", workerThreads); + AppendGauge(output, "dotnet_thread_pool_available_completion_threads", completionThreads); + } + + private void IncrementPrometheus(string name, string labels) => + _prometheusCounters.AddOrUpdate(new(name, labels), 1, static (_, current) => current + 1); + + private void AddPrometheus(string name, long value, string labels) + { + if (value <= 0) + { + return; + } + _prometheusCounters.AddOrUpdate(new(name, labels), value, (_, current) => current + value); + } + + private void ObservePrometheus(string name, double value, string labels) => + _prometheusHistograms.GetOrAdd(new(name, labels), static _ => new()).Observe(value); + + private static string Labels(params (string Name, string Value)[] labels) + { + if (labels.Length == 0) + { + return string.Empty; + } + return "{" + string.Join(',', labels.Select(static label => + $"{label.Name}=\"{EscapeLabel(NormalizeLabel(label.Value))}\"")) + "}"; + } + + private static string NormalizeLabel(string value) + { + if (string.IsNullOrWhiteSpace(value) || value.Length > 64) + { + return "other"; + } + return value.All(static character => char.IsAsciiLetterOrDigit(character) + || character is '-' or '_' or '.') + ? value + : "other"; + } + + private static string EscapeLabel(string value) => value + .Replace("\\", "\\\\", StringComparison.Ordinal) + .Replace("\"", "\\\"", StringComparison.Ordinal) + .Replace("\n", "\\n", StringComparison.Ordinal); + + private static void AppendCounter( + StringBuilder output, + string name, + double value, + string labels = "") + { + output.Append("# TYPE ").Append(name).Append(" counter\n"); + AppendValue(output, name, labels, value); + } + + private static void AppendGauge( + StringBuilder output, + string name, + double value, + string labels = "") + { + output.Append("# TYPE ").Append(name).Append(" gauge\n"); + AppendValue(output, name, labels, value); + } + + private static void AppendValue(StringBuilder output, string name, string labels, double value) => + output.Append(name) + .Append(labels) + .Append(' ') + .Append(value.ToString("R", CultureInfo.InvariantCulture)) + .Append('\n'); public void Dispose() { _activities.Dispose(); _meter.Dispose(); } + + private readonly record struct MetricSeriesKey(string Name, string Labels); + + private sealed class PrometheusHistogram + { + private static readonly double[] Bounds = [1, 5, 10, 25, 50, 100, 250, 500, 1000, 5000]; + private readonly object _gate = new(); + private readonly long[] _buckets = new long[Bounds.Length]; + private long _count; + private double _sum; + + public void Observe(double value) + { + if (!double.IsFinite(value) || value < 0) + { + return; + } + lock (_gate) + { + _count++; + _sum += value; + for (int index = 0; index < Bounds.Length; index++) + { + if (value <= Bounds[index]) + { + _buckets[index]++; + } + } + } + } + + public void Append(StringBuilder output, string name, string labels) + { + lock (_gate) + { + for (int index = 0; index < Bounds.Length; index++) + { + AppendValue( + output, + name + "_bucket", + AddLabel(labels, "le", Bounds[index].ToString("R", CultureInfo.InvariantCulture)), + _buckets[index]); + } + AppendValue(output, name + "_bucket", AddLabel(labels, "le", "+Inf"), _count); + AppendValue(output, name + "_sum", labels, _sum); + AppendValue(output, name + "_count", labels, _count); + } + } + + private static string AddLabel(string labels, string name, string value) => + string.IsNullOrEmpty(labels) + ? $"{{{name}=\"{value}\"}}" + : labels[..^1] + $",{name}=\"{value}\"}}"; + } } diff --git a/src/FinalFactory.Rendezvous.Server/Program.cs b/src/FinalFactory.Rendezvous.Server/Program.cs index de25148..35bdf45 100644 --- a/src/FinalFactory.Rendezvous.Server/Program.cs +++ b/src/FinalFactory.Rendezvous.Server/Program.cs @@ -4,6 +4,7 @@ using FinalFactory.Rendezvous.Server.Abuse; using FinalFactory.Rendezvous.Server.Browser; using FinalFactory.Rendezvous.Server.ConnectionOutcomes; using FinalFactory.Rendezvous.Server.Deployment; +using FinalFactory.Rendezvous.Server.Diagnostics; using FinalFactory.Rendezvous.Server.Http; using FinalFactory.Rendezvous.Server.JoinAttempts; using FinalFactory.Rendezvous.Server.Observability; @@ -251,6 +252,37 @@ if (!builder.Environment.IsDevelopment() && !isOpenApiGeneration) } builder.Services.AddSingleton(Microsoft.Extensions.Options.Options.Create(deploymentOptions)); +DiagnosticDashboardOptions diagnosticDashboardOptions = builder.Configuration + .GetSection(DiagnosticDashboardOptions.SectionName) + .Get() ?? new DiagnosticDashboardOptions(); +IReadOnlyList diagnosticErrors = diagnosticDashboardOptions.Validate(); +if (diagnosticErrors.Count > 0) +{ + throw new DeploymentConfigurationException(diagnosticErrors); +} +builder.Services.AddSingleton( + Microsoft.Extensions.Options.Options.Create(diagnosticDashboardOptions)); +PrometheusMetricsOptions metricsOptions = builder.Configuration + .GetSection(PrometheusMetricsOptions.SectionName) + .Get() ?? new PrometheusMetricsOptions(); +IReadOnlyList metricsErrors = metricsOptions.Validate(); +if (metricsErrors.Count > 0) +{ + throw new DeploymentConfigurationException(metricsErrors); +} +builder.Services.AddSingleton(Microsoft.Extensions.Options.Options.Create(metricsOptions)); +MetricsAccessCredential? metricsCredential = null; +if (metricsOptions.Enabled && !isOpenApiGeneration) +{ + EnvironmentSecretProvider metricsSecrets = new(); + if (!MetricsAccessCredential.TryCreate(metricsOptions, metricsSecrets, out metricsCredential) + || metricsCredential is null) + { + throw new DeploymentConfigurationException( + [$"{PrometheusMetricsOptions.SectionName}:BearerTokenSecretReference did not resolve to 32-128 visible ASCII bytes."]); + } + builder.Services.AddSingleton(metricsCredential); +} builder.Services.Configure(options => options.ShutdownTimeout = TimeSpan.FromSeconds(deploymentOptions.DrainDeadlineSeconds + 10)); @@ -349,6 +381,8 @@ app.MapOpenApi(); app.MapRendezvousContractEndpoints(); app.MapOperatorEndpoints(); app.MapRendezvousHealthEndpoints(); +app.MapDiagnosticDashboardEndpoints(); +app.MapPrometheusMetricsEndpoint(metricsOptions, metricsCredential); await app.RunAsync(); diff --git a/src/FinalFactory.Rendezvous.Server/Transport/NatMediationProcessor.cs b/src/FinalFactory.Rendezvous.Server/Transport/NatMediationProcessor.cs index 1a475d8..70f9265 100644 --- a/src/FinalFactory.Rendezvous.Server/Transport/NatMediationProcessor.cs +++ b/src/FinalFactory.Rendezvous.Server/Transport/NatMediationProcessor.cs @@ -82,7 +82,9 @@ internal sealed class NatMediationProcessor( telemetry?.RecordUdp( "frozen", result.ToString(), - Stopwatch.GetElapsedTime(started).TotalMilliseconds); + Stopwatch.GetElapsedTime(started).TotalMilliseconds, + encoded.Length, + result == NatMediationResult.Introduced ? 2 * ContractLimits.UdpDatagramMaxBytes : 0); return result; } @@ -157,7 +159,7 @@ internal sealed class NatMediationProcessor( observedPublicEndpoint, token, introductionSink, - cancellationToken); + cancellationToken: cancellationToken); } internal NatMediationResult ProcessRequestAfterIngress( @@ -165,6 +167,7 @@ internal sealed class NatMediationProcessor( IPEndPoint observedPublicEndpoint, string token, INatIntroductionSink introductionSink, + int receivedBytes = 0, CancellationToken cancellationToken = default) { long started = Stopwatch.GetTimestamp(); @@ -178,7 +181,9 @@ internal sealed class NatMediationProcessor( telemetry?.RecordUdp( "litenet", result.ToString(), - Stopwatch.GetElapsedTime(started).TotalMilliseconds); + Stopwatch.GetElapsedTime(started).TotalMilliseconds, + receivedBytes, + result == NatMediationResult.Introduced ? 2 * ContractLimits.UdpDatagramMaxBytes : 0); return result; } diff --git a/src/FinalFactory.Rendezvous.Server/Transport/UdpMediatorService.cs b/src/FinalFactory.Rendezvous.Server/Transport/UdpMediatorService.cs index 2166b85..2df02b5 100644 --- a/src/FinalFactory.Rendezvous.Server/Transport/UdpMediatorService.cs +++ b/src/FinalFactory.Rendezvous.Server/Transport/UdpMediatorService.cs @@ -201,7 +201,7 @@ internal sealed class UdpMediatorService : BackgroundService && token is not null) { _ = processor.ProcessRequestAfterIngress( - claimedLocalEndpoint, endPoint, token, sink); + claimedLocalEndpoint, endPoint, token, sink, length); } // Every inbound packet is consumed here. NatPunchModule is used only for outbound introductions. diff --git a/tests/Diagnostics/diagnostic-dashboard.test.mjs b/tests/Diagnostics/diagnostic-dashboard.test.mjs new file mode 100644 index 0000000..660ac95 --- /dev/null +++ b/tests/Diagnostics/diagnostic-dashboard.test.mjs @@ -0,0 +1,135 @@ +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; +import test from "node:test"; +import { + SessionProjection, + buildBrowseUrl, + buildStreamUrl, + chooseSnapshotTransport, + safeText, +} from "../../src/FinalFactory.Rendezvous.Server/Diagnostics/Assets/app.mjs"; + +const cursor = "rvs1.test-cursor"; + +function session(id, name = `Host ${id}`) { + return { + contractVersion: 1, + listingId: `00000000-0000-0000-0000-${String(id).padStart(12, "0")}`, + gameId: "space-game", + environmentId: "smoke", + regionId: "local", + protocolVersion: 1, + buildVersion: "1.0.0", + displayName: name, + visibility: "public", + publisherTrustMode: "managedDedicated", + capacity: { currentPlayers: 1, maximumPlayers: 8 }, + metadata: { mode: "online-coop" }, + }; +} + +function event(kind, values = {}) { + return { + contractVersion: 1, + kind, + cursor: values.cursor ?? cursor, + session: values.session ?? null, + listingId: values.listingId ?? null, + }; +} + +test("snapshot and ordered deltas match the final public projection", () => { + const projection = new SessionProjection(10); + assert.equal(projection.replace([session(1)], cursor), "applied"); + assert.equal(projection.apply(event("sessionUpsert", { + cursor: `${cursor}-2`, + session: session(2), + })), "applied"); + assert.equal(projection.apply(event("sessionUpsert", { + cursor: `${cursor}-3`, + session: session(1, "Updated host"), + })), "applied"); + assert.equal(projection.apply(event("sessionRemove", { + cursor: `${cursor}-4`, + listingId: session(2).listingId, + })), "applied"); + + assert.deepEqual( + projection.sessions().map((entry) => entry.session.displayName), + ["Updated host"], + ); + assert.equal(projection.cursor, `${cursor}-4`); +}); + +test("reset, malformed events, and a partial snapshot fail closed", () => { + const projection = new SessionProjection(2); + assert.equal(projection.replace([session(1)], cursor, true), "applied"); + assert.equal(projection.apply(event("sessionRemove", { + listingId: session(1).listingId, + })), "refresh"); + assert.equal(projection.apply(event("reset")), "reset"); + assert.equal(projection.apply({ kind: "sessionUpsert" }), "invalid"); + assert.equal(projection.replace([session(1), session(2), session(3)], cursor), "overflow"); +}); + +test("bursts coalesce by listing identity and memory stays bounded", () => { + const projection = new SessionProjection(2); + assert.equal(projection.replace([], cursor), "applied"); + for (let index = 0; index < 1_000; index += 1) { + assert.equal(projection.apply(event("sessionUpsert", { + cursor: `${cursor}-${index}`, + session: session(1, `Host ${index}`), + })), "applied"); + } + assert.equal(projection.sessions().length, 1); + assert.equal(projection.sessions()[0].session.displayName, "Host 999"); + assert.equal(projection.apply(event("sessionUpsert", { session: session(2) })), "applied"); + assert.equal(projection.apply(event("sessionUpsert", { session: session(3) })), "overflow"); + assert.equal(projection.sessions().length, 2); +}); + +test("requests are fixed same-origin paths with an exact configured filter", () => { + const filter = { + gameId: "space-game", + environmentId: "smoke", + protocolVersion: 1, + regionId: "local", + excludeFull: true, + }; + const browse = buildBrowseUrl(filter); + const stream = buildStreamUrl(filter, cursor); + assert.match(browse, /^\/v1\/sessions\?/u); + assert.match(stream, /^\/v1\/sessions\/stream\?/u); + assert.match(browse, /gameId=space-game/u); + assert.match(browse, /environmentId=smoke/u); + assert.match(stream, /streamCursor=rvs1.test-cursor/u); + assert.doesNotMatch(browse, /https?:/u); + assert.doesNotMatch(stream, /https?:/u); +}); + +test("snapshot transport preserves deliberate polling and bounds partial snapshots", () => { + assert.equal(chooseSnapshotTransport(false, false), "stream"); + assert.equal(chooseSnapshotTransport(true, false), "boundedPolling"); + assert.equal(chooseSnapshotTransport(false, true), "pollingOnly"); + assert.equal(chooseSnapshotTransport(true, true), "pollingOnly"); +}); + +test("hostile display data remains literal text and no unsafe DOM sink exists", async () => { + const payload = `go`; + assert.equal(safeText(payload, 200), payload); + const source = await readFile(new URL( + "../../src/FinalFactory.Rendezvous.Server/Diagnostics/Assets/app.mjs", + import.meta.url, + ), "utf8"); + for (const forbidden of [ + "innerHTML", + "outerHTML", + "insertAdjacentHTML", + "document.write", + "eval(", + "new Function", + "window.location", + ]) { + assert.equal(source.includes(forbidden), false, `unsafe browser sink: ${forbidden}`); + } +}); diff --git a/tests/FinalFactory.Rendezvous.Tests/Diagnostics/DiagnosticDashboardTests.cs b/tests/FinalFactory.Rendezvous.Tests/Diagnostics/DiagnosticDashboardTests.cs new file mode 100644 index 0000000..9cf61d4 --- /dev/null +++ b/tests/FinalFactory.Rendezvous.Tests/Diagnostics/DiagnosticDashboardTests.cs @@ -0,0 +1,205 @@ +using System.Net; +using System.Net.Sockets; +using System.Text.Json; +using FinalFactory.Rendezvous.Server.Diagnostics; +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Hosting; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; + +namespace FinalFactory.Rendezvous.Tests.Diagnostics; + +public sealed class DiagnosticDashboardTests +{ + [Fact] + public void ConfigurationRequiresBoundedUniqueAllowListedScopes() + { + Assert.Empty(ValidOptions().Validate()); + Assert.Empty(new DiagnosticDashboardOptions().Validate()); + + DiagnosticDashboardOptions invalid = ValidOptions() with + { + PollIntervalSeconds = 1, + MaximumRenderedSessions = 501, + Scopes = + [ + new DiagnosticDashboardScope + { + GameId = "INVALID", + EnvironmentId = "smoke", + ProtocolVersions = [0, 0], + Regions = ["INVALID", "INVALID"], + }, + new DiagnosticDashboardScope + { + GameId = "INVALID", + EnvironmentId = "smoke", + ProtocolVersions = [1], + Regions = ["local"], + }, + ], + }; + + IReadOnlyList errors = invalid.Validate(); + Assert.Contains(errors, error => error.Contains("PollIntervalSeconds", StringComparison.Ordinal)); + Assert.Contains(errors, error => error.Contains("MaximumRenderedSessions", StringComparison.Ordinal)); + Assert.Contains(errors, error => error.Contains("invalid game", StringComparison.Ordinal)); + Assert.Contains(errors, error => error.Contains("protocol versions", StringComparison.Ordinal)); + Assert.Contains(errors, error => error.Contains("regions", StringComparison.Ordinal)); + Assert.Contains(errors, error => error.Contains("duplicate", StringComparison.Ordinal)); + + DiagnosticDashboardOptions nullBound = ValidOptions() with + { + Scopes = null!, + }; + Assert.Contains( + nullBound.Validate(), + error => error.Contains("Scopes must contain", StringComparison.Ordinal)); + DiagnosticDashboardOptions nullScope = ValidOptions() with + { + Scopes = [null!], + }; + Assert.Contains( + nullScope.Validate(), + error => error.Contains("null entries", StringComparison.Ordinal)); + DiagnosticDashboardOptions nullCollections = ValidOptions() with + { + Scopes = + [ + new DiagnosticDashboardScope + { + GameId = null!, + EnvironmentId = null!, + ProtocolVersions = null!, + Regions = null!, + }, + ], + }; + Assert.True(nullCollections.Validate().Count >= 3); + } + + [Fact] + public async Task DisabledDashboardHasNoPublicAssetOrConfigurationSurface() + { + await using DashboardHost host = await DashboardHost.StartAsync(new()); + + Assert.Equal(HttpStatusCode.NotFound, (await host.Client.GetAsync("diagnostics/")).StatusCode); + Assert.Equal(HttpStatusCode.NotFound, (await host.Client.GetAsync("diagnostics/app.mjs")).StatusCode); + Assert.Equal(HttpStatusCode.NotFound, (await host.Client.GetAsync("diagnostics/config.json")).StatusCode); + } + + [Fact] + public async Task EnabledDashboardServesOnlyHardenedSameOriginReadOnlyAssets() + { + await using DashboardHost host = await DashboardHost.StartAsync(ValidOptions()); + using HttpRequestMessage request = new(HttpMethod.Get, "diagnostics/"); + request.Headers.Add("Origin", "https://hostile.example"); + using HttpResponseMessage response = await host.Client.SendAsync(request); + string html = await response.Content.ReadAsStringAsync(); + + Assert.True(response.IsSuccessStatusCode, html); + Assert.Equal("text/html", response.Content.Headers.ContentType?.MediaType); + Assert.Equal("DENY", Header(response, "X-Frame-Options")); + Assert.Equal("nosniff", Header(response, "X-Content-Type-Options")); + Assert.Equal("no-referrer", Header(response, "Referrer-Policy")); + Assert.Contains("default-src 'none'", Header(response, "Content-Security-Policy"), StringComparison.Ordinal); + Assert.Contains("connect-src 'self'", Header(response, "Content-Security-Policy"), StringComparison.Ordinal); + Assert.Contains("frame-ancestors 'none'", Header(response, "Content-Security-Policy"), StringComparison.Ordinal); + Assert.False(response.Headers.Contains("Access-Control-Allow-Origin")); + Assert.Contains("", html, StringComparison.Ordinal); + Assert.Contains("href=\"#main-content\"", html, StringComparison.Ordinal); + Assert.Contains("aria-live=\"polite\"", html, StringComparison.Ordinal); + Assert.Contains("type=\"module\" src=\"/diagnostics/app.mjs\"", html, StringComparison.Ordinal); + Assert.DoesNotContain("