From be732de7c973362cbf123fd7fc6963a895f5ddb0 Mon Sep 17 00:00:00 2001 From: KyuubiYoru Date: Thu, 16 Jul 2026 13:22:16 +0200 Subject: [PATCH] feat(server): add observability and operator controls (#16) --- README.md | 6 +- docs/api/rendezvous-v1.json | 1623 ++++++++++++++++- .../observability-and-operator-runbook.md | 142 ++ docs/security/abuse-protection.md | 7 +- .../Abuse/AbuseProtectionOptions.cs | 14 + .../Abuse/AbuseProtectionService.cs | 178 +- .../Abuse/HttpAbuseProtectionMiddleware.cs | 65 +- .../ConnectionOutcomeService.cs | 7 + .../Http/RendezvousExceptionHandler.cs | 20 +- .../Observability/AuditOptions.cs | 14 + .../Observability/AuditTrail.cs | 134 ++ .../Observability/HealthEndpoints.cs | 30 + .../Observability/RendezvousReadiness.cs | 41 + .../Observability/RendezvousTelemetry.cs | 126 ++ .../Observability/TelemetryMiddleware.cs | 32 + .../Operations/OperatorEndpoints.cs | 428 +++++ .../Operations/OperatorModels.cs | 82 + .../Operations/OperatorService.cs | 81 + src/FinalFactory.Rendezvous.Server/Program.cs | 124 +- .../Provisioning/SigningKeyRing.cs | 41 +- .../State/EphemeralStateContracts.cs | 1 + .../State/InMemoryEphemeralRendezvousStore.cs | 147 +- .../Transport/NatMediationProcessor.cs | 46 +- .../appsettings.Development.json | 11 + .../appsettings.json | 9 + .../Contracts/OpenApiCompatibilityTests.cs | 42 +- .../Observability/ObservabilityTests.cs | 253 +++ .../Operations/OperatorEndpointTests.cs | 519 ++++++ .../Provisioning/PrincipalCredentialTests.cs | 1 + .../Server/Abuse/AbuseProtectionTests.cs | 63 +- .../InMemoryEphemeralRendezvousStoreTests.cs | 2 +- 31 files changed, 4149 insertions(+), 140 deletions(-) create mode 100644 docs/operations/observability-and-operator-runbook.md create mode 100644 src/FinalFactory.Rendezvous.Server/Observability/AuditOptions.cs create mode 100644 src/FinalFactory.Rendezvous.Server/Observability/AuditTrail.cs create mode 100644 src/FinalFactory.Rendezvous.Server/Observability/HealthEndpoints.cs create mode 100644 src/FinalFactory.Rendezvous.Server/Observability/RendezvousReadiness.cs create mode 100644 src/FinalFactory.Rendezvous.Server/Observability/RendezvousTelemetry.cs create mode 100644 src/FinalFactory.Rendezvous.Server/Observability/TelemetryMiddleware.cs create mode 100644 src/FinalFactory.Rendezvous.Server/Operations/OperatorEndpoints.cs create mode 100644 src/FinalFactory.Rendezvous.Server/Operations/OperatorModels.cs create mode 100644 src/FinalFactory.Rendezvous.Server/Operations/OperatorService.cs create mode 100644 tests/FinalFactory.Rendezvous.Tests/Observability/ObservabilityTests.cs create mode 100644 tests/FinalFactory.Rendezvous.Tests/Operations/OperatorEndpointTests.cs diff --git a/README.md b/README.md index c6ec371..3653ca1 100644 --- a/README.md +++ b/README.md @@ -91,6 +91,9 @@ Tenant policy, publisher/operator principals, and production key custody are defined in [game provisioning and signing-key lifecycle](docs/security/provisioning.md). Layered HTTP/UDP budgets, overload behavior, and safe operational tuning are defined in [hostile-input and overload protection](docs/security/abuse-protection.md). +Health semantics, bounded telemetry, alerting, audit privacy, and the authenticated +operator controls are defined in the +[observability and operator runbook](docs/operations/observability-and-operator-runbook.md). The scriptable host/browser/join diagnostic and its stable automation contract are documented in the [TestClient integration guide](docs/integration/test-client.md). The always-on three-party scenarios, optional Linux namespace topology, and @@ -112,7 +115,8 @@ dotnet test Rendezvous.slnx --configuration Release --no-build Run the bootstrap server with `dotnet run --project src/FinalFactory.Rendezvous.Server`. It serves HTTP health endpoints and binds the configured UDP mediator port; both stop through normal host cancellation. -The launch profile uses an ephemeral development-only signing key. Production +The launch profile uses separate ephemeral development-only publisher and operator +signing keys. Production startup fails closed until externally supplied game policies and `env:` signing key references resolve to valid key material; no reusable game secret is stored in this repository or the public Client package. diff --git a/docs/api/rendezvous-v1.json b/docs/api/rendezvous-v1.json index 99f4020..81bf889 100644 --- a/docs/api/rendezvous-v1.json +++ b/docs/api/rendezvous-v1.json @@ -14,6 +14,14 @@ "responses": { "200": { "description": "OK", + "headers": { + "X-Rendezvous-Correlation-ID": { + "description": "Safe request correlation identifier generated by the service.", + "schema": { + "type": "string" + } + } + }, "content": { "application/json": { "schema": { @@ -25,6 +33,12 @@ "429": { "description": "Too Many Requests", "headers": { + "X-Rendezvous-Correlation-ID": { + "description": "Safe request correlation identifier generated by the service.", + "schema": { + "type": "string" + } + }, "Retry-After": { "description": "Whole seconds before the caller should retry (1-60).", "schema": { @@ -53,6 +67,14 @@ "responses": { "200": { "description": "OK", + "headers": { + "X-Rendezvous-Correlation-ID": { + "description": "Safe request correlation identifier generated by the service.", + "schema": { + "type": "string" + } + } + }, "content": { "application/json": { "schema": { @@ -64,6 +86,12 @@ "429": { "description": "Too Many Requests", "headers": { + "X-Rendezvous-Correlation-ID": { + "description": "Safe request correlation identifier generated by the service.", + "schema": { + "type": "string" + } + }, "Retry-After": { "description": "Whole seconds before the caller should retry (1-60).", "schema": { @@ -81,7 +109,15 @@ } }, "503": { - "description": "Service Unavailable" + "description": "Service Unavailable", + "headers": { + "X-Rendezvous-Correlation-ID": { + "description": "Safe request correlation identifier generated by the service.", + "schema": { + "type": "string" + } + } + } } } } @@ -105,6 +141,14 @@ "responses": { "201": { "description": "Created", + "headers": { + "X-Rendezvous-Correlation-ID": { + "description": "Safe request correlation identifier generated by the service.", + "schema": { + "type": "string" + } + } + }, "content": { "application/json": { "schema": { @@ -115,6 +159,14 @@ }, "400": { "description": "Bad Request", + "headers": { + "X-Rendezvous-Correlation-ID": { + "description": "Safe request correlation identifier generated by the service.", + "schema": { + "type": "string" + } + } + }, "content": { "application/json": { "schema": { @@ -125,6 +177,14 @@ }, "413": { "description": "Payload Too Large", + "headers": { + "X-Rendezvous-Correlation-ID": { + "description": "Safe request correlation identifier generated by the service.", + "schema": { + "type": "string" + } + } + }, "content": { "application/json": { "schema": { @@ -135,6 +195,14 @@ }, "401": { "description": "Unauthorized", + "headers": { + "X-Rendezvous-Correlation-ID": { + "description": "Safe request correlation identifier generated by the service.", + "schema": { + "type": "string" + } + } + }, "content": { "application/json": { "schema": { @@ -145,6 +213,14 @@ }, "403": { "description": "Forbidden", + "headers": { + "X-Rendezvous-Correlation-ID": { + "description": "Safe request correlation identifier generated by the service.", + "schema": { + "type": "string" + } + } + }, "content": { "application/json": { "schema": { @@ -155,6 +231,14 @@ }, "409": { "description": "Conflict", + "headers": { + "X-Rendezvous-Correlation-ID": { + "description": "Safe request correlation identifier generated by the service.", + "schema": { + "type": "string" + } + } + }, "content": { "application/json": { "schema": { @@ -165,6 +249,14 @@ }, "410": { "description": "Gone", + "headers": { + "X-Rendezvous-Correlation-ID": { + "description": "Safe request correlation identifier generated by the service.", + "schema": { + "type": "string" + } + } + }, "content": { "application/json": { "schema": { @@ -176,6 +268,12 @@ "429": { "description": "Too Many Requests", "headers": { + "X-Rendezvous-Correlation-ID": { + "description": "Safe request correlation identifier generated by the service.", + "schema": { + "type": "string" + } + }, "Retry-After": { "description": "Whole seconds before the caller should retry (1-60).", "schema": { @@ -194,6 +292,14 @@ }, "503": { "description": "Service Unavailable", + "headers": { + "X-Rendezvous-Correlation-ID": { + "description": "Safe request correlation identifier generated by the service.", + "schema": { + "type": "string" + } + } + }, "content": { "application/json": { "schema": { @@ -282,6 +388,14 @@ "responses": { "200": { "description": "OK", + "headers": { + "X-Rendezvous-Correlation-ID": { + "description": "Safe request correlation identifier generated by the service.", + "schema": { + "type": "string" + } + } + }, "content": { "application/json": { "schema": { @@ -292,6 +406,14 @@ }, "400": { "description": "Bad Request", + "headers": { + "X-Rendezvous-Correlation-ID": { + "description": "Safe request correlation identifier generated by the service.", + "schema": { + "type": "string" + } + } + }, "content": { "application/json": { "schema": { @@ -303,6 +425,12 @@ "429": { "description": "Too Many Requests", "headers": { + "X-Rendezvous-Correlation-ID": { + "description": "Safe request correlation identifier generated by the service.", + "schema": { + "type": "string" + } + }, "Retry-After": { "description": "Whole seconds before the caller should retry (1-60).", "schema": { @@ -321,6 +449,14 @@ }, "503": { "description": "Service Unavailable", + "headers": { + "X-Rendezvous-Correlation-ID": { + "description": "Safe request correlation identifier generated by the service.", + "schema": { + "type": "string" + } + } + }, "content": { "application/json": { "schema": { @@ -361,6 +497,14 @@ "responses": { "200": { "description": "OK", + "headers": { + "X-Rendezvous-Correlation-ID": { + "description": "Safe request correlation identifier generated by the service.", + "schema": { + "type": "string" + } + } + }, "content": { "application/json": { "schema": { @@ -371,6 +515,14 @@ }, "400": { "description": "Bad Request", + "headers": { + "X-Rendezvous-Correlation-ID": { + "description": "Safe request correlation identifier generated by the service.", + "schema": { + "type": "string" + } + } + }, "content": { "application/json": { "schema": { @@ -381,6 +533,14 @@ }, "413": { "description": "Payload Too Large", + "headers": { + "X-Rendezvous-Correlation-ID": { + "description": "Safe request correlation identifier generated by the service.", + "schema": { + "type": "string" + } + } + }, "content": { "application/json": { "schema": { @@ -391,6 +551,14 @@ }, "401": { "description": "Unauthorized", + "headers": { + "X-Rendezvous-Correlation-ID": { + "description": "Safe request correlation identifier generated by the service.", + "schema": { + "type": "string" + } + } + }, "content": { "application/json": { "schema": { @@ -401,6 +569,14 @@ }, "403": { "description": "Forbidden", + "headers": { + "X-Rendezvous-Correlation-ID": { + "description": "Safe request correlation identifier generated by the service.", + "schema": { + "type": "string" + } + } + }, "content": { "application/json": { "schema": { @@ -411,6 +587,14 @@ }, "404": { "description": "Not Found", + "headers": { + "X-Rendezvous-Correlation-ID": { + "description": "Safe request correlation identifier generated by the service.", + "schema": { + "type": "string" + } + } + }, "content": { "application/json": { "schema": { @@ -421,6 +605,14 @@ }, "409": { "description": "Conflict", + "headers": { + "X-Rendezvous-Correlation-ID": { + "description": "Safe request correlation identifier generated by the service.", + "schema": { + "type": "string" + } + } + }, "content": { "application/json": { "schema": { @@ -431,6 +623,14 @@ }, "410": { "description": "Gone", + "headers": { + "X-Rendezvous-Correlation-ID": { + "description": "Safe request correlation identifier generated by the service.", + "schema": { + "type": "string" + } + } + }, "content": { "application/json": { "schema": { @@ -442,6 +642,12 @@ "429": { "description": "Too Many Requests", "headers": { + "X-Rendezvous-Correlation-ID": { + "description": "Safe request correlation identifier generated by the service.", + "schema": { + "type": "string" + } + }, "Retry-After": { "description": "Whole seconds before the caller should retry (1-60).", "schema": { @@ -460,6 +666,14 @@ }, "503": { "description": "Service Unavailable", + "headers": { + "X-Rendezvous-Correlation-ID": { + "description": "Safe request correlation identifier generated by the service.", + "schema": { + "type": "string" + } + } + }, "content": { "application/json": { "schema": { @@ -504,10 +718,26 @@ }, "responses": { "204": { - "description": "No Content" + "description": "No Content", + "headers": { + "X-Rendezvous-Correlation-ID": { + "description": "Safe request correlation identifier generated by the service.", + "schema": { + "type": "string" + } + } + } }, "400": { "description": "Bad Request", + "headers": { + "X-Rendezvous-Correlation-ID": { + "description": "Safe request correlation identifier generated by the service.", + "schema": { + "type": "string" + } + } + }, "content": { "application/json": { "schema": { @@ -518,6 +748,14 @@ }, "413": { "description": "Payload Too Large", + "headers": { + "X-Rendezvous-Correlation-ID": { + "description": "Safe request correlation identifier generated by the service.", + "schema": { + "type": "string" + } + } + }, "content": { "application/json": { "schema": { @@ -528,6 +766,14 @@ }, "401": { "description": "Unauthorized", + "headers": { + "X-Rendezvous-Correlation-ID": { + "description": "Safe request correlation identifier generated by the service.", + "schema": { + "type": "string" + } + } + }, "content": { "application/json": { "schema": { @@ -538,6 +784,14 @@ }, "403": { "description": "Forbidden", + "headers": { + "X-Rendezvous-Correlation-ID": { + "description": "Safe request correlation identifier generated by the service.", + "schema": { + "type": "string" + } + } + }, "content": { "application/json": { "schema": { @@ -548,6 +802,14 @@ }, "404": { "description": "Not Found", + "headers": { + "X-Rendezvous-Correlation-ID": { + "description": "Safe request correlation identifier generated by the service.", + "schema": { + "type": "string" + } + } + }, "content": { "application/json": { "schema": { @@ -559,6 +821,12 @@ "429": { "description": "Too Many Requests", "headers": { + "X-Rendezvous-Correlation-ID": { + "description": "Safe request correlation identifier generated by the service.", + "schema": { + "type": "string" + } + }, "Retry-After": { "description": "Whole seconds before the caller should retry (1-60).", "schema": { @@ -577,6 +845,14 @@ }, "503": { "description": "Service Unavailable", + "headers": { + "X-Rendezvous-Correlation-ID": { + "description": "Safe request correlation identifier generated by the service.", + "schema": { + "type": "string" + } + } + }, "content": { "application/json": { "schema": { @@ -619,10 +895,26 @@ }, "responses": { "204": { - "description": "No Content" + "description": "No Content", + "headers": { + "X-Rendezvous-Correlation-ID": { + "description": "Safe request correlation identifier generated by the service.", + "schema": { + "type": "string" + } + } + } }, "400": { "description": "Bad Request", + "headers": { + "X-Rendezvous-Correlation-ID": { + "description": "Safe request correlation identifier generated by the service.", + "schema": { + "type": "string" + } + } + }, "content": { "application/json": { "schema": { @@ -633,6 +925,14 @@ }, "413": { "description": "Payload Too Large", + "headers": { + "X-Rendezvous-Correlation-ID": { + "description": "Safe request correlation identifier generated by the service.", + "schema": { + "type": "string" + } + } + }, "content": { "application/json": { "schema": { @@ -643,6 +943,14 @@ }, "401": { "description": "Unauthorized", + "headers": { + "X-Rendezvous-Correlation-ID": { + "description": "Safe request correlation identifier generated by the service.", + "schema": { + "type": "string" + } + } + }, "content": { "application/json": { "schema": { @@ -653,6 +961,14 @@ }, "403": { "description": "Forbidden", + "headers": { + "X-Rendezvous-Correlation-ID": { + "description": "Safe request correlation identifier generated by the service.", + "schema": { + "type": "string" + } + } + }, "content": { "application/json": { "schema": { @@ -664,6 +980,12 @@ "429": { "description": "Too Many Requests", "headers": { + "X-Rendezvous-Correlation-ID": { + "description": "Safe request correlation identifier generated by the service.", + "schema": { + "type": "string" + } + }, "Retry-After": { "description": "Whole seconds before the caller should retry (1-60).", "schema": { @@ -682,6 +1004,14 @@ }, "503": { "description": "Service Unavailable", + "headers": { + "X-Rendezvous-Correlation-ID": { + "description": "Safe request correlation identifier generated by the service.", + "schema": { + "type": "string" + } + } + }, "content": { "application/json": { "schema": { @@ -749,6 +1079,14 @@ "responses": { "200": { "description": "OK", + "headers": { + "X-Rendezvous-Correlation-ID": { + "description": "Safe request correlation identifier generated by the service.", + "schema": { + "type": "string" + } + } + }, "content": { "application/json": { "schema": { @@ -759,6 +1097,14 @@ }, "400": { "description": "Bad Request", + "headers": { + "X-Rendezvous-Correlation-ID": { + "description": "Safe request correlation identifier generated by the service.", + "schema": { + "type": "string" + } + } + }, "content": { "application/json": { "schema": { @@ -769,6 +1115,14 @@ }, "404": { "description": "Not Found", + "headers": { + "X-Rendezvous-Correlation-ID": { + "description": "Safe request correlation identifier generated by the service.", + "schema": { + "type": "string" + } + } + }, "content": { "application/json": { "schema": { @@ -780,6 +1134,12 @@ "429": { "description": "Too Many Requests", "headers": { + "X-Rendezvous-Correlation-ID": { + "description": "Safe request correlation identifier generated by the service.", + "schema": { + "type": "string" + } + }, "Retry-After": { "description": "Whole seconds before the caller should retry (1-60).", "schema": { @@ -798,6 +1158,14 @@ }, "503": { "description": "Service Unavailable", + "headers": { + "X-Rendezvous-Correlation-ID": { + "description": "Safe request correlation identifier generated by the service.", + "schema": { + "type": "string" + } + } + }, "content": { "application/json": { "schema": { @@ -860,6 +1228,14 @@ "responses": { "200": { "description": "OK", + "headers": { + "X-Rendezvous-Correlation-ID": { + "description": "Safe request correlation identifier generated by the service.", + "schema": { + "type": "string" + } + } + }, "content": { "application/json": { "schema": { @@ -870,6 +1246,14 @@ }, "400": { "description": "Bad Request", + "headers": { + "X-Rendezvous-Correlation-ID": { + "description": "Safe request correlation identifier generated by the service.", + "schema": { + "type": "string" + } + } + }, "content": { "application/json": { "schema": { @@ -880,6 +1264,14 @@ }, "404": { "description": "Not Found", + "headers": { + "X-Rendezvous-Correlation-ID": { + "description": "Safe request correlation identifier generated by the service.", + "schema": { + "type": "string" + } + } + }, "content": { "application/json": { "schema": { @@ -891,6 +1283,12 @@ "429": { "description": "Too Many Requests", "headers": { + "X-Rendezvous-Correlation-ID": { + "description": "Safe request correlation identifier generated by the service.", + "schema": { + "type": "string" + } + }, "Retry-After": { "description": "Whole seconds before the caller should retry (1-60).", "schema": { @@ -909,6 +1307,14 @@ }, "503": { "description": "Service Unavailable", + "headers": { + "X-Rendezvous-Correlation-ID": { + "description": "Safe request correlation identifier generated by the service.", + "schema": { + "type": "string" + } + } + }, "content": { "application/json": { "schema": { @@ -939,6 +1345,14 @@ "responses": { "201": { "description": "Created", + "headers": { + "X-Rendezvous-Correlation-ID": { + "description": "Safe request correlation identifier generated by the service.", + "schema": { + "type": "string" + } + } + }, "content": { "application/json": { "schema": { @@ -949,6 +1363,14 @@ }, "400": { "description": "Bad Request", + "headers": { + "X-Rendezvous-Correlation-ID": { + "description": "Safe request correlation identifier generated by the service.", + "schema": { + "type": "string" + } + } + }, "content": { "application/json": { "schema": { @@ -959,6 +1381,14 @@ }, "413": { "description": "Payload Too Large", + "headers": { + "X-Rendezvous-Correlation-ID": { + "description": "Safe request correlation identifier generated by the service.", + "schema": { + "type": "string" + } + } + }, "content": { "application/json": { "schema": { @@ -969,6 +1399,14 @@ }, "404": { "description": "Not Found", + "headers": { + "X-Rendezvous-Correlation-ID": { + "description": "Safe request correlation identifier generated by the service.", + "schema": { + "type": "string" + } + } + }, "content": { "application/json": { "schema": { @@ -979,6 +1417,14 @@ }, "409": { "description": "Conflict", + "headers": { + "X-Rendezvous-Correlation-ID": { + "description": "Safe request correlation identifier generated by the service.", + "schema": { + "type": "string" + } + } + }, "content": { "application/json": { "schema": { @@ -989,6 +1435,14 @@ }, "410": { "description": "Gone", + "headers": { + "X-Rendezvous-Correlation-ID": { + "description": "Safe request correlation identifier generated by the service.", + "schema": { + "type": "string" + } + } + }, "content": { "application/json": { "schema": { @@ -1000,6 +1454,12 @@ "429": { "description": "Too Many Requests", "headers": { + "X-Rendezvous-Correlation-ID": { + "description": "Safe request correlation identifier generated by the service.", + "schema": { + "type": "string" + } + }, "Retry-After": { "description": "Whole seconds before the caller should retry (1-60).", "schema": { @@ -1018,6 +1478,14 @@ }, "503": { "description": "Service Unavailable", + "headers": { + "X-Rendezvous-Correlation-ID": { + "description": "Safe request correlation identifier generated by the service.", + "schema": { + "type": "string" + } + } + }, "content": { "application/json": { "schema": { @@ -1055,10 +1523,26 @@ ], "responses": { "204": { - "description": "No Content" + "description": "No Content", + "headers": { + "X-Rendezvous-Correlation-ID": { + "description": "Safe request correlation identifier generated by the service.", + "schema": { + "type": "string" + } + } + } }, "400": { "description": "Bad Request", + "headers": { + "X-Rendezvous-Correlation-ID": { + "description": "Safe request correlation identifier generated by the service.", + "schema": { + "type": "string" + } + } + }, "content": { "application/json": { "schema": { @@ -1069,6 +1553,14 @@ }, "404": { "description": "Not Found", + "headers": { + "X-Rendezvous-Correlation-ID": { + "description": "Safe request correlation identifier generated by the service.", + "schema": { + "type": "string" + } + } + }, "content": { "application/json": { "schema": { @@ -1080,6 +1572,12 @@ "429": { "description": "Too Many Requests", "headers": { + "X-Rendezvous-Correlation-ID": { + "description": "Safe request correlation identifier generated by the service.", + "schema": { + "type": "string" + } + }, "Retry-After": { "description": "Whole seconds before the caller should retry (1-60).", "schema": { @@ -1098,6 +1596,14 @@ }, "503": { "description": "Service Unavailable", + "headers": { + "X-Rendezvous-Correlation-ID": { + "description": "Safe request correlation identifier generated by the service.", + "schema": { + "type": "string" + } + } + }, "content": { "application/json": { "schema": { @@ -1151,6 +1657,14 @@ "responses": { "200": { "description": "OK", + "headers": { + "X-Rendezvous-Correlation-ID": { + "description": "Safe request correlation identifier generated by the service.", + "schema": { + "type": "string" + } + } + }, "content": { "application/json": { "schema": { @@ -1161,6 +1675,14 @@ }, "400": { "description": "Bad Request", + "headers": { + "X-Rendezvous-Correlation-ID": { + "description": "Safe request correlation identifier generated by the service.", + "schema": { + "type": "string" + } + } + }, "content": { "application/json": { "schema": { @@ -1171,6 +1693,14 @@ }, "413": { "description": "Payload Too Large", + "headers": { + "X-Rendezvous-Correlation-ID": { + "description": "Safe request correlation identifier generated by the service.", + "schema": { + "type": "string" + } + } + }, "content": { "application/json": { "schema": { @@ -1181,6 +1711,14 @@ }, "404": { "description": "Not Found", + "headers": { + "X-Rendezvous-Correlation-ID": { + "description": "Safe request correlation identifier generated by the service.", + "schema": { + "type": "string" + } + } + }, "content": { "application/json": { "schema": { @@ -1191,6 +1729,14 @@ }, "409": { "description": "Conflict", + "headers": { + "X-Rendezvous-Correlation-ID": { + "description": "Safe request correlation identifier generated by the service.", + "schema": { + "type": "string" + } + } + }, "content": { "application/json": { "schema": { @@ -1202,6 +1748,12 @@ "429": { "description": "Too Many Requests", "headers": { + "X-Rendezvous-Correlation-ID": { + "description": "Safe request correlation identifier generated by the service.", + "schema": { + "type": "string" + } + }, "Retry-After": { "description": "Whole seconds before the caller should retry (1-60).", "schema": { @@ -1220,6 +1772,14 @@ }, "503": { "description": "Service Unavailable", + "headers": { + "X-Rendezvous-Correlation-ID": { + "description": "Safe request correlation identifier generated by the service.", + "schema": { + "type": "string" + } + } + }, "content": { "application/json": { "schema": { @@ -1235,6 +1795,786 @@ } ] } + }, + "/v1/operator/status": { + "get": { + "tags": [ + "Operator" + ], + "operationId": "GetOperatorStatus", + "responses": { + "200": { + "description": "OK", + "headers": { + "X-Rendezvous-Correlation-ID": { + "description": "Safe request correlation identifier generated by the service.", + "schema": { + "type": "string" + } + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/OperatorStatusResponse" + } + } + } + }, + "401": { + "description": "Unauthorized", + "headers": { + "X-Rendezvous-Correlation-ID": { + "description": "Safe request correlation identifier generated by the service.", + "schema": { + "type": "string" + } + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApiError" + } + } + } + }, + "403": { + "description": "Forbidden", + "headers": { + "X-Rendezvous-Correlation-ID": { + "description": "Safe request correlation identifier generated by the service.", + "schema": { + "type": "string" + } + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApiError" + } + } + } + }, + "404": { + "description": "Not Found", + "headers": { + "X-Rendezvous-Correlation-ID": { + "description": "Safe request correlation identifier generated by the service.", + "schema": { + "type": "string" + } + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApiError" + } + } + } + }, + "429": { + "description": "Too Many Requests", + "headers": { + "X-Rendezvous-Correlation-ID": { + "description": "Safe request correlation identifier generated by the service.", + "schema": { + "type": "string" + } + }, + "Retry-After": { + "description": "Whole seconds before the caller should retry (1-60).", + "schema": { + "type": "integer", + "format": "int32" + } + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApiError" + } + } + } + } + }, + "security": [ + { + "OperatorBearer": [ ] + } + ] + } + }, + "/v1/operator/listings/revoke": { + "post": { + "tags": [ + "Operator" + ], + "operationId": "RevokeOperatorListing", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/RevokeListingRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "OK", + "headers": { + "X-Rendezvous-Correlation-ID": { + "description": "Safe request correlation identifier generated by the service.", + "schema": { + "type": "string" + } + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/OperatorActionResponse" + } + } + } + }, + "400": { + "description": "Bad Request", + "headers": { + "X-Rendezvous-Correlation-ID": { + "description": "Safe request correlation identifier generated by the service.", + "schema": { + "type": "string" + } + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApiError" + } + } + } + }, + "413": { + "description": "Payload Too Large", + "headers": { + "X-Rendezvous-Correlation-ID": { + "description": "Safe request correlation identifier generated by the service.", + "schema": { + "type": "string" + } + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApiError" + } + } + } + }, + "401": { + "description": "Unauthorized", + "headers": { + "X-Rendezvous-Correlation-ID": { + "description": "Safe request correlation identifier generated by the service.", + "schema": { + "type": "string" + } + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApiError" + } + } + } + }, + "403": { + "description": "Forbidden", + "headers": { + "X-Rendezvous-Correlation-ID": { + "description": "Safe request correlation identifier generated by the service.", + "schema": { + "type": "string" + } + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApiError" + } + } + } + }, + "404": { + "description": "Not Found", + "headers": { + "X-Rendezvous-Correlation-ID": { + "description": "Safe request correlation identifier generated by the service.", + "schema": { + "type": "string" + } + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApiError" + } + } + } + }, + "429": { + "description": "Too Many Requests", + "headers": { + "X-Rendezvous-Correlation-ID": { + "description": "Safe request correlation identifier generated by the service.", + "schema": { + "type": "string" + } + }, + "Retry-After": { + "description": "Whole seconds before the caller should retry (1-60).", + "schema": { + "type": "integer", + "format": "int32" + } + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApiError" + } + } + } + }, + "503": { + "description": "Service Unavailable", + "headers": { + "X-Rendezvous-Correlation-ID": { + "description": "Safe request correlation identifier generated by the service.", + "schema": { + "type": "string" + } + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApiError" + } + } + } + } + }, + "security": [ + { + "OperatorBearer": [ ] + } + ] + } + }, + "/v1/operator/principals/revoke": { + "post": { + "tags": [ + "Operator" + ], + "operationId": "RevokeOperatorPrincipal", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/RevokePrincipalRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "OK", + "headers": { + "X-Rendezvous-Correlation-ID": { + "description": "Safe request correlation identifier generated by the service.", + "schema": { + "type": "string" + } + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/OperatorActionResponse" + } + } + } + }, + "400": { + "description": "Bad Request", + "headers": { + "X-Rendezvous-Correlation-ID": { + "description": "Safe request correlation identifier generated by the service.", + "schema": { + "type": "string" + } + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApiError" + } + } + } + }, + "413": { + "description": "Payload Too Large", + "headers": { + "X-Rendezvous-Correlation-ID": { + "description": "Safe request correlation identifier generated by the service.", + "schema": { + "type": "string" + } + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApiError" + } + } + } + }, + "401": { + "description": "Unauthorized", + "headers": { + "X-Rendezvous-Correlation-ID": { + "description": "Safe request correlation identifier generated by the service.", + "schema": { + "type": "string" + } + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApiError" + } + } + } + }, + "403": { + "description": "Forbidden", + "headers": { + "X-Rendezvous-Correlation-ID": { + "description": "Safe request correlation identifier generated by the service.", + "schema": { + "type": "string" + } + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApiError" + } + } + } + }, + "404": { + "description": "Not Found", + "headers": { + "X-Rendezvous-Correlation-ID": { + "description": "Safe request correlation identifier generated by the service.", + "schema": { + "type": "string" + } + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApiError" + } + } + } + }, + "429": { + "description": "Too Many Requests", + "headers": { + "X-Rendezvous-Correlation-ID": { + "description": "Safe request correlation identifier generated by the service.", + "schema": { + "type": "string" + } + }, + "Retry-After": { + "description": "Whole seconds before the caller should retry (1-60).", + "schema": { + "type": "integer", + "format": "int32" + } + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApiError" + } + } + } + }, + "503": { + "description": "Service Unavailable", + "headers": { + "X-Rendezvous-Correlation-ID": { + "description": "Safe request correlation identifier generated by the service.", + "schema": { + "type": "string" + } + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApiError" + } + } + } + } + }, + "security": [ + { + "OperatorBearer": [ ] + } + ] + } + }, + "/v1/operator/keys/revoke": { + "post": { + "tags": [ + "Operator" + ], + "operationId": "RevokeOperatorSigningKey", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/RevokeSigningKeyRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "OK", + "headers": { + "X-Rendezvous-Correlation-ID": { + "description": "Safe request correlation identifier generated by the service.", + "schema": { + "type": "string" + } + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/OperatorActionResponse" + } + } + } + }, + "400": { + "description": "Bad Request", + "headers": { + "X-Rendezvous-Correlation-ID": { + "description": "Safe request correlation identifier generated by the service.", + "schema": { + "type": "string" + } + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApiError" + } + } + } + }, + "413": { + "description": "Payload Too Large", + "headers": { + "X-Rendezvous-Correlation-ID": { + "description": "Safe request correlation identifier generated by the service.", + "schema": { + "type": "string" + } + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApiError" + } + } + } + }, + "401": { + "description": "Unauthorized", + "headers": { + "X-Rendezvous-Correlation-ID": { + "description": "Safe request correlation identifier generated by the service.", + "schema": { + "type": "string" + } + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApiError" + } + } + } + }, + "403": { + "description": "Forbidden", + "headers": { + "X-Rendezvous-Correlation-ID": { + "description": "Safe request correlation identifier generated by the service.", + "schema": { + "type": "string" + } + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApiError" + } + } + } + }, + "404": { + "description": "Not Found", + "headers": { + "X-Rendezvous-Correlation-ID": { + "description": "Safe request correlation identifier generated by the service.", + "schema": { + "type": "string" + } + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApiError" + } + } + } + }, + "429": { + "description": "Too Many Requests", + "headers": { + "X-Rendezvous-Correlation-ID": { + "description": "Safe request correlation identifier generated by the service.", + "schema": { + "type": "string" + } + }, + "Retry-After": { + "description": "Whole seconds before the caller should retry (1-60).", + "schema": { + "type": "integer", + "format": "int32" + } + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApiError" + } + } + } + } + }, + "security": [ + { + "OperatorBearer": [ ] + } + ] + } + }, + "/v1/operator/drain": { + "post": { + "tags": [ + "Operator" + ], + "operationId": "BeginOperatorDrain", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BeginDrainRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "OK", + "headers": { + "X-Rendezvous-Correlation-ID": { + "description": "Safe request correlation identifier generated by the service.", + "schema": { + "type": "string" + } + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/OperatorActionResponse" + } + } + } + }, + "400": { + "description": "Bad Request", + "headers": { + "X-Rendezvous-Correlation-ID": { + "description": "Safe request correlation identifier generated by the service.", + "schema": { + "type": "string" + } + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApiError" + } + } + } + }, + "413": { + "description": "Payload Too Large", + "headers": { + "X-Rendezvous-Correlation-ID": { + "description": "Safe request correlation identifier generated by the service.", + "schema": { + "type": "string" + } + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApiError" + } + } + } + }, + "401": { + "description": "Unauthorized", + "headers": { + "X-Rendezvous-Correlation-ID": { + "description": "Safe request correlation identifier generated by the service.", + "schema": { + "type": "string" + } + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApiError" + } + } + } + }, + "403": { + "description": "Forbidden", + "headers": { + "X-Rendezvous-Correlation-ID": { + "description": "Safe request correlation identifier generated by the service.", + "schema": { + "type": "string" + } + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApiError" + } + } + } + }, + "404": { + "description": "Not Found", + "headers": { + "X-Rendezvous-Correlation-ID": { + "description": "Safe request correlation identifier generated by the service.", + "schema": { + "type": "string" + } + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApiError" + } + } + } + }, + "429": { + "description": "Too Many Requests", + "headers": { + "X-Rendezvous-Correlation-ID": { + "description": "Safe request correlation identifier generated by the service.", + "schema": { + "type": "string" + } + }, + "Retry-After": { + "description": "Whole seconds before the caller should retry (1-60).", + "schema": { + "type": "integer", + "format": "int32" + } + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApiError" + } + } + } + } + }, + "security": [ + { + "OperatorBearer": [ ] + } + ] + } } }, "components": { @@ -1278,6 +2618,17 @@ } } }, + "BeginDrainRequest": { + "required": [ + "confirmation" + ], + "type": "object", + "properties": { + "confirmation": { + "type": "string" + } + } + }, "BrowseHostJoinAttemptsResponse": { "required": [ "contractVersion", @@ -1562,6 +2913,211 @@ } } }, + "OperatorActionResponse": { + "required": [ + "status" + ], + "type": "object", + "properties": { + "status": { + "type": "string" + }, + "affectedResources": { + "type": [ + "null", + "integer" + ], + "format": "int32" + } + } + }, + "OperatorReadinessResponse": { + "required": [ + "httpListener", + "udpIpv4Listener", + "udpIpv6Listener", + "provisioning", + "store", + "draining" + ], + "type": "object", + "properties": { + "httpListener": { + "type": "boolean" + }, + "udpIpv4Listener": { + "type": "boolean" + }, + "udpIpv6Listener": { + "type": "boolean" + }, + "provisioning": { + "type": "boolean" + }, + "store": { + "type": "boolean" + }, + "draining": { + "type": "boolean" + } + } + }, + "OperatorSigningKeyResponse": { + "required": [ + "keyId", + "status", + "signUntil", + "verifyUntil", + "credentialKinds" + ], + "type": "object", + "properties": { + "keyId": { + "type": "string" + }, + "status": { + "type": "string" + }, + "signUntil": { + "type": "string", + "format": "date-time" + }, + "verifyUntil": { + "type": "string", + "format": "date-time" + }, + "gameId": { + "type": [ + "null", + "string" + ] + }, + "environmentId": { + "type": [ + "null", + "string" + ] + }, + "credentialKinds": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "OperatorStatusResponse": { + "required": [ + "status", + "readiness", + "store", + "tenants", + "signingKeys", + "auditCounts" + ], + "type": "object", + "properties": { + "status": { + "type": "string" + }, + "readiness": { + "$ref": "#/components/schemas/OperatorReadinessResponse" + }, + "store": { + "$ref": "#/components/schemas/OperatorStoreResponse" + }, + "tenants": { + "type": "array", + "items": { + "$ref": "#/components/schemas/OperatorTenantResponse" + } + }, + "signingKeys": { + "type": "array", + "items": { + "$ref": "#/components/schemas/OperatorSigningKeyResponse" + } + }, + "auditCounts": { + "type": "object", + "additionalProperties": { + "type": "integer", + "format": "int64" + } + } + } + }, + "OperatorStoreResponse": { + "required": [ + "activeListings", + "freshPresenceBindings", + "activeJoinAttempts", + "retainedOutcomeReports", + "replayMarkers", + "principalRevocations", + "idempotencyEntries", + "maintenanceSweeps", + "expiryChurn" + ], + "type": "object", + "properties": { + "activeListings": { + "type": "integer", + "format": "int32" + }, + "freshPresenceBindings": { + "type": "integer", + "format": "int32" + }, + "activeJoinAttempts": { + "type": "integer", + "format": "int32" + }, + "retainedOutcomeReports": { + "type": "integer", + "format": "int32" + }, + "replayMarkers": { + "type": "integer", + "format": "int32" + }, + "principalRevocations": { + "type": "integer", + "format": "int32" + }, + "idempotencyEntries": { + "type": "integer", + "format": "int32" + }, + "maintenanceSweeps": { + "type": "integer", + "format": "int64" + }, + "expiryChurn": { + "type": "integer", + "format": "int64" + } + } + }, + "OperatorTenantResponse": { + "required": [ + "gameId", + "environmentId", + "status" + ], + "type": "object", + "properties": { + "gameId": { + "type": "string" + }, + "environmentId": { + "type": "string" + }, + "status": { + "type": "string" + } + } + }, "PublisherTrustMode": { "enum": [ "managedDedicated", @@ -1791,6 +3347,56 @@ } } }, + "RevokeListingRequest": { + "required": [ + "listingId", + "confirmListingId" + ], + "type": "object", + "properties": { + "listingId": { + "type": "string" + }, + "confirmListingId": { + "type": "string" + } + } + }, + "RevokePrincipalRequest": { + "required": [ + "subject", + "confirmSubject", + "lifetimeSeconds" + ], + "type": "object", + "properties": { + "subject": { + "type": "string" + }, + "confirmSubject": { + "type": "string" + }, + "lifetimeSeconds": { + "type": "integer", + "format": "int32" + } + } + }, + "RevokeSigningKeyRequest": { + "required": [ + "keyId", + "confirmKeyId" + ], + "type": "object", + "properties": { + "keyId": { + "type": "string" + }, + "confirmKeyId": { + "type": "string" + } + } + }, "SessionCapacity": { "required": [ "currentPlayers", @@ -1940,6 +3546,12 @@ "description": "Attempt-scoped client capability returned only to the joining caller.", "name": "X-Rendezvous-Client-Punch-Capability", "in": "header" + }, + "OperatorBearer": { + "type": "http", + "description": "Operator-only credential with an explicit permission set.", + "scheme": "bearer", + "bearerFormat": "rv1 operator credential" } } }, @@ -1952,6 +3564,9 @@ }, { "name": "Join attempts" + }, + { + "name": "Operator" } ] } \ No newline at end of file diff --git a/docs/operations/observability-and-operator-runbook.md b/docs/operations/observability-and-operator-runbook.md new file mode 100644 index 0000000..b8300c5 --- /dev/null +++ b/docs/operations/observability-and-operator-runbook.md @@ -0,0 +1,142 @@ +# Observability and operator runbook + +This runbook defines the production signals and privileged controls for the +Rendezvous service. The service emits `System.Diagnostics.Metrics` instruments +from the `FinalFactory.Rendezvous` meter and distributed-tracing activities from +`FinalFactory.Rendezvous.Server`. Connect those sources to the deployment's +OpenTelemetry or equivalent collector. Do not add identifiers to metric labels. + +## Health and readiness + +- `GET /health/live` proves that the HTTP process can answer. It deliberately + remains independent of provisioning, the state store, drain state, and optional + listeners so an orchestrator does not restart a recoverable dependency failure. +- `GET /health/ready` returns success only after the HTTP path is answering, the + required IPv4 UDP socket is bound, any configured IPv6 UDP socket is bound, + provisioning loaded successfully, the store is available, and drain has not + started. A failed check returns `503` and removes the instance from new work. +- A graceful drain immediately makes readiness fail while liveness remains healthy. + Existing work may complete until the bounded store drain deadline. + +## Metrics and traces + +| Instrument | Purpose | Bounded dimensions | +| --- | --- | --- | +| `rendezvous.http.requests` / `rendezvous.http.duration` | HTTP volume and latency | operation, status code | +| `rendezvous.udp.results` / `rendezvous.udp.duration` | UDP mediation volume and processing latency | frozen/litenet operation, result | +| `rendezvous.limiter.drops` | Requests shed by admission controls | transport, fixed partition class | +| `rendezvous.operator.authentication` | Accepted, forbidden, and rejected operator authentication | result | +| `rendezvous.audit.events` | Privileged action outcomes | fixed action, result | +| `rendezvous.connection.outcomes` | Client-reported direct-connect outcomes | normalized outcome, elapsed bucket | +| `rendezvous.pairing.latency` | Time from attempt creation to successful peer introduction | none | +| `rendezvous.queue.depth` | Active join-attempt queue depth | none | +| `rendezvous.store.active_listings` / `active_leases` / `active_attempts` / `replay_markers` | Current ephemeral load | none | +| `rendezvous.store.expiry_churn` | Cumulative natural expiry activity | none | +| `rendezvous.store.available` | Store health (`1` available, `0` unavailable) | none | + +HTTP responses include `X-Rendezvous-Correlation-ID`. It is a generated trace ID +or random value, never a caller-supplied session or player identifier. UDP and +HTTP activities contain operation-level data only. Logs and traces must not add +tokens, capabilities, session/listing IDs, player subjects, metadata, raw IP +addresses, or endpoint values. + +Recommended dashboard panels are request rate and p50/p95/p99 latency by fixed +operation, UDP result ratio, direct connection success ratio, pairing latency, +active listings/attempts, expiry churn, limiter drops, store availability, +operator authentication results, audit action results, and signing-key windows. + +## Alerts + +Tune thresholds from the normal production baseline, then keep these conditions +as distinct actionable alerts: + +- **Signing key expiry:** page when any required signing key has less than seven + days before `signUntil`; escalate at 24 hours. Confirm a replacement is signing + and the previous key remains verify-only for the maximum credential lifetime. +- **Authentication spike:** warn when rejected or forbidden operator authentication + exceeds five attempts in five minutes. Treat unexpected publisher-authentication + growth as a possible credential or integration incident. +- **Direct success regression:** warn when the connected outcome ratio falls more + than 20% below its seven-day same-region baseline for 15 minutes, with a minimum + sample floor. Break down only by bounded outcome and time bucket. +- **Saturation:** warn when queue depth remains above 70% of the configured attempt + limit, limiter drops are sustained, or p95 latency exceeds the service objective; + page at 90% or when lease-critical traffic is shed. +- **Store degradation:** page immediately when `rendezvous.store.available` is zero + or readiness fails for the store. Rising expiry churn without corresponding new + work is a warning for stalled clients or clock/configuration mistakes. +- **Listener/config readiness:** page when no ready instances remain. Investigate + UDP bind failures, a configured-but-unbound IPv6 listener, provisioning errors, + and unintended drain state separately. + +## Operator authentication and controls + +Operator credentials use a signing key configured with `CredentialKinds: +["Operator"]`. Operator keys cannot be scoped to a game/environment or used for +publisher credentials. Mint short-lived operator credentials through the trusted +provisioning process, outside the public Rendezvous HTTP service, and grant only +the required permission. Never place credentials in command history, URLs, logs, +or support tickets. + +The application also enforces a default-deny source boundary. Configure at most +32 exact operator source IPs in +`Rendezvous:AbuseProtection:OperatorAllowedAddresses`; an empty list disables all +operator HTTP access. Development permits loopback only. Production must place +`/v1/operator/*` behind a private management listener or reverse-proxy ACL, list +only the resulting trusted management source addresses, and block that path on +the public edge. If forwarded headers are enabled, keep the existing exact-proxy, +single-hop trust policy and allowlist the post-forwarding operator source. Verify +from both an allowed management host and a denied public host before deployment. +Denied sources are charged to the bounded general HTTP partition before credential +or request-body processing, then receive `404`; sustained denied traffic receives +the same typed `429` overload response as other public traffic. + +Operator traffic has a dedicated, bounded rate/concurrency partition and critical +tracker-key reserve. Public browse/join saturation therefore cannot consume the +operator control budget, while compromised management sources remain rate-limited. + +The OpenAPI document defines the separate `OperatorBearer` scheme. All endpoints +are under `/v1/operator`: + +| Endpoint | Permission | Confirmation | +| --- | --- | --- | +| `GET /status` | `ReadPolicy` | none; returns aggregates, tenant status, safe key status, and audit counts | +| `POST /listings/revoke` | `RevokePublisher` | repeat the exact listing ID in `confirmListingId` | +| `POST /principals/revoke` | `RevokePublisher` | repeat the exact subject and choose a 1-600 second revocation lifetime | +| `POST /keys/revoke` | `RotateKeys` | repeat the exact key ID; runtime revocation is immediate | +| `POST /drain` | `ManagePolicy` | send the exact value `DRAIN` | + +Publisher credentials are rejected on this surface even if their subject resembles +an operator. Destructive responses do not echo identifiers. The status response +does not expose player identities, raw endpoints, session metadata, capabilities, +or tokens. Every authenticated operator action, rejected confirmation, and +permission denial is audited with actor and target fingerprints. + +Key revocation is process-local in the current single-instance store. Apply the +same revocation to every instance, then replace configuration before restarting; +a restart reconstructs the configured key ring. Principal revocation is bounded +to ten minutes and removes that principal's active listings and attempts. Use +listing revocation for one targeted session and drain before planned shutdown. + +## Audit retention and incident handling + +The in-process audit trail defaults to 10,000 entries and 30 days. It evicts the +oldest record at capacity and purges expired records on the next write. Configure +`Rendezvous:Audit:MaxEntries` and `RetentionDays` within their validated bounds. +Export the structured `AuditTrail` log events through the deployment's protected +logging pipeline when durable retention is required; the in-memory trail is not a +durable compliance archive. Those events include only timestamps, fixed action +fields, correlation IDs, and actor/target fingerprints. + +Audit records retain timestamp, fixed action/result, target kind, correlation ID, +and 96-bit SHA-256 fingerprints of actor and target. Routine logs contain only the +fixed action/result/target kind and correlation ID. Restrict audit access to the +operator role, retain aggregates only as long as operationally necessary, and +delete raw exported audit data according to the 30-day policy unless an incident +hold is approved. + +During an incident: confirm readiness and store health; capture aggregate graphs +and correlation IDs; revoke the narrowest listing, principal, or key; drain only +when isolation is required; record the action in the incident timeline; and verify +that direct success, limiter drops, and authentication rates return to baseline. +Do not copy player data, endpoints, or credentials into the incident record. diff --git a/docs/security/abuse-protection.md b/docs/security/abuse-protection.md index 066335c..1f340ee 100644 --- a/docs/security/abuse-protection.md +++ b/docs/security/abuse-protection.md @@ -20,6 +20,10 @@ traffic is also isolated by its authenticated scope. Health probes use their own source-prefix budget so public API overload cannot make a healthy instance fail its orchestrator probes, while health traffic is still bounded. + Operator endpoints likewise use a separate bounded rate/concurrency partition + backed by the critical tracker reserve. They first require an exact source IP + from the default-deny `OperatorAllowedAddresses` policy, so public traffic + cannot spend the incident-response budget. 3. Once an endpoint has safely derived identities, it also acquires applicable tenant, principal or capability, and listing/attempt budgets. Secret capabilities are represented only by bounded SHA-256 fingerprints. @@ -41,7 +45,8 @@ load shedding. `Rendezvous:AbuseProtection:MaxTrackedKeys` is a hard combined ceiling for rate and active-concurrency keys. General HTTP and UDP traffic cannot consume the -configured `CriticalTrackedKeyReserve`; lease operations and health probes may +configured `CriticalTrackedKeyReserve`; lease operations, health probes, and +allowlisted operator controls may use that reserve but never exceed the hard ceiling. A request that would exceed its applicable ceiling fails closed without adding state. Fixed-window rate keys are cleared at the next window boundary; concurrency keys are removed as their diff --git a/src/FinalFactory.Rendezvous.Server/Abuse/AbuseProtectionOptions.cs b/src/FinalFactory.Rendezvous.Server/Abuse/AbuseProtectionOptions.cs index b883075..bd27011 100644 --- a/src/FinalFactory.Rendezvous.Server/Abuse/AbuseProtectionOptions.cs +++ b/src/FinalFactory.Rendezvous.Server/Abuse/AbuseProtectionOptions.cs @@ -20,6 +20,8 @@ internal sealed class AbuseProtectionOptions public string[] TrustedProxyAddresses { get; set; } = []; + public string[] OperatorAllowedAddresses { get; set; } = []; + [Range(1, 100_000)] public int HealthGlobalRequestsPerWindow { get; set; } = 1_000; @@ -32,6 +34,18 @@ internal sealed class AbuseProtectionOptions [Range(1, 1_000)] public int HealthIpPrefixConcurrency { get; set; } = 8; + [Range(1, 100_000)] + public int OperatorGlobalRequestsPerWindow { get; set; } = 1_000; + + [Range(1, 10_000)] + public int OperatorGlobalConcurrency { get; set; } = 32; + + [Range(1, 100_000)] + public int OperatorIpPrefixRequestsPerWindow { get; set; } = 120; + + [Range(1, 1_000)] + public int OperatorIpPrefixConcurrency { get; set; } = 8; + [Range(1, 1_000_000)] public int HttpGlobalRequestsPerWindow { get; set; } = 20_000; diff --git a/src/FinalFactory.Rendezvous.Server/Abuse/AbuseProtectionService.cs b/src/FinalFactory.Rendezvous.Server/Abuse/AbuseProtectionService.cs index 2f8d788..cb5d9ea 100644 --- a/src/FinalFactory.Rendezvous.Server/Abuse/AbuseProtectionService.cs +++ b/src/FinalFactory.Rendezvous.Server/Abuse/AbuseProtectionService.cs @@ -2,6 +2,7 @@ using System.Buffers; using System.Net; using System.Security.Cryptography; using System.Text; +using FinalFactory.Rendezvous.Server.Observability; using Microsoft.Extensions.Options; namespace FinalFactory.Rendezvous.Server.Abuse; @@ -12,13 +13,23 @@ internal sealed class AbuseProtectionService private readonly TimeProvider _timeProvider; private readonly TrackerState _httpTracker; private readonly TrackerState _udpTracker; + private readonly RendezvousTelemetry? _telemetry; + private readonly HashSet _operatorAllowedAddresses; public AbuseProtectionService( IOptions options, - TimeProvider? timeProvider = null) + TimeProvider? timeProvider = null, + RendezvousTelemetry? telemetry = null) { _options = options.Value; _timeProvider = timeProvider ?? TimeProvider.System; + _telemetry = telemetry; + _operatorAllowedAddresses = options.Value.OperatorAllowedAddresses + .Select(static value => IPAddress.TryParse(value, out IPAddress? address) + ? NormalizeAddress(address).ToString() + : string.Empty) + .Where(static value => value.Length > 0) + .ToHashSet(StringComparer.Ordinal); DateTimeOffset now = _timeProvider.GetUtcNow(); _httpTracker = new(now); _udpTracker = new(now); @@ -87,6 +98,35 @@ internal sealed class AbuseProtectionService out retryAfterSeconds); } + public bool IsOperatorSourceAllowed(IPAddress? remoteAddress) => + remoteAddress is not null + && _operatorAllowedAddresses.Contains(NormalizeAddress(remoteAddress).ToString()); + + public bool TryAcquireOperatorIngress( + IPAddress? remoteAddress, + out AbuseLease? lease, + out int retryAfterSeconds) + { + string prefix = GetNetworkPrefix(remoteAddress); + RateDimension[] rates = + [ + new("operator:rate:global", _options.OperatorGlobalRequestsPerWindow), + new($"operator:rate:ip:{prefix}", _options.OperatorIpPrefixRequestsPerWindow), + ]; + RateDimension[] concurrency = + [ + new("operator:concurrency:global", _options.OperatorGlobalConcurrency), + new($"operator:concurrency:ip:{prefix}", _options.OperatorIpPrefixConcurrency), + ]; + return TryAcquire( + rates, + concurrency, + TrackerDomain.Http, + true, + out lease, + out retryAfterSeconds); + } + public bool TryAcquireHttpIdentity( string operation, string? tenant, @@ -266,67 +306,96 @@ internal sealed class AbuseProtectionService out int retryAfterSeconds) { TrackerState tracker = domain == TrackerDomain.Udp ? _udpTracker : _httpTracker; + bool accepted; lock (tracker.Gate) { - DateTimeOffset now = _timeProvider.GetUtcNow(); - TimeSpan window = TimeSpan.FromSeconds(_options.WindowSeconds); - if (now - tracker.WindowStartedAt >= window || now < tracker.WindowStartedAt) - { - tracker.WindowCounts.Clear(); - tracker.WindowStartedAt = now; - } + accepted = TryAcquireLocked( + tracker, + rates, + concurrency, + domain, + canUseCriticalReserve, + out lease, + out retryAfterSeconds); + } - retryAfterSeconds = Math.Max( - 1, - (int)Math.Ceiling((window - (now - tracker.WindowStartedAt)).TotalSeconds)); - int stagedNewKeys = 0; - int partitionLimit = domain == TrackerDomain.Udp - ? _options.UdpTrackedKeyLimit - : _options.MaxTrackedKeys - _options.UdpTrackedKeyLimit; - int maxTrackedKeys = domain == TrackerDomain.Udp || canUseCriticalReserve - ? partitionLimit - : partitionLimit - _options.CriticalTrackedKeyReserve; - if (!CanAcquireAll( - tracker, - tracker.WindowCounts, - rates, - maxTrackedKeys, - ref stagedNewKeys) - || !CanAcquireAll( - tracker, - tracker.ConcurrencyCounts, - concurrency, - maxTrackedKeys, - ref stagedNewKeys)) - { - lease = null; - return false; - } + if (!accepted) + { + _telemetry?.RecordLimiterDrop( + domain == TrackerDomain.Udp ? "udp" : "http", + "rate-or-concurrency"); + } - foreach (RateDimension dimension in rates) - { - tracker.WindowCounts[dimension.Key] = - tracker.WindowCounts.GetValueOrDefault(dimension.Key) + 1; - } + return accepted; + } - if (concurrency.IsEmpty) - { - lease = null; - return true; - } + private bool TryAcquireLocked( + TrackerState tracker, + ReadOnlySpan rates, + ReadOnlySpan concurrency, + TrackerDomain domain, + bool canUseCriticalReserve, + out AbuseLease? lease, + out int retryAfterSeconds) + { + DateTimeOffset now = _timeProvider.GetUtcNow(); + TimeSpan window = TimeSpan.FromSeconds(_options.WindowSeconds); + if (now - tracker.WindowStartedAt >= window || now < tracker.WindowStartedAt) + { + tracker.WindowCounts.Clear(); + tracker.WindowStartedAt = now; + } - string[] acquiredConcurrency = new string[concurrency.Length]; - for (int index = 0; index < concurrency.Length; index++) - { - RateDimension dimension = concurrency[index]; - tracker.ConcurrencyCounts[dimension.Key] = - tracker.ConcurrencyCounts.GetValueOrDefault(dimension.Key) + 1; - acquiredConcurrency[index] = dimension.Key; - } + retryAfterSeconds = Math.Max( + 1, + (int)Math.Ceiling((window - (now - tracker.WindowStartedAt)).TotalSeconds)); + int stagedNewKeys = 0; + int partitionLimit = domain == TrackerDomain.Udp + ? _options.UdpTrackedKeyLimit + : _options.MaxTrackedKeys - _options.UdpTrackedKeyLimit; + int maxTrackedKeys = domain == TrackerDomain.Udp || canUseCriticalReserve + ? partitionLimit + : partitionLimit - _options.CriticalTrackedKeyReserve; + if (!CanAcquireAll( + tracker, + tracker.WindowCounts, + rates, + maxTrackedKeys, + ref stagedNewKeys) + || !CanAcquireAll( + tracker, + tracker.ConcurrencyCounts, + concurrency, + maxTrackedKeys, + ref stagedNewKeys)) + { + lease = null; + return false; + } - lease = new AbuseLease(this, tracker, acquiredConcurrency); + foreach (RateDimension dimension in rates) + { + tracker.WindowCounts[dimension.Key] = + tracker.WindowCounts.GetValueOrDefault(dimension.Key) + 1; + } + + if (concurrency.IsEmpty) + { + lease = null; return true; } + + string[] acquiredConcurrency = new string[concurrency.Length]; + for (int index = 0; index < concurrency.Length; index++) + { + RateDimension dimension = concurrency[index]; + tracker.ConcurrencyCounts[dimension.Key] = + tracker.ConcurrencyCounts.GetValueOrDefault(dimension.Key) + 1; + acquiredConcurrency[index] = dimension.Key; + } + + lease = new AbuseLease(this, tracker, acquiredConcurrency); + return true; } private static bool CanAcquireAll( @@ -415,6 +484,9 @@ internal sealed class AbuseProtectionService return "unknown"; } + private static IPAddress NormalizeAddress(IPAddress address) => + address.IsIPv4MappedToIPv6 ? address.MapToIPv4() : address; + private readonly record struct RateDimension(string Key, int Limit); private enum TrackerDomain diff --git a/src/FinalFactory.Rendezvous.Server/Abuse/HttpAbuseProtectionMiddleware.cs b/src/FinalFactory.Rendezvous.Server/Abuse/HttpAbuseProtectionMiddleware.cs index e3b2792..ecd12fc 100644 --- a/src/FinalFactory.Rendezvous.Server/Abuse/HttpAbuseProtectionMiddleware.cs +++ b/src/FinalFactory.Rendezvous.Server/Abuse/HttpAbuseProtectionMiddleware.cs @@ -19,16 +19,71 @@ internal sealed class HttpAbuseProtectionMiddleware( string operation = context.GetEndpoint()?.Metadata.GetMetadata() ?.EndpointName ?? "Unmatched"; bool healthEndpoint = operation is "GetLiveness" or "GetReadiness"; - bool acquired = healthEndpoint - ? protection.TryAcquireHealthIngress( + bool operatorEndpoint = operation is + "GetOperatorStatus" + or "RevokeOperatorListing" + or "RevokeOperatorPrincipal" + or "RevokeOperatorSigningKey" + or "BeginOperatorDrain"; + if (operatorEndpoint + && !protection.IsOperatorSourceAllowed(context.Connection.RemoteIpAddress)) + { + bool deniedSourceAdmitted = protection.TryAcquireHttpIngress( context.Connection.RemoteIpAddress, - out AbuseProtectionService.AbuseLease? lease, - out int retryAfterSeconds) - : protection.TryAcquireHttpIngress( + "Unmatched", + out AbuseProtectionService.AbuseLease? deniedSourceLease, + out int deniedRetryAfterSeconds); + using (deniedSourceLease) + { + if (!deniedSourceAdmitted) + { + context.Response.Headers.RetryAfter = deniedRetryAfterSeconds.ToString( + System.Globalization.CultureInfo.InvariantCulture); + await WriteErrorAsync( + context, + StatusCodes.Status429TooManyRequests, + RendezvousErrorCode.RateLimited, + "The request rate limit was exceeded.", + deniedRetryAfterSeconds).ConfigureAwait(false); + return; + } + + await WriteErrorAsync( + context, + StatusCodes.Status404NotFound, + RendezvousErrorCode.NotFound, + "The requested resource was not found.").ConfigureAwait(false); + } + + return; + } + + AbuseProtectionService.AbuseLease? lease; + int retryAfterSeconds; + bool acquired; + if (healthEndpoint) + { + acquired = protection.TryAcquireHealthIngress( + context.Connection.RemoteIpAddress, + out lease, + out retryAfterSeconds); + } + else if (operatorEndpoint) + { + acquired = protection.TryAcquireOperatorIngress( + context.Connection.RemoteIpAddress, + out lease, + out retryAfterSeconds); + } + else + { + acquired = protection.TryAcquireHttpIngress( context.Connection.RemoteIpAddress, operation, out lease, out retryAfterSeconds); + } + if (!acquired) { context.Response.Headers.RetryAfter = retryAfterSeconds.ToString( diff --git a/src/FinalFactory.Rendezvous.Server/ConnectionOutcomes/ConnectionOutcomeService.cs b/src/FinalFactory.Rendezvous.Server/ConnectionOutcomes/ConnectionOutcomeService.cs index 1dcf061..40b3874 100644 --- a/src/FinalFactory.Rendezvous.Server/ConnectionOutcomes/ConnectionOutcomeService.cs +++ b/src/FinalFactory.Rendezvous.Server/ConnectionOutcomes/ConnectionOutcomeService.cs @@ -1,4 +1,5 @@ using FinalFactory.Rendezvous.Contracts; +using FinalFactory.Rendezvous.Server.Observability; using FinalFactory.Rendezvous.Server.Sessions; using FinalFactory.Rendezvous.Server.State; @@ -15,6 +16,10 @@ internal sealed class ConnectionOutcomeMetrics { private readonly object _gate = new(); private readonly Dictionary<(ConnectionOutcomeKind, ConnectionElapsedBucket), long> _counts = []; + private readonly RendezvousTelemetry? _telemetry; + + public ConnectionOutcomeMetrics(RendezvousTelemetry? telemetry = null) => + _telemetry = telemetry; internal void Record(ConnectionOutcomeKind outcome, ConnectionElapsedBucket elapsedBucket) { @@ -24,6 +29,8 @@ internal sealed class ConnectionOutcomeMetrics _counts.TryGetValue(key, out long count); _counts[key] = count + 1; } + + _telemetry?.RecordConnectionOutcome(outcome.ToString(), elapsedBucket.ToString()); } internal long GetCount(ConnectionOutcomeKind outcome, ConnectionElapsedBucket elapsedBucket) diff --git a/src/FinalFactory.Rendezvous.Server/Http/RendezvousExceptionHandler.cs b/src/FinalFactory.Rendezvous.Server/Http/RendezvousExceptionHandler.cs index 2cf223b..82b0a5e 100644 --- a/src/FinalFactory.Rendezvous.Server/Http/RendezvousExceptionHandler.cs +++ b/src/FinalFactory.Rendezvous.Server/Http/RendezvousExceptionHandler.cs @@ -4,7 +4,8 @@ using Microsoft.AspNetCore.Diagnostics; namespace FinalFactory.Rendezvous.Server.Http; -internal sealed class RendezvousExceptionHandler : IExceptionHandler +internal sealed partial class RendezvousExceptionHandler( + ILogger logger) : IExceptionHandler { public async ValueTask TryHandleAsync( HttpContext httpContext, @@ -26,6 +27,13 @@ internal sealed class RendezvousExceptionHandler : IExceptionHandler : invalidRequest ? StatusCodes.Status400BadRequest : StatusCodes.Status500InternalServerError; + LogRequestFailure( + logger, + payloadTooLarge ? "payload-too-large" : invalidRequest ? "invalid-request" : "internal-error", + httpContext.Response.StatusCode, + httpContext.Response.Headers["X-Rendezvous-Correlation-ID"].ToString() is { Length: > 0 } value + ? value + : "unavailable"); await httpContext.Response.WriteAsJsonAsync( new ApiError { @@ -42,4 +50,14 @@ internal sealed class RendezvousExceptionHandler : IExceptionHandler cancellationToken).ConfigureAwait(false); return true; } + + [LoggerMessage( + EventId = 200, + Level = LogLevel.Warning, + Message = "Request failed with {FailureKind} and HTTP status {StatusCode}; correlation {CorrelationId}")] + private static partial void LogRequestFailure( + ILogger logger, + string failureKind, + int statusCode, + string correlationId); } diff --git a/src/FinalFactory.Rendezvous.Server/Observability/AuditOptions.cs b/src/FinalFactory.Rendezvous.Server/Observability/AuditOptions.cs new file mode 100644 index 0000000..2d4a340 --- /dev/null +++ b/src/FinalFactory.Rendezvous.Server/Observability/AuditOptions.cs @@ -0,0 +1,14 @@ +using System.ComponentModel.DataAnnotations; + +namespace FinalFactory.Rendezvous.Server.Observability; + +internal sealed class AuditOptions +{ + public const string SectionName = "Rendezvous:Audit"; + + [Range(100, 100_000)] + public int MaxEntries { get; set; } = 10_000; + + [Range(1, 30)] + public int RetentionDays { get; set; } = 30; +} diff --git a/src/FinalFactory.Rendezvous.Server/Observability/AuditTrail.cs b/src/FinalFactory.Rendezvous.Server/Observability/AuditTrail.cs new file mode 100644 index 0000000..e4f0501 --- /dev/null +++ b/src/FinalFactory.Rendezvous.Server/Observability/AuditTrail.cs @@ -0,0 +1,134 @@ +using System.Security.Cryptography; +using System.Text; +using Microsoft.Extensions.Options; + +namespace FinalFactory.Rendezvous.Server.Observability; + +internal sealed partial class AuditTrail +{ + private readonly object _gate = new(); + private readonly LinkedList _entries = []; + private readonly AuditOptions _options; + private readonly TimeProvider _timeProvider; + private readonly ILogger _logger; + private readonly RendezvousTelemetry _telemetry; + + public AuditTrail( + IOptions options, + ILogger logger, + RendezvousTelemetry telemetry, + TimeProvider? timeProvider = null) + { + _options = options.Value; + _logger = logger; + _telemetry = telemetry; + _timeProvider = timeProvider ?? TimeProvider.System; + } + + public void Record( + string actorSubject, + string action, + string result, + string targetKind, + string targetIdentifier, + string correlationId) + { + DateTimeOffset now = _timeProvider.GetUtcNow(); + AuditEntry entry = new( + now, + Fingerprint(actorSubject), + action, + result, + targetKind, + Fingerprint(targetIdentifier), + correlationId); + lock (_gate) + { + PurgeExpired(now); + + while (_entries.Count >= _options.MaxEntries) + { + _entries.RemoveFirst(); + } + + _entries.AddLast(entry); + } + + _telemetry.RecordAudit(action, result); + LogOperatorAction( + _logger, + entry.Timestamp, + entry.ActorFingerprint, + action, + result, + targetKind, + entry.TargetFingerprint, + correlationId); + } + + public IReadOnlyDictionary GetAggregateCounts() + { + lock (_gate) + { + PurgeExpired(_timeProvider.GetUtcNow()); + return _entries + .GroupBy(static entry => $"{entry.Action}:{entry.Result}", StringComparer.Ordinal) + .ToDictionary( + static group => group.Key, + static group => (long)group.Count(), + StringComparer.Ordinal); + } + } + + internal IReadOnlyList GetEntriesForTests() + { + lock (_gate) + { + PurgeExpired(_timeProvider.GetUtcNow()); + return _entries.ToArray(); + } + } + + private void PurgeExpired(DateTimeOffset now) + { + DateTimeOffset oldest = now.AddDays(-_options.RetentionDays); + while (_entries.First is { Value.Timestamp: var timestamp } + && timestamp < oldest) + { + _entries.RemoveFirst(); + } + } + + private static string Fingerprint(string value) + { + byte[] digest = SHA256.HashData(Encoding.UTF8.GetBytes(value)); + return Convert.ToHexString(digest.AsSpan(0, 12)); + } + + [LoggerMessage( + EventId = 100, + Level = LogLevel.Information, + Message = "Operator audit at {Timestamp}: actor {ActorFingerprint} action {Action} completed with {Result} for {TargetKind} target {TargetFingerprint}; correlation {CorrelationId}")] + private static partial void LogOperatorAction( + ILogger logger, + DateTimeOffset timestamp, + string actorFingerprint, + string action, + string result, + string targetKind, + string targetFingerprint, + string correlationId); +} + +internal sealed record AuditEntry( + DateTimeOffset Timestamp, + string ActorFingerprint, + string Action, + string Result, + string TargetKind, + string TargetFingerprint, + string CorrelationId) +{ + public override string ToString() => + $"[AuditEntry {Action}/{Result}; actor and target fingerprinted]"; +} diff --git a/src/FinalFactory.Rendezvous.Server/Observability/HealthEndpoints.cs b/src/FinalFactory.Rendezvous.Server/Observability/HealthEndpoints.cs new file mode 100644 index 0000000..a3d1668 --- /dev/null +++ b/src/FinalFactory.Rendezvous.Server/Observability/HealthEndpoints.cs @@ -0,0 +1,30 @@ +using FinalFactory.Rendezvous.Contracts; + +namespace FinalFactory.Rendezvous.Server.Observability; + +internal static class HealthEndpoints +{ + public static IEndpointRouteBuilder MapRendezvousHealthEndpoints( + this IEndpointRouteBuilder endpoints) + { + endpoints.MapGet( + "/health/live", + static () => Results.Ok(new HealthResponse { Status = "live" })) + .Produces() + .Produces(StatusCodes.Status429TooManyRequests) + .WithName("GetLiveness") + .WithTags("Health"); + endpoints.MapGet( + "/health/ready", + static (RendezvousReadiness readiness) => + !readiness.GetSnapshot().IsReady + ? Results.StatusCode(StatusCodes.Status503ServiceUnavailable) + : Results.Ok(new HealthResponse { Status = "ready" })) + .Produces() + .Produces(StatusCodes.Status429TooManyRequests) + .Produces(StatusCodes.Status503ServiceUnavailable) + .WithName("GetReadiness") + .WithTags("Health"); + return endpoints; + } +} diff --git a/src/FinalFactory.Rendezvous.Server/Observability/RendezvousReadiness.cs b/src/FinalFactory.Rendezvous.Server/Observability/RendezvousReadiness.cs new file mode 100644 index 0000000..c4ba87c --- /dev/null +++ b/src/FinalFactory.Rendezvous.Server/Observability/RendezvousReadiness.cs @@ -0,0 +1,41 @@ +using FinalFactory.Rendezvous.Server.Provisioning; +using FinalFactory.Rendezvous.Server.State; +using FinalFactory.Rendezvous.Server.Transport; +using Microsoft.Extensions.Options; + +namespace FinalFactory.Rendezvous.Server.Observability; + +internal sealed class RendezvousReadiness( + UdpMediatorService mediator, + ProvisioningReadiness provisioning, + IEphemeralRendezvousStore state, + IOptions udpOptions) +{ + public ReadinessSnapshot GetSnapshot() + { + bool ipv6Required = !string.IsNullOrWhiteSpace(udpOptions.Value.Ipv6ListenAddress); + return new ReadinessSnapshot( + HttpListenerReady: true, + UdpIpv4ListenerReady: mediator.LocalEndpoint is not null, + UdpIpv6ListenerReady: !ipv6Required || mediator.LocalIpv6Endpoint is not null, + ProvisioningReady: provisioning.IsReady, + StoreAvailable: state.IsAvailable, + Draining: state.IsDraining); + } +} + +internal sealed record ReadinessSnapshot( + bool HttpListenerReady, + bool UdpIpv4ListenerReady, + bool UdpIpv6ListenerReady, + bool ProvisioningReady, + bool StoreAvailable, + bool Draining) +{ + public bool IsReady => HttpListenerReady + && UdpIpv4ListenerReady + && UdpIpv6ListenerReady + && ProvisioningReady + && StoreAvailable + && !Draining; +} diff --git a/src/FinalFactory.Rendezvous.Server/Observability/RendezvousTelemetry.cs b/src/FinalFactory.Rendezvous.Server/Observability/RendezvousTelemetry.cs new file mode 100644 index 0000000..2eb2810 --- /dev/null +++ b/src/FinalFactory.Rendezvous.Server/Observability/RendezvousTelemetry.cs @@ -0,0 +1,126 @@ +using System.Diagnostics; +using System.Diagnostics.Metrics; +using FinalFactory.Rendezvous.Server.State; + +namespace FinalFactory.Rendezvous.Server.Observability; + +internal sealed class RendezvousTelemetry : IDisposable +{ + public const string MeterName = "FinalFactory.Rendezvous"; + public const string ActivitySourceName = "FinalFactory.Rendezvous.Server"; + + private readonly InMemoryEphemeralRendezvousStore _store; + private readonly Meter _meter = new(MeterName, "1.0.0"); + private readonly ActivitySource _activities = new(ActivitySourceName, "1.0.0"); + private readonly Counter _httpRequests; + private readonly Histogram _httpDuration; + private readonly Counter _udpResults; + private readonly Histogram _udpDuration; + private readonly Counter _limiterDrops; + private readonly Counter _auditEvents; + private readonly Counter _connectionOutcomes; + private readonly Counter _operatorAuthentication; + private readonly Histogram _pairingLatency; + + public RendezvousTelemetry(InMemoryEphemeralRendezvousStore store) + { + _store = store; + _httpRequests = _meter.CreateCounter("rendezvous.http.requests"); + _httpDuration = _meter.CreateHistogram( + "rendezvous.http.duration", + "ms"); + _udpResults = _meter.CreateCounter("rendezvous.udp.results"); + _udpDuration = _meter.CreateHistogram( + "rendezvous.udp.duration", + "ms"); + _limiterDrops = _meter.CreateCounter("rendezvous.limiter.drops"); + _auditEvents = _meter.CreateCounter("rendezvous.audit.events"); + _connectionOutcomes = _meter.CreateCounter("rendezvous.connection.outcomes"); + _operatorAuthentication = _meter.CreateCounter("rendezvous.operator.authentication"); + _pairingLatency = _meter.CreateHistogram( + "rendezvous.pairing.latency", + "ms"); + _meter.CreateObservableGauge( + "rendezvous.store.active_listings", + () => _store.GetMetricsSnapshot().ActiveListings); + _meter.CreateObservableGauge( + "rendezvous.store.active_leases", + () => _store.GetMetricsSnapshot().ActiveListings); + _meter.CreateObservableGauge( + "rendezvous.store.active_attempts", + () => _store.GetMetricsSnapshot().ActiveJoinAttempts); + _meter.CreateObservableGauge( + "rendezvous.queue.depth", + () => _store.GetMetricsSnapshot().ActiveJoinAttempts); + _meter.CreateObservableGauge( + "rendezvous.store.replay_markers", + () => _store.GetMetricsSnapshot().ReplayMarkers); + _meter.CreateObservableGauge( + "rendezvous.store.available", + () => _store.GetMetricsSnapshot().IsAvailable ? 1 : 0); + _meter.CreateObservableCounter( + "rendezvous.store.expiry_churn", + () => _store.GetMetricsSnapshot().ExpiryChurn); + } + + public Activity? StartActivity(string name, ActivityKind kind = ActivityKind.Internal) => + _activities.StartActivity(name, kind); + + public void RecordHttp(string operation, int statusCode, double elapsedMilliseconds) + { + TagList tags = new() + { + { "operation", operation }, + { "status_code", statusCode }, + }; + _httpRequests.Add(1, tags); + _httpDuration.Record(elapsedMilliseconds, tags); + } + + public void RecordUdp(string operation, string result, double elapsedMilliseconds) + { + TagList tags = new() + { + { "operation", operation }, + { "result", result }, + }; + _udpResults.Add(1, tags); + _udpDuration.Record(elapsedMilliseconds, tags); + } + + public void RecordLimiterDrop(string transport, string partition) => + _limiterDrops.Add(1, new TagList + { + { "transport", transport }, + { "partition", partition }, + }); + + public void RecordAudit(string action, string result) => + _auditEvents.Add(1, new TagList + { + { "action", action }, + { "result", result }, + }); + + public void RecordConnectionOutcome(string outcome, string elapsedBucket) => + _connectionOutcomes.Add(1, new TagList + { + { "outcome", outcome }, + { "elapsed_bucket", elapsedBucket }, + }); + + public void RecordOperatorAuthentication(string result) => + _operatorAuthentication.Add(1, new TagList + { + { "result", result }, + }); + + public void RecordPairingLatency(double elapsedMilliseconds) => + _pairingLatency.Record(elapsedMilliseconds); + + public void Dispose() + { + _activities.Dispose(); + _meter.Dispose(); + } +} diff --git a/src/FinalFactory.Rendezvous.Server/Observability/TelemetryMiddleware.cs b/src/FinalFactory.Rendezvous.Server/Observability/TelemetryMiddleware.cs new file mode 100644 index 0000000..03cdf66 --- /dev/null +++ b/src/FinalFactory.Rendezvous.Server/Observability/TelemetryMiddleware.cs @@ -0,0 +1,32 @@ +using System.Diagnostics; + +namespace FinalFactory.Rendezvous.Server.Observability; + +internal sealed class TelemetryMiddleware( + RequestDelegate next, + RendezvousTelemetry telemetry) +{ + public async Task InvokeAsync(HttpContext context) + { + string operation = context.GetEndpoint()?.Metadata.GetMetadata() + ?.EndpointName ?? "Unmatched"; + long started = Stopwatch.GetTimestamp(); + using Activity? activity = telemetry.StartActivity( + $"HTTP {operation}", + ActivityKind.Server); + string correlationId = activity?.TraceId.ToString() ?? Guid.NewGuid().ToString("N"); + context.Response.Headers["X-Rendezvous-Correlation-ID"] = correlationId; + activity?.SetTag("rendezvous.operation", operation); + try + { + await next(context).ConfigureAwait(false); + } + finally + { + telemetry.RecordHttp( + operation, + context.Response.StatusCode, + Stopwatch.GetElapsedTime(started).TotalMilliseconds); + } + } +} diff --git a/src/FinalFactory.Rendezvous.Server/Operations/OperatorEndpoints.cs b/src/FinalFactory.Rendezvous.Server/Operations/OperatorEndpoints.cs new file mode 100644 index 0000000..22a0031 --- /dev/null +++ b/src/FinalFactory.Rendezvous.Server/Operations/OperatorEndpoints.cs @@ -0,0 +1,428 @@ +using FinalFactory.Rendezvous.Contracts; +using FinalFactory.Rendezvous.Server.Observability; +using FinalFactory.Rendezvous.Server.Provisioning; +using FinalFactory.Rendezvous.Server.State; +using Microsoft.AspNetCore.Mvc; + +namespace FinalFactory.Rendezvous.Server.Operations; + +internal static class OperatorEndpoints +{ + private const string CorrelationHeader = "X-Rendezvous-Correlation-ID"; + + public static IEndpointRouteBuilder MapOperatorEndpoints(this IEndpointRouteBuilder endpoints) + { + RouteGroupBuilder group = endpoints.MapGroup("/v1/operator").WithTags("Operator"); + group.MapGet("/status", GetStatus) + .Produces() + .Produces(StatusCodes.Status401Unauthorized) + .Produces(StatusCodes.Status403Forbidden) + .Produces(StatusCodes.Status404NotFound) + .Produces(StatusCodes.Status429TooManyRequests) + .WithName("GetOperatorStatus"); + group.MapPost("/listings/revoke", RevokeListing) + .Accepts("application/json") + .Produces() + .Produces(StatusCodes.Status400BadRequest) + .Produces(StatusCodes.Status413PayloadTooLarge) + .Produces(StatusCodes.Status401Unauthorized) + .Produces(StatusCodes.Status403Forbidden) + .Produces(StatusCodes.Status404NotFound) + .Produces(StatusCodes.Status429TooManyRequests) + .Produces(StatusCodes.Status503ServiceUnavailable) + .WithName("RevokeOperatorListing"); + group.MapPost("/principals/revoke", RevokePrincipal) + .Accepts("application/json") + .Produces() + .Produces(StatusCodes.Status400BadRequest) + .Produces(StatusCodes.Status413PayloadTooLarge) + .Produces(StatusCodes.Status401Unauthorized) + .Produces(StatusCodes.Status403Forbidden) + .Produces(StatusCodes.Status404NotFound) + .Produces(StatusCodes.Status429TooManyRequests) + .Produces(StatusCodes.Status503ServiceUnavailable) + .WithName("RevokeOperatorPrincipal"); + group.MapPost("/keys/revoke", RevokeSigningKey) + .Accepts("application/json") + .Produces() + .Produces(StatusCodes.Status400BadRequest) + .Produces(StatusCodes.Status413PayloadTooLarge) + .Produces(StatusCodes.Status401Unauthorized) + .Produces(StatusCodes.Status403Forbidden) + .Produces(StatusCodes.Status404NotFound) + .Produces(StatusCodes.Status429TooManyRequests) + .WithName("RevokeOperatorSigningKey"); + group.MapPost("/drain", BeginDrain) + .Accepts("application/json") + .Produces() + .Produces(StatusCodes.Status400BadRequest) + .Produces(StatusCodes.Status413PayloadTooLarge) + .Produces(StatusCodes.Status401Unauthorized) + .Produces(StatusCodes.Status403Forbidden) + .Produces(StatusCodes.Status404NotFound) + .Produces(StatusCodes.Status429TooManyRequests) + .WithName("BeginOperatorDrain"); + return endpoints; + } + + private static IResult GetStatus( + [FromHeader(Name = "Authorization")] string? authorization, + [FromServices] PrincipalCredentialService credentials, + [FromServices] IWallClock clock, + [FromServices] OperatorService service, + [FromServices] AuditTrail audit, + [FromServices] RendezvousTelemetry telemetry, + HttpContext context) + { + if (!TryAuthorize( + authorization, + OperatorPermission.ReadPolicy, + "inspect-status", + credentials, + clock, + audit, + telemetry, + context, + out OperatorPrincipal? principal, + out IResult? failure)) + { + return failure!; + } + + OperatorStatusResponse response = service.GetStatus(); + audit.Record( + principal!.Subject, + "inspect-status", + "succeeded", + "service", + "rendezvous", + Correlation(context)); + return Results.Ok(response); + } + + private static IResult RevokeListing( + [FromBody] RevokeListingRequest request, + [FromHeader(Name = "Authorization")] string? authorization, + [FromServices] PrincipalCredentialService credentials, + [FromServices] IWallClock clock, + [FromServices] OperatorService service, + [FromServices] AuditTrail audit, + [FromServices] RendezvousTelemetry telemetry, + HttpContext context, + CancellationToken cancellationToken) + { + if (!TryAuthorize( + authorization, + OperatorPermission.RevokePublisher, + "revoke-listing", + credentials, + clock, + audit, + telemetry, + context, + out OperatorPrincipal? principal, + out IResult? failure)) + { + return failure!; + } + + bool valid = SessionListingId.TryParse(request.ListingId, out SessionListingId listingId); + if (!valid || !string.Equals(request.ListingId, request.ConfirmListingId, StringComparison.Ordinal)) + { + AuditRejected(audit, principal!, "revoke-listing", "listing", request.ListingId, context); + return BadRequest("A valid listing ID and an exact repeated confirmation are required."); + } + + StoreResult result = service.RevokeListing(listingId, cancellationToken); + return StoreActionResult( + result.Code, + audit, + principal!, + "revoke-listing", + "listing", + request.ListingId, + context, + affectedResources: result.Succeeded ? 1 : null); + } + + private static IResult RevokePrincipal( + [FromBody] RevokePrincipalRequest request, + [FromHeader(Name = "Authorization")] string? authorization, + [FromServices] PrincipalCredentialService credentials, + [FromServices] IWallClock clock, + [FromServices] OperatorService service, + [FromServices] AuditTrail audit, + [FromServices] RendezvousTelemetry telemetry, + HttpContext context, + CancellationToken cancellationToken) + { + if (!TryAuthorize( + authorization, + OperatorPermission.RevokePublisher, + "revoke-principal", + credentials, + clock, + audit, + telemetry, + context, + out OperatorPrincipal? principal, + out IResult? failure)) + { + return failure!; + } + + bool safeSubject = request.Subject is { Length: > 0 and <= 128 } + && request.Subject.All(static character => character is >= '!' and <= '~'); + if (!safeSubject + || !string.Equals(request.Subject, request.ConfirmSubject, StringComparison.Ordinal) + || request.LifetimeSeconds is < 1 or > 600) + { + AuditRejected(audit, principal!, "revoke-principal", "principal", request.Subject, context); + return BadRequest("A valid subject, exact repeated confirmation, and 1-600 second lifetime are required."); + } + + StoreResult result = service.RevokePrincipal( + request.Subject, + TimeSpan.FromSeconds(request.LifetimeSeconds), + cancellationToken); + return StoreActionResult( + result.Code, + audit, + principal!, + "revoke-principal", + "principal", + request.Subject, + context, + result.Value); + } + + private static IResult RevokeSigningKey( + [FromBody] RevokeSigningKeyRequest request, + [FromHeader(Name = "Authorization")] string? authorization, + [FromServices] PrincipalCredentialService credentials, + [FromServices] IWallClock clock, + [FromServices] OperatorService service, + [FromServices] AuditTrail audit, + [FromServices] RendezvousTelemetry telemetry, + HttpContext context) + { + if (!TryAuthorize( + authorization, + OperatorPermission.RotateKeys, + "revoke-signing-key", + credentials, + clock, + audit, + telemetry, + context, + out OperatorPrincipal? principal, + out IResult? failure)) + { + return failure!; + } + + bool safeKeyId = request.KeyId is { Length: > 0 and <= 64 } + && request.KeyId.All(static character => character is + >= 'A' and <= 'Z' + or >= 'a' and <= 'z' + or >= '0' and <= '9' + or '-' + or '_'); + if (!safeKeyId || !string.Equals(request.KeyId, request.ConfirmKeyId, StringComparison.Ordinal)) + { + AuditRejected(audit, principal!, "revoke-signing-key", "signing-key", request.KeyId, context); + return BadRequest("A valid key ID and an exact repeated confirmation are required."); + } + + bool revoked = service.RevokeSigningKey(request.KeyId); + string result = revoked ? "succeeded" : "not-found"; + audit.Record( + principal!.Subject, + "revoke-signing-key", + result, + "signing-key", + request.KeyId, + Correlation(context)); + return revoked + ? Results.Ok(new OperatorActionResponse { Status = "completed" }) + : Error(RendezvousErrorCode.NotFound, "The requested resource was not found."); + } + + private static IResult BeginDrain( + [FromBody] BeginDrainRequest request, + [FromHeader(Name = "Authorization")] string? authorization, + [FromServices] PrincipalCredentialService credentials, + [FromServices] IWallClock clock, + [FromServices] OperatorService service, + [FromServices] AuditTrail audit, + [FromServices] RendezvousTelemetry telemetry, + HttpContext context, + CancellationToken cancellationToken) + { + if (!TryAuthorize( + authorization, + OperatorPermission.ManagePolicy, + "begin-drain", + credentials, + clock, + audit, + telemetry, + context, + out OperatorPrincipal? principal, + out IResult? failure)) + { + return failure!; + } + + if (!string.Equals(request.Confirmation, "DRAIN", StringComparison.Ordinal)) + { + AuditRejected(audit, principal!, "begin-drain", "service", "rendezvous", context); + return BadRequest("The confirmation value must be exactly 'DRAIN'."); + } + + service.BeginDrain(cancellationToken); + audit.Record( + principal!.Subject, + "begin-drain", + "succeeded", + "service", + "rendezvous", + Correlation(context)); + return Results.Ok(new OperatorActionResponse { Status = "draining" }); + } + + private static bool TryAuthorize( + string? authorization, + OperatorPermission requiredPermission, + string operation, + PrincipalCredentialService credentials, + IWallClock clock, + AuditTrail audit, + RendezvousTelemetry telemetry, + HttpContext context, + out OperatorPrincipal? principal, + out IResult? failure) + { + principal = null; + failure = null; + const string prefix = "Bearer "; + if (authorization is null + || !authorization.StartsWith(prefix, StringComparison.OrdinalIgnoreCase)) + { + telemetry.RecordOperatorAuthentication("rejected"); + failure = AuthenticationRequired(context); + return false; + } + + CredentialValidationResult validation = credentials.Validate( + authorization[prefix.Length..], + clock.UtcNow); + if (!validation.IsValid || validation.Principal is not OperatorPrincipal candidate) + { + telemetry.RecordOperatorAuthentication("rejected"); + failure = AuthenticationRequired(context); + return false; + } + + if (!candidate.Permissions.Contains(requiredPermission)) + { + telemetry.RecordOperatorAuthentication("forbidden"); + audit.Record( + candidate.Subject, + operation, + "forbidden", + "operator-operation", + operation, + Correlation(context)); + failure = Error(RendezvousErrorCode.Forbidden, "The operator is not authorized for this operation."); + return false; + } + + telemetry.RecordOperatorAuthentication("accepted"); + principal = candidate; + return true; + } + + private static IResult StoreActionResult( + StoreResultCode code, + AuditTrail audit, + OperatorPrincipal principal, + string action, + string targetKind, + string targetIdentifier, + HttpContext context, + int? affectedResources) + { + string auditResult = code == StoreResultCode.Success + ? "succeeded" + : code.ToString().ToLowerInvariant(); + audit.Record( + principal.Subject, + action, + auditResult, + targetKind, + targetIdentifier, + Correlation(context)); + return code switch + { + StoreResultCode.Success => Results.Ok(new OperatorActionResponse + { + Status = "completed", + AffectedResources = affectedResources, + }), + StoreResultCode.NotFound => Error( + RendezvousErrorCode.NotFound, + "The requested resource was not found."), + StoreResultCode.CapacityExceeded => Error( + RendezvousErrorCode.CapacityExceeded, + "The operation could not be retained within the configured capacity."), + StoreResultCode.ServiceUnavailable or StoreResultCode.Draining => Error( + RendezvousErrorCode.ServiceUnavailable, + "The service is not available for this operation."), + _ => Error(RendezvousErrorCode.Conflict, "The operation could not be completed."), + }; + } + + private static void AuditRejected( + AuditTrail audit, + OperatorPrincipal principal, + string action, + string targetKind, + string? targetIdentifier, + HttpContext context) => audit.Record( + principal.Subject, + action, + "rejected", + targetKind, + targetIdentifier ?? string.Empty, + Correlation(context)); + + private static string Correlation(HttpContext context) => + context.Response.Headers[CorrelationHeader].ToString() is { Length: > 0 } value + ? value + : "unavailable"; + + private static IResult AuthenticationRequired(HttpContext context) + { + context.Response.Headers.WWWAuthenticate = "Bearer realm=\"operator\""; + return Error( + RendezvousErrorCode.AuthenticationRequired, + "A valid operator bearer credential is required."); + } + + private static IResult BadRequest(string message) => Error(RendezvousErrorCode.InvalidRequest, message); + + private static IResult Error(RendezvousErrorCode code, string message) => Results.Json( + new ApiError { Code = code, Message = message }, + ContractJson.Options, + statusCode: code switch + { + RendezvousErrorCode.AuthenticationRequired => StatusCodes.Status401Unauthorized, + RendezvousErrorCode.Forbidden => StatusCodes.Status403Forbidden, + RendezvousErrorCode.NotFound => StatusCodes.Status404NotFound, + RendezvousErrorCode.CapacityExceeded => StatusCodes.Status429TooManyRequests, + RendezvousErrorCode.ServiceUnavailable => StatusCodes.Status503ServiceUnavailable, + RendezvousErrorCode.Conflict => StatusCodes.Status409Conflict, + _ => StatusCodes.Status400BadRequest, + }); +} diff --git a/src/FinalFactory.Rendezvous.Server/Operations/OperatorModels.cs b/src/FinalFactory.Rendezvous.Server/Operations/OperatorModels.cs new file mode 100644 index 0000000..4aa2101 --- /dev/null +++ b/src/FinalFactory.Rendezvous.Server/Operations/OperatorModels.cs @@ -0,0 +1,82 @@ +namespace FinalFactory.Rendezvous.Server.Operations; + +internal sealed record OperatorStatusResponse +{ + public required string Status { get; init; } + public required OperatorReadinessResponse Readiness { get; init; } + public required OperatorStoreResponse Store { get; init; } + public required IReadOnlyList Tenants { get; init; } + public required IReadOnlyList SigningKeys { get; init; } + public required IReadOnlyDictionary AuditCounts { get; init; } +} + +internal sealed record OperatorReadinessResponse +{ + public required bool HttpListener { get; init; } + public required bool UdpIpv4Listener { get; init; } + public required bool UdpIpv6Listener { get; init; } + public required bool Provisioning { get; init; } + public required bool Store { get; init; } + public required bool Draining { get; init; } +} + +internal sealed record OperatorStoreResponse +{ + public required int ActiveListings { get; init; } + public required int FreshPresenceBindings { get; init; } + public required int ActiveJoinAttempts { get; init; } + public required int RetainedOutcomeReports { get; init; } + public required int ReplayMarkers { get; init; } + public required int PrincipalRevocations { get; init; } + public required int IdempotencyEntries { get; init; } + public required long MaintenanceSweeps { get; init; } + public required long ExpiryChurn { get; init; } +} + +internal sealed record OperatorTenantResponse +{ + public required string GameId { get; init; } + public required string EnvironmentId { get; init; } + public required string Status { get; init; } +} + +internal sealed record OperatorSigningKeyResponse +{ + public required string KeyId { get; init; } + public required string Status { get; init; } + public required DateTimeOffset SignUntil { get; init; } + public required DateTimeOffset VerifyUntil { get; init; } + public string? GameId { get; init; } + public string? EnvironmentId { get; init; } + public required IReadOnlyList CredentialKinds { get; init; } +} + +internal sealed record OperatorActionResponse +{ + public required string Status { get; init; } + public int? AffectedResources { get; init; } +} + +internal sealed record RevokeListingRequest +{ + public required string ListingId { get; init; } + public required string ConfirmListingId { get; init; } +} + +internal sealed record RevokePrincipalRequest +{ + public required string Subject { get; init; } + public required string ConfirmSubject { get; init; } + public required int LifetimeSeconds { get; init; } +} + +internal sealed record RevokeSigningKeyRequest +{ + public required string KeyId { get; init; } + public required string ConfirmKeyId { get; init; } +} + +internal sealed record BeginDrainRequest +{ + public required string Confirmation { get; init; } +} diff --git a/src/FinalFactory.Rendezvous.Server/Operations/OperatorService.cs b/src/FinalFactory.Rendezvous.Server/Operations/OperatorService.cs new file mode 100644 index 0000000..1447ba3 --- /dev/null +++ b/src/FinalFactory.Rendezvous.Server/Operations/OperatorService.cs @@ -0,0 +1,81 @@ +using FinalFactory.Rendezvous.Contracts; +using FinalFactory.Rendezvous.Server.Observability; +using FinalFactory.Rendezvous.Server.Provisioning; +using FinalFactory.Rendezvous.Server.State; + +namespace FinalFactory.Rendezvous.Server.Operations; + +internal sealed class OperatorService( + InMemoryEphemeralRendezvousStore store, + ProvisioningRuntime provisioning, + RendezvousReadiness readiness, + AuditTrail audit, + IWallClock clock) +{ + public OperatorStatusResponse GetStatus() + { + ReadinessSnapshot readinessSnapshot = readiness.GetSnapshot(); + EphemeralStoreSnapshot storeSnapshot = store.GetSnapshot(); + return new OperatorStatusResponse + { + Status = readinessSnapshot.IsReady ? "ready" : "not-ready", + Readiness = new OperatorReadinessResponse + { + HttpListener = readinessSnapshot.HttpListenerReady, + UdpIpv4Listener = readinessSnapshot.UdpIpv4ListenerReady, + UdpIpv6Listener = readinessSnapshot.UdpIpv6ListenerReady, + Provisioning = readinessSnapshot.ProvisioningReady, + Store = readinessSnapshot.StoreAvailable, + Draining = readinessSnapshot.Draining, + }, + Store = new OperatorStoreResponse + { + ActiveListings = storeSnapshot.ActiveListings, + FreshPresenceBindings = storeSnapshot.FreshPresenceBindings, + ActiveJoinAttempts = storeSnapshot.ActiveJoinAttempts, + RetainedOutcomeReports = storeSnapshot.RetainedOutcomeReports, + ReplayMarkers = storeSnapshot.ReplayMarkers, + PrincipalRevocations = storeSnapshot.PrincipalRevocations, + IdempotencyEntries = storeSnapshot.IdempotencyEntries, + MaintenanceSweeps = storeSnapshot.MaintenanceSweeps, + ExpiryChurn = storeSnapshot.ExpiryChurn, + }, + Tenants = provisioning.Policies.EnabledPolicies + .OrderBy(static policy => policy.GameId.Value, StringComparer.Ordinal) + .ThenBy(static policy => policy.EnvironmentId.Value, StringComparer.Ordinal) + .Select(static policy => new OperatorTenantResponse + { + GameId = policy.GameId.Value, + EnvironmentId = policy.EnvironmentId.Value, + Status = "enabled", + }) + .ToArray(), + SigningKeys = provisioning.SigningKeys.GetStatuses(clock.UtcNow) + .Select(static key => new OperatorSigningKeyResponse + { + KeyId = key.KeyId, + Status = key.Status, + SignUntil = key.SignUntil, + VerifyUntil = key.VerifyUntil, + GameId = key.GameId, + EnvironmentId = key.EnvironmentId, + CredentialKinds = key.CredentialKinds, + }) + .ToArray(), + AuditCounts = audit.GetAggregateCounts(), + }; + } + + public StoreResult RevokeListing( + SessionListingId listingId, + CancellationToken cancellationToken) => store.RevokeListing(listingId, cancellationToken); + + public StoreResult RevokePrincipal( + string subject, + TimeSpan lifetime, + CancellationToken cancellationToken) => store.RevokePrincipal(subject, lifetime, cancellationToken); + + public bool RevokeSigningKey(string keyId) => provisioning.SigningKeys.Revoke(keyId); + + public void BeginDrain(CancellationToken cancellationToken) => store.BeginDrain(cancellationToken); +} diff --git a/src/FinalFactory.Rendezvous.Server/Program.cs b/src/FinalFactory.Rendezvous.Server/Program.cs index 1cd6180..6693eab 100644 --- a/src/FinalFactory.Rendezvous.Server/Program.cs +++ b/src/FinalFactory.Rendezvous.Server/Program.cs @@ -5,6 +5,8 @@ using FinalFactory.Rendezvous.Server.Browser; using FinalFactory.Rendezvous.Server.ConnectionOutcomes; using FinalFactory.Rendezvous.Server.Http; using FinalFactory.Rendezvous.Server.JoinAttempts; +using FinalFactory.Rendezvous.Server.Observability; +using FinalFactory.Rendezvous.Server.Operations; using FinalFactory.Rendezvous.Server.Provisioning; using FinalFactory.Rendezvous.Server.Sessions; using FinalFactory.Rendezvous.Server.State; @@ -13,6 +15,9 @@ using Microsoft.AspNetCore.HttpOverrides; using Microsoft.OpenApi; WebApplicationBuilder builder = WebApplication.CreateBuilder(args); +builder.Logging.AddFilter( + "Microsoft.AspNetCore.Diagnostics.ExceptionHandlerMiddleware", + LogLevel.None); bool isOpenApiGeneration = string.Equals( System.Reflection.Assembly.GetEntryAssembly()?.GetName().Name, "GetDocument.Insider", @@ -61,6 +66,14 @@ builder.Services.AddOpenApi("v1", static options => In = ParameterLocation.Header, Description = "Attempt-scoped client capability returned only to the joining caller.", }; + const string operatorSchemeName = "OperatorBearer"; + document.Components.SecuritySchemes[operatorSchemeName] = new OpenApiSecurityScheme + { + Type = SecuritySchemeType.Http, + Scheme = "bearer", + BearerFormat = "rv1 operator credential", + Description = "Operator-only credential with an explicit permission set.", + }; HashSet securedOperations = new(StringComparer.Ordinal) { @@ -69,8 +82,17 @@ builder.Services.AddOpenApi("v1", static options => "UpdateSession", "DeleteSession", }; + HashSet operatorOperations = new(StringComparer.Ordinal) + { + "GetOperatorStatus", + "RevokeOperatorListing", + "RevokeOperatorPrincipal", + "RevokeOperatorSigningKey", + "BeginOperatorDrain", + }; OpenApiSecuritySchemeReference reference = new(schemeName, document, null); OpenApiSecuritySchemeReference attemptReference = new(attemptSchemeName, document, null); + OpenApiSecuritySchemeReference operatorReference = new(operatorSchemeName, document, null); foreach (OpenApiPathItem path in document.Paths.Values) { if (path.Operations is null) @@ -99,29 +121,58 @@ builder.Services.AddOpenApi("v1", static options => }); } + foreach (OpenApiOperation operation in path.Operations.Values.Where( + operation => operatorOperations.Contains( + operation.OperationId ?? string.Empty))) + { + operation.Security ??= []; + operation.Security.Add(new OpenApiSecurityRequirement + { + [operatorReference] = [], + }); + } + foreach (OpenApiOperation operation in path.Operations.Values) { - if (operation.Responses is null - || !operation.Responses.TryGetValue( - StatusCodes.Status429TooManyRequests.ToString( - System.Globalization.CultureInfo.InvariantCulture), - out IOpenApiResponse? response) - || response is not OpenApiResponse concreteResponse) + if (operation.Responses is null) { continue; } - concreteResponse.Headers ??= - new Dictionary(StringComparer.OrdinalIgnoreCase); - concreteResponse.Headers["Retry-After"] = new OpenApiHeader + foreach ((string status, IOpenApiResponse response) in operation.Responses) { - Description = "Whole seconds before the caller should retry (1-60).", - Schema = new OpenApiSchema + if (response is not OpenApiResponse concreteResponse) { - Type = JsonSchemaType.Integer, - Format = "int32", - }, - }; + continue; + } + + concreteResponse.Headers ??= + new Dictionary(StringComparer.OrdinalIgnoreCase); + concreteResponse.Headers["X-Rendezvous-Correlation-ID"] = new OpenApiHeader + { + Description = "Safe request correlation identifier generated by the service.", + Schema = new OpenApiSchema + { + Type = JsonSchemaType.String, + }, + }; + if (string.Equals( + status, + StatusCodes.Status429TooManyRequests.ToString( + System.Globalization.CultureInfo.InvariantCulture), + StringComparison.Ordinal)) + { + concreteResponse.Headers["Retry-After"] = new OpenApiHeader + { + Description = "Whole seconds before the caller should retry (1-60).", + Schema = new OpenApiSchema + { + Type = JsonSchemaType.Integer, + Format = "int32", + }, + }; + } + } } } @@ -166,8 +217,18 @@ builder.Services && addresses.All( static value => IPAddress.TryParse(value, out _)), "Trusted proxy addresses must contain at most 32 literal IP addresses.") + .Validate( + options => options.OperatorAllowedAddresses is { Length: <= 32 } addresses + && addresses.All( + static value => IPAddress.TryParse(value, out _)), + "Operator allowed addresses must contain at most 32 literal IP addresses.") .ValidateOnStart(); builder.Services.AddSingleton(); +builder.Services + .AddOptions() + .BindConfiguration(AuditOptions.SectionName) + .ValidateDataAnnotations() + .ValidateOnStart(); AbuseProtectionOptions configuredAbuseProtection = builder.Configuration .GetSection(AbuseProtectionOptions.SectionName) .Get() ?? new AbuseProtectionOptions(); @@ -180,8 +241,13 @@ InMemoryEphemeralRendezvousStore stateStore = new( stateOptions, rendezvousClock, rendezvousClock); +builder.Services.AddSingleton(stateStore); builder.Services.AddSingleton(stateStore); builder.Services.AddSingleton(rendezvousClock); +builder.Services.AddSingleton(rendezvousClock); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); if (isOpenApiGeneration) { @@ -214,6 +280,7 @@ else builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); + builder.Services.AddSingleton(); builder.Services.AddSingleton(new ProvisioningReadiness(true)); } @@ -246,34 +313,13 @@ if (TrustedProxyForwarding.IsEnabled(configuredAbuseProtection)) { app.UseForwardedHeaders(); } +app.UseMiddleware(); app.UseExceptionHandler(); app.UseMiddleware(); app.MapOpenApi(); app.MapRendezvousContractEndpoints(); -app.MapGet( - "/health/live", - static () => Results.Ok(new HealthResponse { Status = "live" })) - .Produces() - .Produces(StatusCodes.Status429TooManyRequests) - .WithName("GetLiveness") - .WithTags("Health"); -app.MapGet( - "/health/ready", - static ( - UdpMediatorService mediator, - ProvisioningReadiness provisioning, - IEphemeralRendezvousStore state) => - mediator.LocalEndpoint is null - || !provisioning.IsReady - || !state.IsAvailable - || state.IsDraining - ? Results.StatusCode(StatusCodes.Status503ServiceUnavailable) - : Results.Ok(new HealthResponse { Status = "ready" })) - .Produces() - .Produces(StatusCodes.Status429TooManyRequests) - .Produces(StatusCodes.Status503ServiceUnavailable) - .WithName("GetReadiness") - .WithTags("Health"); +app.MapOperatorEndpoints(); +app.MapRendezvousHealthEndpoints(); await app.RunAsync(); diff --git a/src/FinalFactory.Rendezvous.Server/Provisioning/SigningKeyRing.cs b/src/FinalFactory.Rendezvous.Server/Provisioning/SigningKeyRing.cs index f199364..da50389 100644 --- a/src/FinalFactory.Rendezvous.Server/Provisioning/SigningKeyRing.cs +++ b/src/FinalFactory.Rendezvous.Server/Provisioning/SigningKeyRing.cs @@ -142,8 +142,36 @@ internal sealed class SigningKeyRing : IDisposable return VerificationKeyLookup.Available; } - public bool Revoke(string keyId) => - _keys.ContainsKey(keyId) && _runtimeRevocations.TryAdd(keyId, 0); + public bool Revoke(string keyId) + { + if (!_keys.ContainsKey(keyId)) + { + return false; + } + + _runtimeRevocations.TryAdd(keyId, 0); + return true; + } + + public IReadOnlyList GetStatuses(DateTimeOffset now) => _keys.Values + .OrderBy(static key => key.KeyId, StringComparer.Ordinal) + .Select(key => new SigningKeyStatus( + key.KeyId, + IsRevoked(key) + ? "revoked" + : now < key.NotBefore + ? "not-yet-valid" + : now < key.SignUntil + ? "signing" + : now < key.VerifyUntil + ? "verify-only" + : "retired", + key.SignUntil, + key.VerifyUntil, + key.GameId, + key.EnvironmentId, + key.CredentialKinds.Select(static kind => kind.ToString()).Order().ToArray())) + .ToArray(); public void Dispose() { @@ -210,6 +238,15 @@ internal sealed class SigningKeyRing : IDisposable } } +internal sealed record SigningKeyStatus( + string KeyId, + string Status, + DateTimeOffset SignUntil, + DateTimeOffset VerifyUntil, + string? GameId, + string? EnvironmentId, + IReadOnlyList CredentialKinds); + internal sealed class SigningKey : IDisposable { private byte[]? _material; diff --git a/src/FinalFactory.Rendezvous.Server/State/EphemeralStateContracts.cs b/src/FinalFactory.Rendezvous.Server/State/EphemeralStateContracts.cs index 4b8fa7e..cd15893 100644 --- a/src/FinalFactory.Rendezvous.Server/State/EphemeralStateContracts.cs +++ b/src/FinalFactory.Rendezvous.Server/State/EphemeralStateContracts.cs @@ -297,6 +297,7 @@ internal sealed record StoredJoinAttempt public required SecretFingerprint ConnectionTicketFingerprint { get; init; } public NetworkEndpoint? DedicatedFallback { get; init; } public required DateTimeOffset ExpiresAt { get; init; } + public required TimeSpan CreatedAtMonotonic { get; init; } public required DateTimeOffset ConnectionTicketExpiresAt { get; init; } public AttemptEndpointBinding? HostEndpoint { get; init; } public AttemptEndpointBinding? ClientEndpoint { get; init; } diff --git a/src/FinalFactory.Rendezvous.Server/State/InMemoryEphemeralRendezvousStore.cs b/src/FinalFactory.Rendezvous.Server/State/InMemoryEphemeralRendezvousStore.cs index 060d0e1..40343ca 100644 --- a/src/FinalFactory.Rendezvous.Server/State/InMemoryEphemeralRendezvousStore.cs +++ b/src/FinalFactory.Rendezvous.Server/State/InMemoryEphemeralRendezvousStore.cs @@ -23,7 +23,10 @@ internal sealed class InMemoryEphemeralRendezvousStore : IEphemeralRendezvousSto private TimeSpan? _drainDeadline; private TimeSpan _nextUdpMaintenance; private long _maintenanceSweepCount; + private long _expiryChurn; private bool _available = true; + private EphemeralStoreSnapshot? _metricsSnapshot; + private TimeSpan _metricsSnapshotAt = TimeSpan.MinValue; public InMemoryEphemeralRendezvousStore( EphemeralStoreOptions options, @@ -66,6 +69,50 @@ internal sealed class InMemoryEphemeralRendezvousStore : IEphemeralRendezvousSto } } + internal EphemeralStoreSnapshot GetSnapshot() + { + lock (_gate) + { + TimeSpan now = _monotonicClock.Elapsed; + Cleanup(now); + EphemeralStoreSnapshot snapshot = CreateSnapshot(); + _metricsSnapshot = snapshot; + _metricsSnapshotAt = now; + return snapshot; + } + } + + internal EphemeralStoreSnapshot GetMetricsSnapshot() + { + lock (_gate) + { + TimeSpan now = _monotonicClock.Elapsed; + if (_metricsSnapshot is null + || now < _metricsSnapshotAt + || now - _metricsSnapshotAt >= TimeSpan.FromMilliseconds(100)) + { + Cleanup(now); + _metricsSnapshot = CreateSnapshot(); + _metricsSnapshotAt = now; + } + + return _metricsSnapshot; + } + } + + private EphemeralStoreSnapshot CreateSnapshot() => new( + _listings.Count, + _presence.Count, + _attempts.Count, + _outcomeReports.Count, + _replay.Count, + _revocations.Count, + _idempotency.Count, + _maintenanceSweepCount, + _expiryChurn, + _available, + _drainDeadline.HasValue); + public StoreResult CreateListing( CreateListingCommand command, CancellationToken cancellationToken = default) => Atomic(now => @@ -400,6 +447,7 @@ internal sealed class InMemoryEphemeralRendezvousStore : IEphemeralRendezvousSto AttemptEntry attempt = new( command, + now, now + _options.JoinAttemptLifetime, WallDeadline(now, _options.JoinAttemptLifetime)); _attempts.Add(command.AttemptId, attempt); @@ -729,6 +777,10 @@ internal sealed class InMemoryEphemeralRendezvousStore : IEphemeralRendezvousSto return new(StoreResultCode.CapacityExceeded); } + int activeResourcesBefore = _listings.Count + + _presence.Count + + _attempts.Count + + _outcomeReports.Count; _revocations[subject] = now + lifetime; SessionListingId[] listings = _listings .Where(item => string.Equals(item.Value.Definition.OwnerSubject, subject, StringComparison.Ordinal)) @@ -756,7 +808,11 @@ internal sealed class InMemoryEphemeralRendezvousStore : IEphemeralRendezvousSto _outcomeReports.Remove(attemptId); } - return new(StoreResultCode.Success, listings.Length + attempts.Length); + int activeResourcesAfter = _listings.Count + + _presence.Count + + _attempts.Count + + _outcomeReports.Count; + return new(StoreResultCode.Success, activeResourcesBefore - activeResourcesAfter); }, cancellationToken); public void BeginDrain(CancellationToken cancellationToken = default) @@ -768,6 +824,7 @@ internal sealed class InMemoryEphemeralRendezvousStore : IEphemeralRendezvousSto if (!_drainDeadline.HasValue) { _drainDeadline = _monotonicClock.Elapsed + _options.GracefulDrainLifetime; + _metricsSnapshot = null; } } } @@ -778,6 +835,7 @@ internal sealed class InMemoryEphemeralRendezvousStore : IEphemeralRendezvousSto { _available = false; ClearActiveState(); + _metricsSnapshot = null; } } @@ -801,7 +859,9 @@ internal sealed class InMemoryEphemeralRendezvousStore : IEphemeralRendezvousSto _nextUdpMaintenance = now + UdpMaintenanceInterval; } - return operation(now); + StoreResult result = operation(now); + _metricsSnapshot = null; + return result; } } @@ -838,44 +898,54 @@ internal sealed class InMemoryEphemeralRendezvousStore : IEphemeralRendezvousSto ClearActiveState(); } - RemoveExpired(_revocations, now); - RemoveExpired(_replay, now); - foreach (string key in _idempotency - .Where(item => item.Value.Deadline <= now) - .Select(static item => item.Key) - .ToArray()) + _expiryChurn += RemoveExpired(_revocations, now); + _expiryChurn += RemoveExpired(_replay, now); + string[] expiredIdempotency = _idempotency + .Where(item => item.Value.Deadline <= now) + .Select(static item => item.Key) + .ToArray(); + _expiryChurn += expiredIdempotency.Length; + foreach (string key in expiredIdempotency) { _idempotency.Remove(key); } - foreach (MediationHandle handle in _presence - .Where(item => item.Value.Deadline <= now) - .Select(static item => item.Key) - .ToArray()) + MediationHandle[] expiredPresence = _presence + .Where(item => item.Value.Deadline <= now) + .Select(static item => item.Key) + .ToArray(); + _expiryChurn += expiredPresence.Length; + foreach (MediationHandle handle in expiredPresence) { _presence.Remove(handle); } - foreach (JoinAttemptId attemptId in _attempts - .Where(item => item.Value.Deadline <= now) - .Select(static item => item.Key) - .ToArray()) + JoinAttemptId[] expiredAttempts = _attempts + .Where(item => item.Value.Deadline <= now) + .Select(static item => item.Key) + .ToArray(); + _expiryChurn += expiredAttempts.Length; + foreach (JoinAttemptId attemptId in expiredAttempts) { RemoveAttempt(attemptId); } - foreach (JoinAttemptId attemptId in _outcomeReports - .Where(item => item.Value.Deadline <= now) - .Select(static item => item.Key) - .ToArray()) + JoinAttemptId[] expiredOutcomes = _outcomeReports + .Where(item => item.Value.Deadline <= now) + .Select(static item => item.Key) + .ToArray(); + _expiryChurn += expiredOutcomes.Length; + foreach (JoinAttemptId attemptId in expiredOutcomes) { _outcomeReports.Remove(attemptId); } - foreach (SessionListingId listingId in _listings - .Where(item => item.Value.LeaseDeadline <= now) - .Select(static item => item.Key) - .ToArray()) + SessionListingId[] expiredListings = _listings + .Where(item => item.Value.LeaseDeadline <= now) + .Select(static item => item.Key) + .ToArray(); + _expiryChurn += expiredListings.Length; + foreach (SessionListingId listingId in expiredListings) { RemoveListing(listingId); } @@ -959,6 +1029,7 @@ internal sealed class InMemoryEphemeralRendezvousStore : IEphemeralRendezvousSto ClientCapabilityFingerprint = entry.Command.ClientCapabilityFingerprint, ConnectionTicketFingerprint = entry.Command.ConnectionTicketFingerprint, DedicatedFallback = StoredListing.CopyEndpoint(entry.Command.DedicatedFallback), + CreatedAtMonotonic = entry.CreatedAtMonotonic, ExpiresAt = entry.WallExpiresAt, ConnectionTicketExpiresAt = entry.TicketWallExpiresAt ?? default, HostEndpoint = entry.HostEndpoint, @@ -968,15 +1039,18 @@ internal sealed class InMemoryEphemeralRendezvousStore : IEphemeralRendezvousSto IsCancelled = entry.IsCancelled, }; - private static void RemoveExpired(Dictionary entries, TimeSpan now) + private static int RemoveExpired(Dictionary entries, TimeSpan now) { - foreach (string key in entries - .Where(item => item.Value <= now) - .Select(static item => item.Key) - .ToArray()) + string[] expired = entries + .Where(item => item.Value <= now) + .Select(static item => item.Key) + .ToArray(); + foreach (string key in expired) { entries.Remove(key); } + + return expired.Length; } private static void ValidateListing(ListingDefinition listing) @@ -1101,10 +1175,12 @@ internal sealed class InMemoryEphemeralRendezvousStore : IEphemeralRendezvousSto private sealed class AttemptEntry( CreateJoinAttemptCommand command, + TimeSpan createdAtMonotonic, TimeSpan deadline, DateTimeOffset wallExpiresAt) { public CreateJoinAttemptCommand Command { get; } = command; + public TimeSpan CreatedAtMonotonic { get; } = createdAtMonotonic; public SecretFingerprint HostCapabilityFingerprint { get; } = command.HostCapabilityFingerprint; public SecretFingerprint ClientCapabilityFingerprint { get; } = command.ClientCapabilityFingerprint; public SecretFingerprint ConnectionTicketFingerprint { get; } = command.ConnectionTicketFingerprint; @@ -1137,3 +1213,16 @@ internal sealed class InMemoryEphemeralRendezvousStore : IEphemeralRendezvousSto object ResourceId, TimeSpan Deadline); } + +internal sealed record EphemeralStoreSnapshot( + int ActiveListings, + int FreshPresenceBindings, + int ActiveJoinAttempts, + int RetainedOutcomeReports, + int ReplayMarkers, + int PrincipalRevocations, + int IdempotencyEntries, + long MaintenanceSweeps, + long ExpiryChurn, + bool IsAvailable, + bool IsDraining); diff --git a/src/FinalFactory.Rendezvous.Server/Transport/NatMediationProcessor.cs b/src/FinalFactory.Rendezvous.Server/Transport/NatMediationProcessor.cs index 83cc448..1a475d8 100644 --- a/src/FinalFactory.Rendezvous.Server/Transport/NatMediationProcessor.cs +++ b/src/FinalFactory.Rendezvous.Server/Transport/NatMediationProcessor.cs @@ -1,8 +1,10 @@ +using System.Diagnostics; using System.Net; using System.Net.Sockets; using FinalFactory.Rendezvous.Contracts; using FinalFactory.Rendezvous.Server.Abuse; using FinalFactory.Rendezvous.Server.JoinAttempts; +using FinalFactory.Rendezvous.Server.Observability; using FinalFactory.Rendezvous.Server.Sessions; using FinalFactory.Rendezvous.Server.State; @@ -38,7 +40,9 @@ internal sealed class NatMediationProcessor( IEphemeralRendezvousStore store, ISessionCapabilityService capabilities, JoinAttemptService joinAttempts, - AbuseProtectionService? abuseProtection = null) + AbuseProtectionService? abuseProtection = null, + RendezvousTelemetry? telemetry = null, + IMonotonicClock? monotonicClock = null) { public NatMediationResult ProcessDatagram( ReadOnlySpan encoded, @@ -68,6 +72,26 @@ internal sealed class NatMediationProcessor( INatIntroductionSink introductionSink, CancellationToken cancellationToken = default) { + long started = Stopwatch.GetTimestamp(); + using Activity? activity = telemetry?.StartActivity("UDP frozen", ActivityKind.Server); + NatMediationResult result = ProcessDatagramCore( + encoded, + observedPublicEndpoint, + introductionSink, + cancellationToken); + telemetry?.RecordUdp( + "frozen", + result.ToString(), + Stopwatch.GetElapsedTime(started).TotalMilliseconds); + return result; + } + + private NatMediationResult ProcessDatagramCore( + ReadOnlySpan encoded, + IPEndPoint observedPublicEndpoint, + INatIntroductionSink introductionSink, + CancellationToken cancellationToken) + { if (!RendezvousUdpCodec.TryDecode(encoded, out PresenceDatagram? datagram, out _) || datagram is null @@ -141,12 +165,22 @@ internal sealed class NatMediationProcessor( IPEndPoint observedPublicEndpoint, string token, INatIntroductionSink introductionSink, - CancellationToken cancellationToken = default) => ProcessRequestCore( + CancellationToken cancellationToken = default) + { + long started = Stopwatch.GetTimestamp(); + using Activity? activity = telemetry?.StartActivity("UDP litenet", ActivityKind.Server); + NatMediationResult result = ProcessRequestCore( claimedLocalEndpoint, observedPublicEndpoint, token, introductionSink, cancellationToken); + telemetry?.RecordUdp( + "litenet", + result.ToString(), + Stopwatch.GetElapsedTime(started).TotalMilliseconds); + return result; + } private NatMediationResult ProcessRequestCore( IPEndPoint claimedLocalEndpoint, @@ -255,6 +289,14 @@ internal sealed class NatMediationProcessor( try { introductionSink.Introduce(CreatePlan(consumed.Value, ticket.Value)); + if (telemetry is not null && monotonicClock is not null) + { + telemetry.RecordPairingLatency(Math.Max( + 0, + (monotonicClock.Elapsed - consumed.Value.Attempt.CreatedAtMonotonic) + .TotalMilliseconds)); + } + return NatMediationResult.Introduced; } catch (Exception exception) when (exception is SocketException diff --git a/src/FinalFactory.Rendezvous.Server/appsettings.Development.json b/src/FinalFactory.Rendezvous.Server/appsettings.Development.json index 7c433e5..e79baaa 100644 --- a/src/FinalFactory.Rendezvous.Server/appsettings.Development.json +++ b/src/FinalFactory.Rendezvous.Server/appsettings.Development.json @@ -1,5 +1,8 @@ { "Rendezvous": { + "AbuseProtection": { + "OperatorAllowedAddresses": ["127.0.0.1", "::1"] + }, "Provisioning": { "Issuer": "final-factory-rendezvous-development", "Audience": "final-factory-rendezvous", @@ -14,6 +17,14 @@ "NotBefore": "2025-01-01T00:00:00Z", "SignUntil": "2035-01-01T00:00:00Z", "VerifyUntil": "2035-01-02T00:00:00Z" + }, + { + "KeyId": "development-operator-1", + "SecretReference": "development:ephemeral/rendezvous-operator-signing", + "CredentialKinds": ["Operator"], + "NotBefore": "2025-01-01T00:00:00Z", + "SignUntil": "2035-01-01T00:00:00Z", + "VerifyUntil": "2035-01-02T00:00:00Z" } ], "Games": [ diff --git a/src/FinalFactory.Rendezvous.Server/appsettings.json b/src/FinalFactory.Rendezvous.Server/appsettings.json index 08dc57a..8a84953 100644 --- a/src/FinalFactory.Rendezvous.Server/appsettings.json +++ b/src/FinalFactory.Rendezvous.Server/appsettings.json @@ -6,16 +6,25 @@ "MaxDatagramsPerPoll": 256, "PollIntervalMilliseconds": 2 }, + "Audit": { + "MaxEntries": 10000, + "RetentionDays": 30 + }, "AbuseProtection": { "WindowSeconds": 1, "MaxTrackedKeys": 100000, "CriticalTrackedKeyReserve": 2048, "UdpTrackedKeyLimit": 70000, "TrustedProxyAddresses": [], + "OperatorAllowedAddresses": [], "HealthGlobalRequestsPerWindow": 1000, "HealthGlobalConcurrency": 32, "HealthIpPrefixRequestsPerWindow": 120, "HealthIpPrefixConcurrency": 8, + "OperatorGlobalRequestsPerWindow": 1000, + "OperatorGlobalConcurrency": 32, + "OperatorIpPrefixRequestsPerWindow": 120, + "OperatorIpPrefixConcurrency": 8, "HttpGlobalRequestsPerWindow": 20000, "HttpOptionalRequestsPerWindow": 18000, "HttpIpPrefixRequestsPerWindow": 500, diff --git a/tests/FinalFactory.Rendezvous.Tests/Contracts/OpenApiCompatibilityTests.cs b/tests/FinalFactory.Rendezvous.Tests/Contracts/OpenApiCompatibilityTests.cs index 798b12e..58f6235 100644 --- a/tests/FinalFactory.Rendezvous.Tests/Contracts/OpenApiCompatibilityTests.cs +++ b/tests/FinalFactory.Rendezvous.Tests/Contracts/OpenApiCompatibilityTests.cs @@ -11,6 +11,11 @@ public sealed class OpenApiCompatibilityTests "/v1/join-attempts", "/v1/join-attempts/{attemptId}", "/v1/join-attempts/{attemptId}/outcome", + "/v1/operator/drain", + "/v1/operator/keys/revoke", + "/v1/operator/listings/revoke", + "/v1/operator/principals/revoke", + "/v1/operator/status", "/v1/sessions", "/v1/sessions/{listingId}", "/v1/sessions/{listingId}/join-attempts", @@ -104,6 +109,11 @@ public sealed class OpenApiCompatibilityTests Assert.Equal( "X-Rendezvous-Client-Punch-Capability", attemptCapability.GetProperty("name").GetString()); + JsonElement operatorBearer = root.GetProperty("components") + .GetProperty("securitySchemes") + .GetProperty("OperatorBearer"); + Assert.Equal("http", operatorBearer.GetProperty("type").GetString()); + Assert.Equal("bearer", operatorBearer.GetProperty("scheme").GetString()); (string Path, string Method)[] publisherOperations = [ ("/v1/sessions", "post"), @@ -120,6 +130,23 @@ public sealed class OpenApiCompatibilityTests Assert.True(security[0].TryGetProperty("PublisherBearer", out _)); } + (string Path, string Method)[] operatorOperations = + [ + ("/v1/operator/status", "get"), + ("/v1/operator/listings/revoke", "post"), + ("/v1/operator/principals/revoke", "post"), + ("/v1/operator/keys/revoke", "post"), + ("/v1/operator/drain", "post"), + ]; + foreach ((string operationPath, string method) in operatorOperations) + { + JsonElement security = root.GetProperty("paths") + .GetProperty(operationPath) + .GetProperty(method) + .GetProperty("security"); + Assert.True(security[0].TryGetProperty("OperatorBearer", out _)); + } + JsonElement cancelParameters = root.GetProperty("paths") .GetProperty("/v1/join-attempts/{attemptId}") .GetProperty("delete") @@ -157,6 +184,15 @@ public sealed class OpenApiCompatibilityTests static item => item.Name is "get" or "post" or "put" or "delete")) { JsonElement responses = operation.Value.GetProperty("responses"); + foreach (JsonProperty response in responses.EnumerateObject()) + { + JsonElement correlation = response.Value.GetProperty("headers") + .GetProperty("X-Rendezvous-Correlation-ID"); + Assert.Equal( + "string", + correlation.GetProperty("schema").GetProperty("type").GetString()); + } + if (!responses.TryGetProperty("429", out JsonElement overloaded)) { continue; @@ -171,7 +207,7 @@ public sealed class OpenApiCompatibilityTests } } - Assert.Equal(12, overloadContracts); + Assert.Equal(17, overloadContracts); (string Path, string Method)[] bodyOperations = [ ("/v1/sessions", "post"), @@ -180,6 +216,10 @@ public sealed class OpenApiCompatibilityTests ("/v1/sessions/{listingId}", "delete"), ("/v1/join-attempts", "post"), ("/v1/join-attempts/{attemptId}/outcome", "post"), + ("/v1/operator/listings/revoke", "post"), + ("/v1/operator/principals/revoke", "post"), + ("/v1/operator/keys/revoke", "post"), + ("/v1/operator/drain", "post"), ]; foreach ((string operationPath, string method) in bodyOperations) { diff --git a/tests/FinalFactory.Rendezvous.Tests/Observability/ObservabilityTests.cs b/tests/FinalFactory.Rendezvous.Tests/Observability/ObservabilityTests.cs new file mode 100644 index 0000000..3b603d3 --- /dev/null +++ b/tests/FinalFactory.Rendezvous.Tests/Observability/ObservabilityTests.cs @@ -0,0 +1,253 @@ +using System.Collections.Concurrent; +using System.Diagnostics; +using System.Diagnostics.Metrics; +using System.Net; +using FinalFactory.Rendezvous.Server.Observability; +using FinalFactory.Rendezvous.Server.State; +using FinalFactory.Rendezvous.Server.Transport; +using FinalFactory.Rendezvous.Tests.State; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +namespace FinalFactory.Rendezvous.Tests.Observability; + +[CollectionDefinition(RendezvousTelemetryIsolation.Name, DisableParallelization = true)] +public sealed class RendezvousTelemetryIsolation +{ + public const string Name = "Rendezvous telemetry"; +} + +[Collection(RendezvousTelemetryIsolation.Name)] +public sealed class ObservabilityTests +{ + private static readonly HashSet AllowedTagKeys = + [ + "operation", + "status_code", + "result", + "transport", + "partition", + "action", + "outcome", + "elapsed_bucket", + ]; + + [Fact] + public void MetricsAndTracesUseBoundedDimensionsWithoutSensitiveValues() + { + EphemeralStateFixture fixture = new(); + using RendezvousTelemetry telemetry = new(fixture.Store); + List measurements = []; + using MeterListener meterListener = new(); + meterListener.InstrumentPublished = (instrument, listener) => + { + if (instrument.Meter.Name == RendezvousTelemetry.MeterName) + { + listener.EnableMeasurementEvents(instrument); + } + }; + meterListener.SetMeasurementEventCallback((instrument, value, tags, _) => + measurements.Add(new(instrument.Name, value, Tags(tags)))); + meterListener.SetMeasurementEventCallback((instrument, value, tags, _) => + measurements.Add(new(instrument.Name, value, Tags(tags)))); + meterListener.SetMeasurementEventCallback((instrument, value, tags, _) => + measurements.Add(new(instrument.Name, value, Tags(tags)))); + meterListener.Start(); + + Activity? observed = null; + List activityData = []; + using ActivityListener activityListener = new() + { + ShouldListenTo = source => source.Name == RendezvousTelemetry.ActivitySourceName, + Sample = static (ref ActivityCreationOptions _) => + ActivitySamplingResult.AllData, + ActivityStopped = activity => + { + observed = activity; + activityData.Add(activity.DisplayName); + activityData.AddRange(activity.TagObjects.Select(static tag => $"{tag.Key}={tag.Value}")); + }, + }; + ActivitySource.AddActivityListener(activityListener); + + const string secret = "secret-player-token-canary"; + using (telemetry.StartActivity("HTTP GetOperatorStatus", ActivityKind.Server)) + { + telemetry.RecordHttp("GetOperatorStatus", 200, 3.5); + telemetry.RecordUdp("frozen", "Introduced", 1.25); + telemetry.RecordLimiterDrop("udp", "rate-or-concurrency"); + telemetry.RecordAudit("revoke-listing", "succeeded"); + telemetry.RecordConnectionOutcome("Connected", "UnderOneSecond"); + telemetry.RecordOperatorAuthentication("accepted"); + telemetry.RecordPairingLatency(12.5); + } + NatMediationProcessor processor = new(null!, null!, null!, telemetry: telemetry); + Assert.Equal( + NatMediationResult.Dropped, + processor.ProcessRequest( + new IPEndPoint(IPAddress.Parse("10.0.0.8"), 9000), + new IPEndPoint(IPAddress.Parse("203.0.113.8"), 50000), + secret, + NoopIntroductionSink.Instance)); + + meterListener.RecordObservableInstruments(); + Assert.NotNull(observed); + string flattened = string.Join('|', measurements.Select(static item => item.ToString())); + Assert.DoesNotContain(secret, flattened, StringComparison.Ordinal); + Assert.DoesNotContain(secret, string.Join('|', activityData), StringComparison.Ordinal); + Assert.Contains(measurements, static item => item.Name == "rendezvous.http.requests"); + Assert.Contains(measurements, static item => item.Name == "rendezvous.udp.results"); + Assert.Contains(measurements, static item => item.Name == "rendezvous.limiter.drops"); + Assert.Contains(measurements, static item => item.Name == "rendezvous.queue.depth"); + Assert.Contains(measurements, static item => item.Name == "rendezvous.store.active_leases"); + Assert.Contains(measurements, static item => item.Name == "rendezvous.store.expiry_churn"); + Assert.Contains(measurements, static item => item.Name == "rendezvous.pairing.latency"); + Assert.All(measurements.SelectMany(static item => item.Tags), static tag => + Assert.Contains(tag.Key, AllowedTagKeys)); + } + + [Fact] + public void MetricScrapeExpiresIdleStateAndReportsExpiryChurn() + { + EphemeralStateFixture fixture = new(); + Assert.True(fixture.Store.CreateListing(fixture.ListingCommand()).Succeeded); + using RendezvousTelemetry telemetry = new(fixture.Store); + List measurements = []; + using MeterListener listener = new(); + listener.InstrumentPublished = (instrument, meterListener) => + { + if (instrument.Meter.Name == RendezvousTelemetry.MeterName) + { + meterListener.EnableMeasurementEvents(instrument); + } + }; + listener.SetMeasurementEventCallback((instrument, value, tags, _) => + measurements.Add(new(instrument.Name, value, Tags(tags)))); + listener.SetMeasurementEventCallback((instrument, value, tags, _) => + measurements.Add(new(instrument.Name, value, Tags(tags)))); + listener.Start(); + + listener.RecordObservableInstruments(); + Assert.Equal( + 1, + Assert.Single(measurements, static item => item.Name == "rendezvous.store.active_leases").Value); + + fixture.Clock.Advance(TimeSpan.FromSeconds(61)); + measurements.Clear(); + listener.RecordObservableInstruments(); + Assert.Equal( + 0, + Assert.Single(measurements, static item => item.Name == "rendezvous.store.active_leases").Value); + Assert.True(Assert.Single( + measurements, + static item => item.Name == "rendezvous.store.expiry_churn").Value >= 1); + } + + [Fact] + public void AuditTrailFingerprintsIdentifiersEnforcesRetentionAndBoundsCapacity() + { + EphemeralStateFixture fixture = new(); + using RendezvousTelemetry telemetry = new(fixture.Store); + CapturingLogger logger = new(); + ManualTimeProvider time = new(new DateTimeOffset(2026, 7, 16, 0, 0, 0, TimeSpan.Zero)); + AuditTrail audit = new( + Options.Create(new AuditOptions { MaxEntries = 100, RetentionDays = 1 }), + logger, + telemetry, + time); + const string actor = "operator-secret-subject"; + const string target = "player-secret-subject"; + + for (int index = 0; index < 101; index++) + { + audit.Record(actor, "revoke-principal", "succeeded", "principal", target, "safe-correlation"); + } + + IReadOnlyList bounded = audit.GetEntriesForTests(); + Assert.Equal(100, bounded.Count); + Assert.All(bounded, entry => + { + Assert.DoesNotContain(actor, entry.ToString(), StringComparison.Ordinal); + Assert.DoesNotContain(target, entry.ToString(), StringComparison.Ordinal); + Assert.NotEqual(actor, entry.ActorFingerprint); + Assert.NotEqual(target, entry.TargetFingerprint); + }); + Assert.DoesNotContain(actor, string.Join('|', logger.Messages), StringComparison.Ordinal); + Assert.DoesNotContain(target, string.Join('|', logger.Messages), StringComparison.Ordinal); + + time.Advance(TimeSpan.FromDays(2)); + Assert.Empty(audit.GetEntriesForTests()); + Assert.Empty(audit.GetAggregateCounts()); + audit.Record(actor, "inspect-status", "succeeded", "service", "rendezvous", "safe-correlation"); + Assert.Single(audit.GetEntriesForTests()); + Assert.Equal(1, audit.GetAggregateCounts()["inspect-status:succeeded"]); + } + + private static KeyValuePair[] Tags( + ReadOnlySpan> tags) => tags.ToArray(); + + private sealed record Measurement( + string Name, + double Value, + KeyValuePair[] Tags); + + private sealed class ManualTimeProvider(DateTimeOffset now) : TimeProvider + { + private DateTimeOffset _now = now; + public override DateTimeOffset GetUtcNow() => _now; + public void Advance(TimeSpan duration) => _now += duration; + } + + private sealed class NoopIntroductionSink : INatIntroductionSink + { + public static NoopIntroductionSink Instance { get; } = new(); + public void Introduce(NatIntroductionPlan plan) + { + } + } +} + +internal sealed class CapturingLogger : ILogger +{ + public List Messages { get; } = []; + + public IDisposable? BeginScope(TState state) where TState : notnull => null; + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log( + LogLevel logLevel, + EventId eventId, + TState state, + Exception? exception, + Func formatter) => Messages.Add(formatter(state, exception)); +} + +internal sealed class CapturingLoggerProvider : ILoggerProvider +{ + public ConcurrentQueue Messages { get; } = new(); + + public ILogger CreateLogger(string categoryName) => new Sink(Messages); + + public void Dispose() => GC.SuppressFinalize(this); + + private sealed class Sink(ConcurrentQueue messages) : ILogger + { + public IDisposable BeginScope(TState state) where TState : notnull => Scope.Instance; + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log( + LogLevel logLevel, + EventId eventId, + TState state, + Exception? exception, + Func formatter) => messages.Enqueue(formatter(state, exception)); + } + + private sealed class Scope : IDisposable + { + public static Scope Instance { get; } = new(); + public void Dispose() + { + } + } +} diff --git a/tests/FinalFactory.Rendezvous.Tests/Operations/OperatorEndpointTests.cs b/tests/FinalFactory.Rendezvous.Tests/Operations/OperatorEndpointTests.cs new file mode 100644 index 0000000..1231aaf --- /dev/null +++ b/tests/FinalFactory.Rendezvous.Tests/Operations/OperatorEndpointTests.cs @@ -0,0 +1,519 @@ +using System.Diagnostics; +using System.Diagnostics.Metrics; +using System.Net; +using System.Net.Http.Headers; +using System.Net.Http.Json; +using System.Text.Json; +using FinalFactory.Rendezvous.Contracts; +using FinalFactory.Rendezvous.Server.Abuse; +using FinalFactory.Rendezvous.Server.Http; +using FinalFactory.Rendezvous.Server.JoinAttempts; +using FinalFactory.Rendezvous.Server.Observability; +using FinalFactory.Rendezvous.Server.Operations; +using FinalFactory.Rendezvous.Server.Provisioning; +using FinalFactory.Rendezvous.Server.Sessions; +using FinalFactory.Rendezvous.Server.State; +using FinalFactory.Rendezvous.Server.Transport; +using FinalFactory.Rendezvous.Tests.Observability; +using FinalFactory.Rendezvous.Tests.Provisioning; +using FinalFactory.Rendezvous.Tests.State; +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.Hosting.Server; +using Microsoft.AspNetCore.Hosting.Server.Features; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Routing; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; + +namespace FinalFactory.Rendezvous.Tests.Operations; + +[Collection(RendezvousTelemetryIsolation.Name)] +public sealed class OperatorEndpointTests +{ + [Fact] + public async Task OperatorSurfaceSeparatesAuthenticationConfirmsActionsAndRedactsInspection() + { + await using OperatorTestHost host = await OperatorTestHost.StartAsync(); + List telemetryData = []; + using MeterListener meterListener = new(); + meterListener.InstrumentPublished = (instrument, listener) => + { + if (instrument.Meter.Name == RendezvousTelemetry.MeterName) + { + listener.EnableMeasurementEvents(instrument); + } + }; + meterListener.SetMeasurementEventCallback((instrument, value, tags, _) => + CaptureMeasurement(telemetryData, instrument, value, tags)); + meterListener.SetMeasurementEventCallback((instrument, value, tags, _) => + CaptureMeasurement(telemetryData, instrument, value, tags)); + meterListener.SetMeasurementEventCallback((instrument, value, tags, _) => + CaptureMeasurement(telemetryData, instrument, value, tags)); + meterListener.Start(); + using ActivityListener activityListener = new() + { + ShouldListenTo = static source => source.Name == RendezvousTelemetry.ActivitySourceName, + Sample = static (ref ActivityCreationOptions _) => + ActivitySamplingResult.AllData, + ActivityStopped = activity => + { + telemetryData.Add(activity.DisplayName); + telemetryData.AddRange(activity.TagObjects.Select(static tag => $"{tag.Key}={tag.Value}")); + }, + }; + ActivitySource.AddActivityListener(activityListener); + + using HttpResponseMessage liveBeforeDependencies = await host.Client.GetAsync("/health/live"); + Assert.Equal(HttpStatusCode.OK, liveBeforeDependencies.StatusCode); + using HttpResponseMessage readyBeforeUdp = await host.Client.GetAsync("/health/ready"); + Assert.Equal(HttpStatusCode.ServiceUnavailable, readyBeforeUdp.StatusCode); + + using HttpResponseMessage unauthenticated = await host.Client.GetAsync("/v1/operator/status"); + Assert.Equal(HttpStatusCode.Unauthorized, unauthenticated.StatusCode); + AuthenticationHeaderValue challenge = Assert.Single( + unauthenticated.Headers.WwwAuthenticate); + Assert.Equal("Bearer", challenge.Scheme); + Assert.Equal("realm=\"operator\"", challenge.Parameter); + + using HttpResponseMessage publisher = await SendAsync( + host, + HttpMethod.Get, + "/v1/operator/status", + host.PublisherCredential); + Assert.Equal(HttpStatusCode.Unauthorized, publisher.StatusCode); + + using HttpResponseMessage status = await SendAsync( + host, + HttpMethod.Get, + "/v1/operator/status", + host.ReadOnlyOperatorCredential); + Assert.Equal(HttpStatusCode.OK, status.StatusCode); + string statusJson = await status.Content.ReadAsStringAsync(); + Assert.Contains("not-ready", statusJson, StringComparison.Ordinal); + Assert.DoesNotContain(host.OwnerCanary, statusJson, StringComparison.Ordinal); + Assert.DoesNotContain("203.0.113.25", statusJson, StringComparison.Ordinal); + Assert.DoesNotContain("metadata", statusJson, StringComparison.OrdinalIgnoreCase); + OperatorStatusResponse? operatorStatus = JsonSerializer.Deserialize( + statusJson, + ContractJson.Options); + Assert.Contains(operatorStatus!.Tenants, static tenant => + tenant.GameId == "space-game" + && tenant.EnvironmentId == "production" + && tenant.Status == "enabled"); + Assert.Contains(operatorStatus.SigningKeys, static key => + key.KeyId == OperatorTestHost.OperatorKeyId + && key.Status == "signing" + && key.CredentialKinds.SequenceEqual(["Operator"])); + + await host.StartUdpAsync(); + using HttpResponseMessage readyAfterUdp = await host.Client.GetAsync("/health/ready"); + Assert.Equal(HttpStatusCode.OK, readyAfterUdp.StatusCode); + + using HttpResponseMessage exception = await SendAsync( + host, + HttpMethod.Post, + "/test/exception", + host.FullOperatorCredential); + Assert.Equal(HttpStatusCode.InternalServerError, exception.StatusCode); + using HttpResponseMessage saturatedPublic = await host.Client.GetAsync("/test/public"); + Assert.Equal(HttpStatusCode.TooManyRequests, saturatedPublic.StatusCode); + + using HttpResponseMessage forbidden = await SendAsync( + host, + HttpMethod.Post, + "/v1/operator/drain", + host.ReadOnlyOperatorCredential, + new BeginDrainRequest { Confirmation = "DRAIN" }); + Assert.Equal(HttpStatusCode.Forbidden, forbidden.StatusCode); + + SessionListingId listingId = host.CreateListing(host.OwnerCanary); + using HttpResponseMessage unconfirmedListing = await SendAsync( + host, + HttpMethod.Post, + "/v1/operator/listings/revoke", + host.FullOperatorCredential, + new RevokeListingRequest + { + ListingId = listingId.ToString(), + ConfirmListingId = Guid.NewGuid().ToString("D"), + }); + Assert.Equal(HttpStatusCode.BadRequest, unconfirmedListing.StatusCode); + Assert.True(host.Store.GetListing(listingId, false).Succeeded); + + using HttpResponseMessage revokedListing = await SendAsync( + host, + HttpMethod.Post, + "/v1/operator/listings/revoke", + host.FullOperatorCredential, + new RevokeListingRequest + { + ListingId = listingId.ToString(), + ConfirmListingId = listingId.ToString(), + }); + Assert.Equal(HttpStatusCode.OK, revokedListing.StatusCode); + Assert.Equal(StoreResultCode.NotFound, host.Store.GetListing(listingId, false).Code); + + const string principalCanary = "publisher-player-canary"; + SessionListingId principalListing = host.CreateListing(principalCanary); + using HttpResponseMessage unconfirmedPrincipal = await SendAsync( + host, + HttpMethod.Post, + "/v1/operator/principals/revoke", + host.FullOperatorCredential, + new RevokePrincipalRequest + { + Subject = principalCanary, + ConfirmSubject = "different-subject", + LifetimeSeconds = 60, + }); + Assert.Equal(HttpStatusCode.BadRequest, unconfirmedPrincipal.StatusCode); + Assert.True(host.Store.GetListing(principalListing, false).Succeeded); + + using HttpResponseMessage revokedPrincipal = await SendAsync( + host, + HttpMethod.Post, + "/v1/operator/principals/revoke", + host.FullOperatorCredential, + new RevokePrincipalRequest + { + Subject = principalCanary, + ConfirmSubject = principalCanary, + LifetimeSeconds = 60, + }); + Assert.Equal(HttpStatusCode.OK, revokedPrincipal.StatusCode); + OperatorActionResponse? principalResult = await revokedPrincipal.Content + .ReadFromJsonAsync(ContractJson.Options); + Assert.Equal(1, principalResult!.AffectedResources); + Assert.Equal(StoreResultCode.NotFound, host.Store.GetListing(principalListing, false).Code); + StoreResult blockedPublisher = host.CreateListingResult( + principalCanary, + out _); + Assert.Equal(StoreResultCode.Revoked, blockedPublisher.Code); + + using HttpResponseMessage drain = await SendAsync( + host, + HttpMethod.Post, + "/v1/operator/drain", + host.FullOperatorCredential, + new BeginDrainRequest { Confirmation = "DRAIN" }); + Assert.Equal(HttpStatusCode.OK, drain.StatusCode); + Assert.True(host.Store.IsDraining); + using HttpResponseMessage liveDuringDrain = await host.Client.GetAsync("/health/live"); + Assert.Equal(HttpStatusCode.OK, liveDuringDrain.StatusCode); + using HttpResponseMessage readyDuringDrain = await host.Client.GetAsync("/health/ready"); + Assert.Equal(HttpStatusCode.ServiceUnavailable, readyDuringDrain.StatusCode); + + using HttpResponseMessage firstPublisherKeyRevocation = await SendAsync( + host, + HttpMethod.Post, + "/v1/operator/keys/revoke", + host.FullOperatorCredential, + new RevokeSigningKeyRequest { KeyId = "key-1", ConfirmKeyId = "key-1" }); + Assert.Equal(HttpStatusCode.OK, firstPublisherKeyRevocation.StatusCode); + using HttpResponseMessage repeatedPublisherKeyRevocation = await SendAsync( + host, + HttpMethod.Post, + "/v1/operator/keys/revoke", + host.FullOperatorCredential, + new RevokeSigningKeyRequest { KeyId = "key-1", ConfirmKeyId = "key-1" }); + Assert.Equal(HttpStatusCode.OK, repeatedPublisherKeyRevocation.StatusCode); + + using HttpResponseMessage unconfirmedKey = await SendAsync( + host, + HttpMethod.Post, + "/v1/operator/keys/revoke", + host.FullOperatorCredential, + new RevokeSigningKeyRequest + { + KeyId = OperatorTestHost.OperatorKeyId, + ConfirmKeyId = "different-key", + }); + Assert.Equal(HttpStatusCode.BadRequest, unconfirmedKey.StatusCode); + + using HttpResponseMessage revokedKey = await SendAsync( + host, + HttpMethod.Post, + "/v1/operator/keys/revoke", + host.FullOperatorCredential, + new RevokeSigningKeyRequest + { + KeyId = OperatorTestHost.OperatorKeyId, + ConfirmKeyId = OperatorTestHost.OperatorKeyId, + }); + Assert.Equal(HttpStatusCode.OK, revokedKey.StatusCode); + using HttpResponseMessage afterKeyRevocation = await SendAsync( + host, + HttpMethod.Get, + "/v1/operator/status", + host.FullOperatorCredential); + Assert.Equal(HttpStatusCode.Unauthorized, afterKeyRevocation.StatusCode); + + string auditText = string.Join('|', host.Audit.GetEntriesForTests()); + string logText = string.Join('|', host.AuditLogger.Messages.Concat(host.AllLogs.Messages)); + Assert.DoesNotContain(host.OwnerCanary, auditText, StringComparison.Ordinal); + Assert.DoesNotContain(principalCanary, auditText, StringComparison.Ordinal); + Assert.DoesNotContain(listingId.ToString(), auditText, StringComparison.Ordinal); + Assert.DoesNotContain(host.OwnerCanary, logText, StringComparison.Ordinal); + Assert.DoesNotContain(principalCanary, logText, StringComparison.Ordinal); + Assert.DoesNotContain("exception-secret-canary", logText, StringComparison.Ordinal); + Assert.DoesNotContain(host.FullOperatorCredential, logText, StringComparison.Ordinal); + meterListener.RecordObservableInstruments(); + string telemetryText = string.Join('|', telemetryData); + Assert.DoesNotContain(host.OwnerCanary, telemetryText, StringComparison.Ordinal); + Assert.DoesNotContain(principalCanary, telemetryText, StringComparison.Ordinal); + Assert.DoesNotContain("exception-secret-canary", telemetryText, StringComparison.Ordinal); + Assert.DoesNotContain(host.FullOperatorCredential, telemetryText, StringComparison.Ordinal); + Assert.DoesNotContain(listingId.ToString(), telemetryText, StringComparison.Ordinal); + Assert.DoesNotContain("203.0.113.25", telemetryText, StringComparison.Ordinal); + Assert.Contains(host.Audit.GetEntriesForTests(), static entry => + entry.Action == "begin-drain" && entry.Result == "succeeded"); + Assert.Contains(host.Audit.GetEntriesForTests(), static entry => + entry.Action == "revoke-listing" && entry.Result == "rejected"); + Assert.Contains(host.Audit.GetEntriesForTests(), static entry => + entry.Action == "begin-drain" && entry.Result == "forbidden"); + } + + private static async Task SendAsync( + OperatorTestHost host, + HttpMethod method, + string path, + string bearer, + object? body = null) + { + using HttpRequestMessage request = new(method, path); + request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", bearer); + if (body is not null) + { + request.Content = JsonContent.Create(body, options: ContractJson.Options); + } + + return await host.Client.SendAsync(request); + } + + private static void CaptureMeasurement( + List destination, + Instrument instrument, + T value, + ReadOnlySpan> tags) + where T : struct + { + destination.Add($"{instrument.Name}={value}"); + destination.AddRange(tags.ToArray().Select(static tag => $"{tag.Key}={tag.Value}")); + } + + private sealed class OperatorTestHost : IAsyncDisposable + { + private readonly WebApplication _application; + private int _listingSequence; + private bool _udpStarted; + + private OperatorTestHost( + WebApplication application, + HttpClient client, + ManualRendezvousClock clock, + InMemoryEphemeralRendezvousStore store, + AuditTrail audit, + CapturingLogger auditLogger, + CapturingLoggerProvider allLogs, + string publisherCredential, + string readOnlyOperatorCredential, + string fullOperatorCredential) + { + _application = application; + Client = client; + Clock = clock; + Store = store; + Audit = audit; + AuditLogger = auditLogger; + AllLogs = allLogs; + PublisherCredential = publisherCredential; + ReadOnlyOperatorCredential = readOnlyOperatorCredential; + FullOperatorCredential = fullOperatorCredential; + } + + internal const string OperatorKeyId = "operator-key"; + internal string OwnerCanary { get; } = "publisher-owner-canary"; + internal HttpClient Client { get; } + internal ManualRendezvousClock Clock { get; } + internal InMemoryEphemeralRendezvousStore Store { get; } + internal AuditTrail Audit { get; } + internal CapturingLogger AuditLogger { get; } + internal CapturingLoggerProvider AllLogs { get; } + internal string PublisherCredential { get; } + internal string ReadOnlyOperatorCredential { get; } + internal string FullOperatorCredential { get; } + + internal static async Task StartAsync() + { + ManualRendezvousClock clock = new(ProvisioningTestData.Now); + EphemeralStoreOptions stateOptions = new(); + InMemoryEphemeralRendezvousStore store = new(stateOptions, clock, clock); + SigningKeyOptions publisherKey = ProvisioningTestData.CreateKey(); + SigningKeyOptions operatorKey = ProvisioningTestData.CreateKey( + OperatorKeyId, + "operator-secret", + credentialKinds: [PrincipalCredentialKind.Operator], + gameId: null, + environmentId: null); + ProvisioningOptions options = ProvisioningTestData.CreateOptions(); + options.SigningKeys = [publisherKey, operatorKey]; + ProvisioningRuntime provisioning = ProvisioningRuntime.Create( + options, + ProvisioningTestData.CreateSecrets("secret-1", "operator-secret"), + clock.UtcNow); + string publisherCredential = provisioning.Credentials.Issue( + ProvisioningTestData.CreateDedicatedPublisher(), + clock.UtcNow); + string readOnlyCredential = provisioning.Credentials.Issue( + new OperatorPrincipal( + "operator-readonly", + clock.UtcNow.AddMinutes(10), + [OperatorPermission.ReadPolicy]), + clock.UtcNow); + string fullCredential = provisioning.Credentials.Issue( + new OperatorPrincipal( + "operator-full", + clock.UtcNow.AddMinutes(10), + Enum.GetValues()), + clock.UtcNow); + CapturingLogger auditLogger = new(); + CapturingLoggerProvider allLogs = new(); + EphemeralCapabilityIssuer capabilities = new(); + + WebApplicationBuilder builder = WebApplication.CreateBuilder(); + builder.WebHost.UseUrls("http://127.0.0.1:0"); + builder.Logging.ClearProviders(); + builder.Logging.SetMinimumLevel(LogLevel.Debug); + builder.Logging.AddProvider(allLogs); + builder.Logging.AddFilter( + "Microsoft.AspNetCore.Diagnostics.ExceptionHandlerMiddleware", + LogLevel.None); + builder.Services.ConfigureHttpJsonOptions(static json => + ContractJson.Configure(json.SerializerOptions)); + builder.Services.Configure(static route => + route.ThrowOnBadRequest = true); + builder.Services.AddProblemDetails(); + builder.Services.AddExceptionHandler(); + builder.Services.AddOptions().Configure(static abuse => + { + abuse.OperatorAllowedAddresses = ["127.0.0.1"]; + abuse.HttpGlobalRequestsPerWindow = 1; + abuse.HttpOptionalRequestsPerWindow = 1; + abuse.HttpIpPrefixRequestsPerWindow = 1; + abuse.HttpOptionalIpPrefixRequestsPerWindow = 1; + }); + builder.Services.AddOptions(); + builder.Services.AddOptions().Configure(static udp => + { + udp.ListenAddress = "127.0.0.1"; + udp.Port = 0; + }); + builder.Services.AddSingleton(provisioning); + builder.Services.AddSingleton(provisioning.Policies); + builder.Services.AddSingleton(provisioning.Credentials); + builder.Services.AddSingleton(provisioning.PublisherAuthorization); + builder.Services.AddSingleton(store); + builder.Services.AddSingleton(store); + builder.Services.AddSingleton(clock); + builder.Services.AddSingleton(clock); + builder.Services.AddSingleton(capabilities); + builder.Services.AddSingleton(capabilities); + builder.Services.AddSingleton(); + builder.Services.AddSingleton(); + builder.Services.AddSingleton(); + builder.Services.AddSingleton(); + builder.Services.AddSingleton(); + builder.Services.AddSingleton(); + builder.Services.AddSingleton(new ProvisioningReadiness(true)); + builder.Services.AddSingleton(); + builder.Services.AddSingleton>(auditLogger); + builder.Services.AddSingleton(); + builder.Services.AddSingleton(); + + WebApplication app = builder.Build(); + app.UseMiddleware(); + app.UseExceptionHandler(); + app.UseMiddleware(); + app.MapOperatorEndpoints(); + app.MapRendezvousHealthEndpoints(); + app.MapGet("/test/public", static () => Results.Ok()).WithName("TestPublic"); + app.MapPost( + "/test/exception", + static IResult () => throw new InvalidOperationException("exception-secret-canary")) + .WithName("TestSecretException"); + await app.StartAsync(); + IServer server = app.Services.GetRequiredService(); + string address = Assert.Single(server.Features.Get()!.Addresses); + return new OperatorTestHost( + app, + new HttpClient { BaseAddress = new Uri(address) }, + clock, + store, + app.Services.GetRequiredService(), + auditLogger, + allLogs, + publisherCredential, + readOnlyCredential, + fullCredential); + } + + internal SessionListingId CreateListing(string owner) + { + StoreResult result = CreateListingResult(owner, out SessionListingId listingId); + Assert.True(result.Succeeded); + return listingId; + } + + internal StoreResult CreateListingResult( + string owner, + out SessionListingId listingId) + { + int sequence = Interlocked.Increment(ref _listingSequence); + listingId = new(Guid.NewGuid()); + StoreResult result = Store.CreateListing(new( + $"operator-listing-{sequence}", + $"operator-request-{sequence}", + new ListingDefinition + { + ListingId = listingId, + LeaseId = new(Guid.NewGuid()), + Scope = new(new GameId("space-game"), new EnvironmentId("production")), + OwnerSubject = owner, + RegionId = new("eu-central"), + ProtocolVersion = 7, + BuildVersion = "1.0.0", + DisplayName = "Operator test listing", + Visibility = ListingVisibility.Public, + TrustMode = PublisherTrustMode.ManagedDedicated, + CurrentPlayers = 1, + MaximumPlayers = 4, + Metadata = new Dictionary { ["mode"] = "online-coop" }, + LeaseFingerprint = new("lease-fingerprint"), + HostPresenceHandle = new(Guid.NewGuid()), + HostPresenceFingerprint = new("presence-fingerprint"), + CapabilityDerivationSalt = new string('A', 43), + })); + return result; + } + + internal async Task StartUdpAsync() + { + await _application.Services.GetRequiredService() + .StartAsync(CancellationToken.None); + _udpStarted = true; + } + + public async ValueTask DisposeAsync() + { + Client.Dispose(); + if (_udpStarted) + { + await _application.Services.GetRequiredService() + .StopAsync(CancellationToken.None); + } + await _application.StopAsync(); + await _application.DisposeAsync(); + } + } +} diff --git a/tests/FinalFactory.Rendezvous.Tests/Provisioning/PrincipalCredentialTests.cs b/tests/FinalFactory.Rendezvous.Tests/Provisioning/PrincipalCredentialTests.cs index 376f229..ea62fa4 100644 --- a/tests/FinalFactory.Rendezvous.Tests/Provisioning/PrincipalCredentialTests.cs +++ b/tests/FinalFactory.Rendezvous.Tests/Provisioning/PrincipalCredentialTests.cs @@ -94,6 +94,7 @@ public sealed class PrincipalCredentialTests CredentialValidationError.SignatureInvalid, service.Validate(tampered, ProvisioningTestData.Now).Error); Assert.True(keys.Revoke("key-1")); + Assert.True(keys.Revoke("key-1")); Assert.Equal( CredentialValidationError.KeyRevoked, service.Validate(token, ProvisioningTestData.Now).Error); diff --git a/tests/FinalFactory.Rendezvous.Tests/Server/Abuse/AbuseProtectionTests.cs b/tests/FinalFactory.Rendezvous.Tests/Server/Abuse/AbuseProtectionTests.cs index a0c8ae9..a6de688 100644 --- a/tests/FinalFactory.Rendezvous.Tests/Server/Abuse/AbuseProtectionTests.cs +++ b/tests/FinalFactory.Rendezvous.Tests/Server/Abuse/AbuseProtectionTests.cs @@ -6,6 +6,7 @@ using FinalFactory.Rendezvous.Server.Http; using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.HttpOverrides; +using Microsoft.AspNetCore.Routing; using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Options; @@ -86,6 +87,9 @@ public sealed class AbuseProtectionTests AbuseProtectionService protection = new(Options.Create(options)); IPAddress source = IPAddress.Parse("198.51.100.10"); + Assert.True(protection.IsOperatorSourceAllowed(source)); + Assert.True(protection.IsOperatorSourceAllowed(IPAddress.Parse("::ffff:198.51.100.10"))); + Assert.False(protection.IsOperatorSourceAllowed(IPAddress.Parse("198.51.100.11"))); AssertAccepted(protection, source, "BrowseSessions"); AssertAccepted(protection, source, "BrowseSessions"); AssertRejected(protection, source, "BrowseSessions"); @@ -93,6 +97,57 @@ public sealed class AbuseProtectionTests AssertRejected(protection, source, "RenewSessionLease"); } + [Fact] + public void PublicSaturationCannotConsumeTheOperatorPartition() + { + AbuseProtectionOptions options = PermissiveOptions(); + options.HttpGlobalRequestsPerWindow = 1; + options.HttpOptionalRequestsPerWindow = 1; + options.OperatorGlobalRequestsPerWindow = 1; + AbuseProtectionService protection = new(Options.Create(options)); + IPAddress source = IPAddress.Parse("198.51.100.10"); + + AssertAccepted(protection, source, "BrowseSessions"); + AssertRejected(protection, source, "BrowseSessions"); + Assert.True(protection.TryAcquireOperatorIngress(source, out var lease, out _)); + lease!.Dispose(); + Assert.False(protection.TryAcquireOperatorIngress(source, out _, out _)); + } + + [Fact] + public async Task DeniedOperatorSourcesConsumeTheBoundedPublicPartition() + { + AbuseProtectionOptions options = PermissiveOptions(); + options.OperatorAllowedAddresses = ["192.0.2.10"]; + options.HttpGlobalRequestsPerWindow = 1; + options.HttpOptionalRequestsPerWindow = 1; + options.HttpIpPrefixRequestsPerWindow = 1; + options.HttpOptionalIpPrefixRequestsPerWindow = 1; + AbuseProtectionService protection = new(Options.Create(options)); + bool dispatched = false; + HttpAbuseProtectionMiddleware middleware = new( + _ => + { + dispatched = true; + return Task.CompletedTask; + }, + protection); + + DefaultHttpContext first = Context("198.51.100.10"); + first.SetEndpoint(new Endpoint( + _ => Task.CompletedTask, + new EndpointMetadataCollection(new EndpointNameMetadata("GetOperatorStatus")), + "operator-status")); + await middleware.InvokeAsync(first); + Assert.Equal(StatusCodes.Status404NotFound, first.Response.StatusCode); + + DefaultHttpContext repeated = Context("198.51.100.10"); + repeated.SetEndpoint(first.GetEndpoint()); + await middleware.InvokeAsync(repeated); + Assert.Equal(StatusCodes.Status429TooManyRequests, repeated.Response.StatusCode); + Assert.False(dispatched); + } + [Fact] public void ResourceBudgetsRemainIsolatedAcrossTenantAndPrincipalScopes() { @@ -354,7 +409,8 @@ public sealed class AbuseProtectionTests { const string canary = "credential-canary