diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index 538de03..316ef74 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -36,6 +36,9 @@ jobs: - name: Test run: dotnet test Rendezvous.slnx --configuration Release --no-build + - name: Run quick capacity and resilience gate + run: ./scripts/run-capacity-gate.sh + - name: Test privileged Linux namespace topology when available shell: bash run: | diff --git a/.gitignore b/.gitignore index 85020f5..cf35970 100644 --- a/.gitignore +++ b/.gitignore @@ -7,5 +7,6 @@ TestResults/ *.user *.userosscache deploy/compose/.smoke.env +artifacts/ deploy/compose/secrets/* !deploy/compose/secrets/.gitignore diff --git a/README.md b/README.md index 2be281d..a0363f4 100644 --- a/README.md +++ b/README.md @@ -83,9 +83,9 @@ Rendezvous is under active roadmap development. The versioned contracts, directory leases, authenticated join attempts, LiteNetLib mediator, caller-owned SDK coordination, typed connection outcomes, thin public-SDK diagnostic client, deterministic NAT topology harness, hostile-input controls, -observability/operator surface, and secure single-active Linux deployment are -implemented. Capacity, resilience, packaging, and final production-readiness -gates remain in progress; +observability/operator surface, secure single-active Linux deployment, and +numeric capacity/resilience gates are implemented. Packaging, consumer pilots, +and final production-readiness gates remain in progress; participating games must not treat the current repository as a finished production service until those gates land. @@ -103,6 +103,9 @@ operator controls are defined in the The pinned non-root container, production topology, graceful drain, Linux hardening, smoke procedure, and recovery lifecycle are documented in [secure single-active Linux deployment](docs/deployment/linux.md). +The numeric core-state candidate profile, public launch objectives, accelerated +soak, resilience matrix, and single-active scaling decision are recorded in +[capacity and resilience gates](docs/operations/capacity-and-resilience.md). The scriptable host/browser/join diagnostic and its stable automation contract are documented in the [TestClient integration guide](docs/integration/test-client.md). The always-on three-party scenarios, optional Linux namespace topology, and diff --git a/Rendezvous.slnx b/Rendezvous.slnx index b375c75..1699ad7 100644 --- a/Rendezvous.slnx +++ b/Rendezvous.slnx @@ -6,6 +6,7 @@ + diff --git a/deploy/systemd/rendezvous.service b/deploy/systemd/rendezvous.service index 92a5574..6b220eb 100644 --- a/deploy/systemd/rendezvous.service +++ b/deploy/systemd/rendezvous.service @@ -40,7 +40,8 @@ LockPersonality=true SystemCallArchitectures=native UMask=0077 LimitNOFILE=4096 -MemoryMax=512M +CPUQuota=200% +MemoryMax=2G TasksMax=128 [Install] diff --git a/docs/architecture/0003-state-privacy-availability-and-budgets.md b/docs/architecture/0003-state-privacy-availability-and-budgets.md index 3aad737..559f752 100644 --- a/docs/architecture/0003-state-privacy-availability-and-budgets.md +++ b/docs/architecture/0003-state-privacy-availability-and-budgets.md @@ -129,9 +129,18 @@ until the owner records: - which games may enable anonymous unlisted player hosting; - deployment regions, data-processing jurisdiction, and approval of the stated 30-day audit/13-month aggregate retention periods; -- the per-game dedicated fallback endpoint policy; -- the measured supported profile and whether the 99.5% single-active objective - is sufficient or shared-state/high-availability work must be brought forward. +- the per-game dedicated fallback endpoint policy. + +Issue #18 measured and ratified the original 2-vCPU/2-GiB, 25,000-listing, +10,000-attempt core-state candidate profile and retained the 99.5% single-active +topology. It does not claim that core measurements prove public HTTP/UDP SLOs. +The versioned evidence, RTO, failure domains, and explicit signals that trigger +shared-state/high-availability work are recorded in the +[capacity and resilience gate](../operations/capacity-and-resilience.md). The +real-network canary in #23 must confirm that the proposed regional launch load +fits this profile and validate the public SLOs; it may lower the launch cap but +may not silently enable a +second active instance. These are configuration and launch decisions, not permission to weaken the tenant, replay, endpoint-verification, or secret-handling controls. diff --git a/docs/deployment/linux.md b/docs/deployment/linux.md index a18314e..27fe280 100644 --- a/docs/deployment/linux.md +++ b/docs/deployment/linux.md @@ -118,8 +118,11 @@ Keep the host clock synchronized with authenticated NTP. Credential and key windows use wall time; lease, timeout, drain, and rate-limit deadlines use a monotonic clock. Alert on clock synchronization loss before rotating keys. -Start with the Compose limits (one CPU, 512 MiB, 128 PIDs, 4096 descriptors), -measure real traffic, then change the limits and the server budgets together. +The checked-in Compose limits (one CPU and 512 MiB) are for its isolated smoke +profile, not a production capacity claim. The measured core-state candidate +uses 2 vCPU and 2 GiB with the same 128-PID/4096-descriptor ceilings; see +the [capacity and resilience gate](../operations/capacity-and-resilience.md). +Measure real traffic, then change resource limits and server budgets together. Memory pressure or CPU throttling must not extend orchestrator termination past `DrainDeadlineSeconds` plus five seconds. @@ -157,8 +160,8 @@ sudo systemctl enable --now rendezvous.service Create the dedicated `rendezvous` user without a login shell. Keep `/opt/rendezvous` and `/etc/rendezvous` root-owned and non-writable by that user; install each required key with `root:rendezvous` ownership and mode `0440`. The -unit applies the same resource, filesystem, privilege, network-family, and -shutdown constraints as Compose. +unit applies the measured 2-vCPU/2-GiB core-state candidate profile plus the +same filesystem, privilege, network-family, and shutdown hardening as Compose. ## HTTP and UDP smoke diff --git a/docs/operations/capacity-and-resilience.md b/docs/operations/capacity-and-resilience.md new file mode 100644 index 0000000..e19eb0c --- /dev/null +++ b/docs/operations/capacity-and-resilience.md @@ -0,0 +1,189 @@ +# Capacity, resilience, and availability gate + +Tracking: #18 + +This gate turns the v1 budgets in ADR 0003 into a repeatable release decision. +It does not turn Rendezvous into a horizontally scalable service: v1 remains one +active process with bounded in-memory state. A second process may be a cold +standby, but it must not accept traffic until the first process has stopped and +released the public HTTP and UDP endpoints. + +## Launch envelope and approved core-state profile + +The approved core-state profile is one Linux process limited to 2 vCPU and +2 GiB RAM. Public HTTP/UDP numbers are launch objectives that require the #23 +real-network canary before they become a supported service claim: + +| Dimension | Value | Evidence status | +| --- | --- | --- | +| Visible listings | 25,000 | Enforced and measured here | +| Active join attempts | 10,000 | Enforced and measured here | +| Core control path | 200 operations/second; p95 at most 200 ms | Measured here | +| Core mediation path | 2,000 pairings/second; p95 at most 100 ms | Measured here | +| Sustained HTTP demand | 200 requests/second | #23 launch objective; not yet a supported claim | +| Sustained UDP demand | 2,000 datagrams/second | #23 launch objective; not yet a supported claim | +| Public HTTP/UDP latency | p95 at most 200 ms / 100 ms | #23 launch objective; not yet a supported claim | +| Capacity-phase average CPU / peak working memory | below 70% / below 1.5 GiB | Measured for the core candidate | +| Valid in-profile monthly availability | 99.5%, excluding announced maintenance | Operational objective | +| Process-ready RTO / host-visible recovery | 15 seconds / 90 seconds | 15 seconds automated; 90-second deployment drill required | + +The proposed public-network mix is 20% registration/update, 30% lease-critical +renew/delete, 30% browse, and 20% join authorization for HTTP. The UDP mix is +60% authenticated host-presence refresh, 30% attempt contributions, and 10% +invalid or duplicate traffic that must be dropped early. A deployment may use a +lower per-game profile, but must not claim a higher one without new versioned +evidence. + +The capacity harness fills the complete state ceilings, then measures +registration plus presence, renewal, a 100-item compatible browse, join +issuance, simultaneous two-peer pairing, principal revocation, and telemetry. +It applies 200/100 ms guardrails and minimum 200 control / 2,000 mediation +operations per second to the core hot path. Those measurements deliberately +exclude Kestrel, LiteNetLib, TLS, JSON, socket scheduling, and the documented +mixed traffic shape. The #23 real-network canary must exercise those layers, +rate-shape the mix, record errors and shedding, and meet the public objectives +before launch; a core result is not a public-network latency or throughput claim. + +## Reproduce the evidence + +Every push runs the quick profile and the selected fault matrix: + +```bash +./scripts/run-capacity-gate.sh +``` + +Run the production candidate on an otherwise idle Linux host and restrict the +runtime to two logical CPUs. The default candidate includes a five-minute, +high-intensity expiry soak; use 3,600 seconds for a release-candidate endurance +run: + +```bash +export RENDEZVOUS_CAPACITY_PROFILE=candidate +export RENDEZVOUS_CAPACITY_CPUSET=0,1 +export RENDEZVOUS_CAPACITY_OUTPUT="$PWD/artifacts/capacity/candidate.json" +./scripts/run-capacity-gate.sh + +# Release-candidate endurance override: +dotnet run --project tests/FinalFactory.Rendezvous.Capacity \ + --configuration Release --no-build -- \ + --profile candidate --soak-seconds 3600 \ + --output artifacts/capacity/candidate-endurance.json +``` + +The machine must have at least 2 GiB available to the process. For formal +deployment evidence, run inside the same cgroup/container shape as production. +The v2 JSON embeds the commit and tree state, command, image context, CPU model, +kernel, affinity, cgroup quota/limit, collector mode, and workload seed. Supply +`RENDEZVOUS_EVIDENCE_IMAGE_DIGEST` when running a release image. Do not compare +results collected under a debugger, +concurrent build, thermal throttling, or oversubscribed CI host. + +The checked-in baseline is +[`candidate-2cpu.json`](../evidence/capacity/v2/candidate-2cpu.json). It was +produced on .NET 10.0.9/Linux x64 with CPU affinity restricted to two logical +CPUs. It filled 25,000 listings and 10,000 attempts, peaked at about 162 MiB, +and cleared all active/retained state. The five-minute baseline supersedes any +earlier local probe when its timestamp and target duration differ. + +## Soak and bounded-state interpretation + +Each soak cycle creates a listing, repeatedly renews its lease and refreshes +presence, creates a join attempt, replay marker, and retained outcome, checks +that scheduled expiry entries remain proportional to live keys, then advances +the injected monotonic clock beyond all +deadlines, and verifies that listings, attempts, replay, idempotency, and outcome +state return to zero. The candidate also measures managed-memory and process +handle deltas after full collection. Failure is any retained state, more than +64 MiB retained managed memory, more than eight retained handles, a working set +above 1.5 GiB, an untyped capacity result, or failure to admit work after expiry. + +This accelerated soak intentionally executes far more state lifecycle/cleanup +events than wall-clock traffic would permit. It catches stale deadline-queue +entries, cache growth, replay/idempotency retention, and cleanup cost. Because +it does not open Kestrel/LiteNetLib connections, its process-handle delta is only +a harness guard and is not evidence of transport stability by itself. The +selected production-process gate adds a ten-second real HTTP/UDP transport soak, +samples child-process handles and RSS, asserts bounded growth, then verifies a +clean SIGTERM and socket release. #23 must extend that into the full rate-shaped +multi-client canary while sampling queues, managed memory, and state +cardinalities. A one-hour core override remains required before tagging a +production release. + +## Fault and recovery matrix + +`run-capacity-gate.sh` runs these deterministic production paths before the +numeric profile: + +| Fault | Required result | +| --- | --- | +| HTTP/UDP overload and tracker exhaustion | Typed HTTP `429`/`CapacityExceeded`, silent UDP drop, bounded tracker keys, recovery after the window | +| Optional traffic saturation | Lease-critical renew/update/delete capacity remains available | +| Store/dependency unavailable | Readiness fails; new authorization returns typed `ServiceUnavailable`; liveness remains independent | +| Graceful drain/SIGTERM | New work returns `Draining`; existing pairing may finish; process exits 0 and releases TCP/UDP before the deadline | +| Hard restart | In-flight state is lost; SDK reports typed `ServiceUnavailable`; a host re-registers, rebinds presence, and becomes the only browser-visible replacement | +| UDP listener bind/restart | Readiness stays false without the required listener; rebinding the advertised port restores native LiteNetLib pairing | +| Wall-clock jump/skew | Monotonic lease/attempt authority is neither shortened nor extended; credential skew remains capped at 30 seconds | +| Signing-secret rotation | New key signs, overlap verifies, retired/revoked key rejects, missing material fails startup | +| Principal revocation | Listing, presence, attempts, and outcome paths are removed atomically within the latency budget | + +No external database exists in v1, so “dependency/store failure” means the +process-local atomic store is marked unavailable or a required listener/key is +unready. The service fails closed rather than pretending a degraded writable +mode exists. + +## Bandwidth and amplification + +- Accepted application datagrams are at most 1,200 bytes. +- Malformed, oversized, unauthenticated, stale, replayed, wrong-role, and + rate-limited traffic receives zero response bytes. +- A completing authenticated contribution produces at most one introduction to + each observed peer, and the combined response is at most 2.0 times that + contribution's bytes. +- The frozen-envelope and native LiteNetLib socket tests measure this on the real + UDP listener; the hostile corpus and allocation gate exercise 10,000+ inputs + without input-sized logs, tasks, or queues. + +Bandwidth planning must therefore reserve ingress for the configured 2,000 +datagrams/second plus edge overhead and egress for a worst-case verified 2.0 +amplification. Actual successful pairs normally use two contributions and two +introductions; normal gameplay leaves Rendezvous entirely. + +## Availability decision + +Single-active remains the v1 topology. The measured core profile proves bounded +state and substantial core-path headroom, while public launch capacity remains +conditional on #23. The service has a bounded stop-before-start restart path. +Its failure domain is deliberately +one process/node/public UDP endpoint: node, kernel, host network, DNS/TLS edge, +secret configuration, or operator error can remove all readiness until the cold +replacement owns the same source-preserving endpoint. + +The 99.5% objective permits about 216 minutes of unannounced downtime in a +30-day month. Operations must target process readiness within 15 seconds and +host-visible re-registration within 90 seconds, page when no ready instance +exists, and include detection plus recovery in the monthly budget. The current +in-process test validates typed downtime, same-port HTTP restart, fresh +registration, presence rebinding, and browser visibility in under five seconds; +the production-process test separately validates graceful termination, TCP/UDP +release, replacement startup on the same endpoints, UDP readiness, and the +15-second process-ready RTO. Cold-standby activation policy and the 90-second +operator-to-host recovery objective still require a deployment drill before +release. Rollout and rollback use the +deployment runbook's drain, stop, socket-release, start, smoke sequence; never +overlap old and new active processes. + +Bring shared TTL/CAS state and deterministic mediator routing forward before +enabling two active instances if any of these occurs: + +- one node cannot sustain 150% of the measured 30-day peak while meeting SLOs; +- CPU stays above 70%, memory above 75%, attempt depth above 70%, or limiter + drops/latency remain elevated after abusive traffic is excluded; +- the availability target rises above 99.5% or planned maintenance must preserve + listings; or +- one region requires multiple simultaneously active mediator endpoints. + +Rendezvous makes no multi-instance claim today, so a two-node atomic-pairing +test is intentionally not applicable. It becomes a hard release gate with the +shared-state/routing implementation; until then `SingleActiveInstance=false` +fails production startup. Multi-region and relay remain separate evidence-driven +decisions. diff --git a/docs/operations/observability-and-operator-runbook.md b/docs/operations/observability-and-operator-runbook.md index b8300c5..c77ac14 100644 --- a/docs/operations/observability-and-operator-runbook.md +++ b/docs/operations/observability-and-operator-runbook.md @@ -115,7 +115,9 @@ permission denial is audited with actor and target fingerprints. Key revocation is process-local in the current single-instance store. Apply the same revocation to every instance, then replace configuration before restarting; a restart reconstructs the configured key ring. Principal revocation is bounded -to ten minutes and removes that principal's active listings and attempts. Use +to ten minutes and removes that principal's active listings and attempts. A +repeat action may extend an active revocation but never shortens it; wait for its +original deadline rather than treating a shorter repeat as an un-revoke. Use listing revocation for one targeted session and drain before planned shutdown. ## Audit retention and incident handling diff --git a/scripts/run-capacity-gate.sh b/scripts/run-capacity-gate.sh new file mode 100755 index 0000000..448638c --- /dev/null +++ b/scripts/run-capacity-gate.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +PROFILE="${RENDEZVOUS_CAPACITY_PROFILE:-quick}" +OUTPUT="${RENDEZVOUS_CAPACITY_OUTPUT:-$ROOT/artifacts/capacity/rendezvous-capacity-v2.json}" +CPUSET="${RENDEZVOUS_CAPACITY_CPUSET:-}" +PROJECT="$ROOT/tests/FinalFactory.Rendezvous.Capacity/FinalFactory.Rendezvous.Capacity.csproj" +TESTS="$ROOT/tests/FinalFactory.Rendezvous.Tests/FinalFactory.Rendezvous.Tests.csproj" + +if [[ "$PROFILE" != quick && "$PROFILE" != candidate ]]; then + printf 'RENDEZVOUS_CAPACITY_PROFILE must be quick or candidate.\n' >&2 + exit 2 +fi +command -v dotnet >/dev/null || { + printf 'Missing required command: dotnet\n' >&2 + exit 2 +} +if [[ -n "$CPUSET" ]]; then + command -v taskset >/dev/null || { + printf 'taskset is required when RENDEZVOUS_CAPACITY_CPUSET is set.\n' >&2 + exit 2 + } +fi + +cd "$ROOT" +dotnet restore "$ROOT/Rendezvous.slnx" --locked-mode +dotnet build "$ROOT/Rendezvous.slnx" --configuration Release --no-restore + +filter='FullyQualifiedName~TrackerCapacityFailsClosedWithoutGrowingAndAWindowResetRecovers|FullyQualifiedName~OptionalTrafficCannotConsumeTheLeaseOperationReserve|FullyQualifiedName~ConcurrentAbusiveBurstStaysBoundedAndCannotBlockCriticalHttp|FullyQualifiedName~HttpOverloadIsTypedAndOversizedBodiesAreRejectedBeforeDispatch|FullyQualifiedName~WallClockMovementDoesNotExpireOrExtendLease|FullyQualifiedName~RepeatedMutableDeadlineRefreshesKeepOneScheduledEntryPerKey|FullyQualifiedName~RepeatedPrincipalRevocationCanExtendButCannotShortenProtection|FullyQualifiedName~RestartHasNewGenerationAndNoEphemeralState|FullyQualifiedName~RestartReturnsTypedUnavailabilityThenAllowsHostReregistration|FullyQualifiedName~DrainRejectsNewWorkAllowsInflightCompletionThenClearsState|FullyQualifiedName~UnavailableStoreFailsNewAuthorizationClosedAndErasesActiveState|FullyQualifiedName~KeyRotationHonorsOverlapAndRejectsRetiredKeys|FullyQualifiedName~OperatorSurfaceSeparatesAuthenticationConfirmsActionsAndRedactsInspection|FullyQualifiedName~SigtermDrainsThenReleasesHttpAndUdpSockets|FullyQualifiedName~ProductionTransportSoakKeepsHandlesMemoryAndSocketsBounded|FullyQualifiedName~NativeLiteNetLibRequestsIntroduceTheAuthorizedPair' +dotnet test "$TESTS" --configuration Release --no-build --filter "$filter" \ + --logger 'console;verbosity=minimal' + +mkdir -p "$(dirname "$OUTPUT")" +arguments=( + dotnet run --project "$PROJECT" --configuration Release --no-build -- + --profile "$PROFILE" --output "$OUTPUT" +) +export RENDEZVOUS_EVIDENCE_COMMIT="${RENDEZVOUS_EVIDENCE_COMMIT:-$(git rev-parse HEAD)}" +if [[ -z "${RENDEZVOUS_EVIDENCE_TREE_STATE:-}" ]]; then + if [[ -n "$(git status --porcelain)" ]]; then + export RENDEZVOUS_EVIDENCE_TREE_STATE=dirty + else + export RENDEZVOUS_EVIDENCE_TREE_STATE=clean + fi +fi +export RENDEZVOUS_EVIDENCE_CPUSET="${CPUSET:-unrestricted}" +export RENDEZVOUS_EVIDENCE_COMMAND="RENDEZVOUS_CAPACITY_PROFILE=$PROFILE RENDEZVOUS_CAPACITY_CPUSET=${CPUSET:-unrestricted} ./scripts/run-capacity-gate.sh" +if [[ -n "$CPUSET" ]]; then + taskset -c "$CPUSET" "${arguments[@]}" +else + "${arguments[@]}" +fi + +printf 'Capacity and resilience gate passed; evidence: %s\n' "$OUTPUT" diff --git a/src/FinalFactory.Rendezvous.Server/Properties/AssemblyInfo.cs b/src/FinalFactory.Rendezvous.Server/Properties/AssemblyInfo.cs index 024def4..220f8a4 100644 --- a/src/FinalFactory.Rendezvous.Server/Properties/AssemblyInfo.cs +++ b/src/FinalFactory.Rendezvous.Server/Properties/AssemblyInfo.cs @@ -1,3 +1,4 @@ using System.Runtime.CompilerServices; [assembly: InternalsVisibleTo("FinalFactory.Rendezvous.Tests")] +[assembly: InternalsVisibleTo("FinalFactory.Rendezvous.Capacity")] diff --git a/src/FinalFactory.Rendezvous.Server/State/InMemoryEphemeralRendezvousStore.cs b/src/FinalFactory.Rendezvous.Server/State/InMemoryEphemeralRendezvousStore.cs index 9e7fcff..f046a5b 100644 --- a/src/FinalFactory.Rendezvous.Server/State/InMemoryEphemeralRendezvousStore.cs +++ b/src/FinalFactory.Rendezvous.Server/State/InMemoryEphemeralRendezvousStore.cs @@ -11,16 +11,29 @@ internal sealed class InMemoryEphemeralRendezvousStore : IEphemeralRendezvousSto private readonly DateTimeOffset _wallOrigin; private readonly TimeSpan _monotonicOrigin; private readonly Dictionary _listings = []; + private readonly Dictionary _listingCountsByOwner = new(StringComparer.Ordinal); + private readonly PriorityQueue, long> _listingExpiries = new(); + private readonly HashSet _scheduledListingExpiries = []; private readonly Dictionary _leases = []; private readonly Dictionary _presenceHandles = []; private readonly Dictionary _presence = []; + private readonly PriorityQueue, long> _presenceExpiries = new(); + private readonly HashSet _scheduledPresenceExpiries = []; private readonly Dictionary _attempts = []; - private readonly PriorityQueue _attemptExpiries = new(); + private readonly Dictionary _attemptCountsByScope = []; + private readonly Dictionary> _attemptsByListing = []; + private readonly PriorityQueue, long> _attemptExpiries = new(); private readonly Dictionary _outcomeReports = []; + private readonly Dictionary> _outcomesByListing = []; + private readonly PriorityQueue, long> _outcomeExpiries = new(); private readonly Dictionary _attemptHandles = []; private readonly Dictionary _idempotency = new(StringComparer.Ordinal); + private readonly PriorityQueue, long> _idempotencyExpiries = new(); private readonly Dictionary _replay = new(StringComparer.Ordinal); + private readonly PriorityQueue, long> _replayExpiries = new(); private readonly Dictionary _revocations = new(StringComparer.Ordinal); + private readonly PriorityQueue, long> _revocationExpiries = new(); + private readonly HashSet _scheduledRevocationExpiries = new(StringComparer.Ordinal); private TimeSpan? _drainDeadline; private TimeSpan _nextUdpMaintenance; private long _maintenanceSweepCount; @@ -47,6 +60,22 @@ internal sealed class InMemoryEphemeralRendezvousStore : IEphemeralRendezvousSto public Guid InstanceId { get; } internal long MaintenanceSweepCount => Interlocked.Read(ref _maintenanceSweepCount); + internal int ScheduledExpiryEntryCount + { + get + { + lock (_gate) + { + return _listingExpiries.Count + + _presenceExpiries.Count + + _attemptExpiries.Count + + _outcomeExpiries.Count + + _idempotencyExpiries.Count + + _replayExpiries.Count + + _revocationExpiries.Count; + } + } + } public bool IsAvailable { @@ -161,10 +190,8 @@ internal sealed class InMemoryEphemeralRendezvousStore : IEphemeralRendezvousSto if (_listings.Count >= _options.MaxListings || _idempotency.Count >= _options.MaxIdempotencyEntries - || _listings.Values.Count(entry => string.Equals( - entry.Definition.OwnerSubject, - command.Listing.OwnerSubject, - StringComparison.Ordinal)) >= command.OwnerListingLimit) + || _listingCountsByOwner.GetValueOrDefault(command.Listing.OwnerSubject) + >= command.OwnerListingLimit) { return new(StoreResultCode.CapacityExceeded); } @@ -183,12 +210,21 @@ internal sealed class InMemoryEphemeralRendezvousStore : IEphemeralRendezvousSto WallDeadline(now, _options.LeaseLifetime), version: 1); _listings.Add(frozen.ListingId, entry); + ScheduleMutableDeadline( + _listingExpiries, + _scheduledListingExpiries, + frozen.ListingId, + entry.LeaseDeadline); + _listingCountsByOwner[frozen.OwnerSubject] = + _listingCountsByOwner.GetValueOrDefault(frozen.OwnerSubject) + 1; _leases.Add(frozen.LeaseId, frozen.ListingId); _presenceHandles.Add(frozen.HostPresenceHandle, frozen.ListingId); - _idempotency.Add(idempotencyKey, new( + IdempotencyEntry idempotency = new( command.RequestFingerprint, frozen.ListingId, - now + _options.IdempotencyLifetime)); + now + _options.IdempotencyLifetime); + _idempotency.Add(idempotencyKey, idempotency); + EnqueueDeadline(_idempotencyExpiries, idempotencyKey, idempotency.Deadline); return new(StoreResultCode.Success, Snapshot(entry)); }, cancellationToken); @@ -346,10 +382,20 @@ internal sealed class InMemoryEphemeralRendezvousStore : IEphemeralRendezvousSto return new(StoreResultCode.CapacityExceeded); } - _presence[command.Handle] = new( + bool isNewPresence = !_presence.ContainsKey(command.Handle); + PresenceEntry presence = new( command.PublicEndpoint, command.LocalEndpoint, now + _options.PresenceLifetime); + _presence[command.Handle] = presence; + if (isNewPresence) + { + ScheduleMutableDeadline( + _presenceExpiries, + _scheduledPresenceExpiries, + command.Handle, + presence.Deadline); + } return new(StoreResultCode.Success, Snapshot(entry)); }, cancellationToken, eagerCleanup: false); @@ -444,8 +490,7 @@ internal sealed class InMemoryEphemeralRendezvousStore : IEphemeralRendezvousSto if (_attempts.Count >= _options.MaxJoinAttempts || _outcomeReports.Count >= _options.MaxOutcomeReports || _idempotency.Count >= _options.MaxIdempotencyEntries - || _attempts.Values.Count(entry => entry.Command.Scope == command.Scope) - >= command.ScopeAttemptLimit) + || _attemptCountsByScope.GetValueOrDefault(command.Scope) >= command.ScopeAttemptLimit) { return new(StoreResultCode.CapacityExceeded); } @@ -461,19 +506,25 @@ internal sealed class InMemoryEphemeralRendezvousStore : IEphemeralRendezvousSto now + _options.JoinAttemptLifetime, WallDeadline(now, _options.JoinAttemptLifetime)); _attempts.Add(command.AttemptId, attempt); - _attemptExpiries.Enqueue( - new AttemptExpiry(command.AttemptId, attempt.Deadline), - attempt.Deadline.Ticks); - _outcomeReports.Add(command.AttemptId, new( + AddToIndex(_attemptsByListing, command.ListingId, command.AttemptId); + _attemptCountsByScope[command.Scope] = + _attemptCountsByScope.GetValueOrDefault(command.Scope) + 1; + EnqueueDeadline(_attemptExpiries, command.AttemptId, attempt.Deadline); + OutcomeReportEntry outcome = new( command.ListingId, command.ClientSubject, command.ClientCapabilityFingerprint, - now + _options.JoinAttemptLifetime + _options.IdempotencyLifetime)); + now + _options.JoinAttemptLifetime + _options.IdempotencyLifetime); + _outcomeReports.Add(command.AttemptId, outcome); + AddToIndex(_outcomesByListing, command.ListingId, command.AttemptId); + EnqueueDeadline(_outcomeExpiries, command.AttemptId, outcome.Deadline); _attemptHandles.Add(command.MediationHandle, command.AttemptId); - _idempotency.Add(idempotencyKey, new( + IdempotencyEntry idempotency = new( command.RequestFingerprint, command.AttemptId, - now + _options.IdempotencyLifetime)); + now + _options.IdempotencyLifetime); + _idempotency.Add(idempotencyKey, idempotency); + EnqueueDeadline(_idempotencyExpiries, idempotencyKey, idempotency.Deadline); return new(StoreResultCode.Success, Snapshot(attempt)); }, cancellationToken); @@ -757,7 +808,9 @@ internal sealed class InMemoryEphemeralRendezvousStore : IEphemeralRendezvousSto throw new ArgumentOutOfRangeException(nameof(consumption), "Replay lifetime exceeds the configured ceiling."); } - _replay.Add(key, now + lifetime); + TimeSpan deadline = now + lifetime; + _replay.Add(key, deadline); + EnqueueDeadline(_replayExpiries, key, deadline); return new(StoreResultCode.Success, true); }, cancellationToken); @@ -794,7 +847,24 @@ internal sealed class InMemoryEphemeralRendezvousStore : IEphemeralRendezvousSto + _presence.Count + _attempts.Count + _outcomeReports.Count; - _revocations[subject] = now + lifetime; + TimeSpan deadline = now + lifetime; + bool isNewRevocation = !_revocations.TryGetValue(subject, out TimeSpan existingDeadline); + if (!isNewRevocation && existingDeadline > deadline) + { + // A repeated operator action may extend protection but cannot silently + // shorten an already-authoritative security revocation. + deadline = existingDeadline; + } + + _revocations[subject] = deadline; + if (isNewRevocation) + { + ScheduleMutableDeadline( + _revocationExpiries, + _scheduledRevocationExpiries, + subject, + deadline); + } SessionListingId[] listings = _listings .Where(item => string.Equals(item.Value.Definition.OwnerSubject, subject, StringComparison.Ordinal)) .Select(static item => item.Key) @@ -818,7 +888,7 @@ internal sealed class InMemoryEphemeralRendezvousStore : IEphemeralRendezvousSto } foreach (JoinAttemptId attemptId in outcomeReports) { - _outcomeReports.Remove(attemptId); + RemoveOutcome(attemptId); } int activeResourcesAfter = _listings.Count @@ -911,63 +981,42 @@ internal sealed class InMemoryEphemeralRendezvousStore : IEphemeralRendezvousSto ClearActiveState(); } - _expiryChurn += RemoveExpired(_revocations, now); - _expiryChurn += RemoveExpired(_replay, now); - string[] expiredIdempotency = _idempotency - .Where(item => item.Value.Deadline <= now) - .Select(static item => item.Key) - .ToArray(); - _expiryChurn += expiredIdempotency.Length; - foreach (string key in expiredIdempotency) - { - _idempotency.Remove(key); - } - - MediationHandle[] expiredPresence = _presence - .Where(item => item.Value.Deadline <= now) - .Select(static item => item.Key) - .ToArray(); - _expiryChurn += expiredPresence.Length; - foreach (MediationHandle handle in expiredPresence) - { - _presence.Remove(handle); - } - + _expiryChurn += RemoveExpiredMutableDeadlines( + _revocations, + _revocationExpiries, + _scheduledRevocationExpiries, + now); + _expiryChurn += RemoveExpiredDeadlines(_replay, _replayExpiries, now); + _expiryChurn += RemoveExpiredIdempotency(now); + _expiryChurn += RemoveExpiredPresence(now); _expiryChurn += RemoveExpiredAttempts(now); - - JoinAttemptId[] expiredOutcomes = _outcomeReports - .Where(item => item.Value.Deadline <= now) - .Select(static item => item.Key) - .ToArray(); - _expiryChurn += expiredOutcomes.Length; - foreach (JoinAttemptId attemptId in expiredOutcomes) - { - _outcomeReports.Remove(attemptId); - } - - SessionListingId[] expiredListings = _listings - .Where(item => item.Value.LeaseDeadline <= now) - .Select(static item => item.Key) - .ToArray(); - _expiryChurn += expiredListings.Length; - foreach (SessionListingId listingId in expiredListings) - { - RemoveListing(listingId); - } + _expiryChurn += RemoveExpiredOutcomes(now); + _expiryChurn += RemoveExpiredListings(now); } private void ClearActiveState() { _listings.Clear(); + _listingCountsByOwner.Clear(); + _listingExpiries.Clear(); + _scheduledListingExpiries.Clear(); _leases.Clear(); _presenceHandles.Clear(); _presence.Clear(); + _presenceExpiries.Clear(); + _scheduledPresenceExpiries.Clear(); _attempts.Clear(); + _attemptCountsByScope.Clear(); + _attemptsByListing.Clear(); _attemptExpiries.Clear(); _outcomeReports.Clear(); + _outcomesByListing.Clear(); + _outcomeExpiries.Clear(); _attemptHandles.Clear(); _idempotency.Clear(); + _idempotencyExpiries.Clear(); _replay.Clear(); + _replayExpiries.Clear(); } private void RemoveListing(SessionListingId listingId) @@ -978,23 +1027,23 @@ internal sealed class InMemoryEphemeralRendezvousStore : IEphemeralRendezvousSto } _leases.Remove(listing.Definition.LeaseId); + DecrementCount(_listingCountsByOwner, listing.Definition.OwnerSubject); _presenceHandles.Remove(listing.Definition.HostPresenceHandle); _presence.Remove(listing.Definition.HostPresenceHandle); - foreach (JoinAttemptId attemptId in _attempts - .Where(item => item.Value.Command.ListingId == listingId) - .Select(static item => item.Key) - .ToArray()) + if (_attemptsByListing.TryGetValue(listingId, out HashSet? attempts)) { - RemoveAttempt(attemptId); + foreach (JoinAttemptId attemptId in attempts.ToArray()) + { + RemoveAttempt(attemptId); + } } - - foreach (JoinAttemptId attemptId in _outcomeReports - .Where(item => item.Value.ListingId == listingId) - .Select(static item => item.Key) - .ToArray()) + if (_outcomesByListing.TryGetValue(listingId, out HashSet? outcomes)) { - _outcomeReports.Remove(attemptId); + foreach (JoinAttemptId attemptId in outcomes.ToArray()) + { + RemoveOutcome(attemptId); + } } } @@ -1003,20 +1052,75 @@ internal sealed class InMemoryEphemeralRendezvousStore : IEphemeralRendezvousSto if (_attempts.Remove(attemptId, out AttemptEntry? attempt)) { _attemptHandles.Remove(attempt.Command.MediationHandle); + DecrementCount(_attemptCountsByScope, attempt.Command.Scope); + RemoveFromIndex(_attemptsByListing, attempt.Command.ListingId, attemptId); + } + } + + private void RemoveOutcome(JoinAttemptId attemptId) + { + if (_outcomeReports.Remove(attemptId, out OutcomeReportEntry? outcome)) + { + RemoveFromIndex(_outcomesByListing, outcome.ListingId, attemptId); + } + } + + private static void AddToIndex( + Dictionary> index, + TKey key, + JoinAttemptId attemptId) + where TKey : notnull + { + if (!index.TryGetValue(key, out HashSet? values)) + { + values = []; + index.Add(key, values); + } + + values.Add(attemptId); + } + + private static void RemoveFromIndex( + Dictionary> index, + TKey key, + JoinAttemptId attemptId) + where TKey : notnull + { + if (index.TryGetValue(key, out HashSet? values) + && values.Remove(attemptId) + && values.Count == 0) + { + index.Remove(key); + } + } + + private static void DecrementCount(Dictionary counts, TKey key) + where TKey : notnull + { + int remaining = counts[key] - 1; + if (remaining == 0) + { + counts.Remove(key); + } + else + { + counts[key] = remaining; } } private int RemoveExpiredAttempts(TimeSpan now) { int removed = 0; - while (_attemptExpiries.TryPeek(out AttemptExpiry candidate, out long deadlineTicks) + while (_attemptExpiries.TryPeek( + out DeadlineEntry candidate, + out long deadlineTicks) && deadlineTicks <= now.Ticks) { _attemptExpiries.Dequeue(); - if (_attempts.TryGetValue(candidate.AttemptId, out AttemptEntry? current) + if (_attempts.TryGetValue(candidate.Key, out AttemptEntry? current) && current.Deadline == candidate.Deadline) { - RemoveAttempt(candidate.AttemptId); + RemoveAttempt(candidate.Key); removed++; } } @@ -1024,6 +1128,184 @@ internal sealed class InMemoryEphemeralRendezvousStore : IEphemeralRendezvousSto return removed; } + private int RemoveExpiredListings(TimeSpan now) + { + int removed = 0; + while (_listingExpiries.TryPeek( + out DeadlineEntry candidate, + out long deadlineTicks) + && deadlineTicks <= now.Ticks) + { + _listingExpiries.Dequeue(); + _scheduledListingExpiries.Remove(candidate.Key); + if (!_listings.TryGetValue(candidate.Key, out ListingEntry? current)) + { + continue; + } + + if (current.LeaseDeadline > now) + { + ScheduleMutableDeadline( + _listingExpiries, + _scheduledListingExpiries, + candidate.Key, + current.LeaseDeadline); + } + else + { + RemoveListing(candidate.Key); + removed++; + } + } + + return removed; + } + + private int RemoveExpiredPresence(TimeSpan now) + { + int removed = 0; + while (_presenceExpiries.TryPeek( + out DeadlineEntry candidate, + out long deadlineTicks) + && deadlineTicks <= now.Ticks) + { + _presenceExpiries.Dequeue(); + _scheduledPresenceExpiries.Remove(candidate.Key); + if (!_presence.TryGetValue(candidate.Key, out PresenceEntry? current)) + { + continue; + } + + if (current.Deadline > now) + { + ScheduleMutableDeadline( + _presenceExpiries, + _scheduledPresenceExpiries, + candidate.Key, + current.Deadline); + } + else if (_presence.Remove(candidate.Key)) + { + removed++; + } + } + + return removed; + } + + private int RemoveExpiredOutcomes(TimeSpan now) + { + int removed = 0; + while (_outcomeExpiries.TryPeek( + out DeadlineEntry candidate, + out long deadlineTicks) + && deadlineTicks <= now.Ticks) + { + _outcomeExpiries.Dequeue(); + if (_outcomeReports.TryGetValue(candidate.Key, out OutcomeReportEntry? current) + && current.Deadline == candidate.Deadline) + { + RemoveOutcome(candidate.Key); + removed++; + } + } + + return removed; + } + + private int RemoveExpiredIdempotency(TimeSpan now) + { + int removed = 0; + while (_idempotencyExpiries.TryPeek( + out DeadlineEntry candidate, + out long deadlineTicks) + && deadlineTicks <= now.Ticks) + { + _idempotencyExpiries.Dequeue(); + if (_idempotency.TryGetValue(candidate.Key, out IdempotencyEntry? current) + && current.Deadline == candidate.Deadline + && _idempotency.Remove(candidate.Key)) + { + removed++; + } + } + + return removed; + } + + private static int RemoveExpiredDeadlines( + Dictionary entries, + PriorityQueue, long> expiries, + TimeSpan now) + where TKey : notnull + { + int removed = 0; + while (expiries.TryPeek(out DeadlineEntry candidate, out long deadlineTicks) + && deadlineTicks <= now.Ticks) + { + expiries.Dequeue(); + if (entries.TryGetValue(candidate.Key, out TimeSpan current) + && current == candidate.Deadline + && entries.Remove(candidate.Key)) + { + removed++; + } + } + + return removed; + } + + private static int RemoveExpiredMutableDeadlines( + Dictionary entries, + PriorityQueue, long> expiries, + HashSet scheduled, + TimeSpan now) + where TKey : notnull + { + int removed = 0; + while (expiries.TryPeek(out DeadlineEntry candidate, out long deadlineTicks) + && deadlineTicks <= now.Ticks) + { + expiries.Dequeue(); + scheduled.Remove(candidate.Key); + if (!entries.TryGetValue(candidate.Key, out TimeSpan current)) + { + continue; + } + + if (current > now) + { + ScheduleMutableDeadline(expiries, scheduled, candidate.Key, current); + } + else if (entries.Remove(candidate.Key)) + { + removed++; + } + } + + return removed; + } + + private static void ScheduleMutableDeadline( + PriorityQueue, long> expiries, + HashSet scheduled, + TKey key, + TimeSpan deadline) + where TKey : notnull + { + if (scheduled.Add(key)) + { + EnqueueDeadline(expiries, key, deadline); + } + } + + private static void EnqueueDeadline( + PriorityQueue, long> expiries, + TKey key, + TimeSpan deadline) + where TKey : notnull => + expiries.Enqueue(new(key, deadline), deadline.Ticks); + private bool HandleExists(MediationHandle handle) => _presenceHandles.ContainsKey(handle) || _attemptHandles.ContainsKey(handle); @@ -1063,20 +1345,6 @@ internal sealed class InMemoryEphemeralRendezvousStore : IEphemeralRendezvousSto IsCancelled = entry.IsCancelled, }; - private static int RemoveExpired(Dictionary entries, TimeSpan now) - { - string[] expired = entries - .Where(item => item.Value <= now) - .Select(static item => item.Key) - .ToArray(); - foreach (string key in expired) - { - entries.Remove(key); - } - - return expired.Length; - } - private static void ValidateListing(ListingDefinition listing) { ArgumentNullException.ThrowIfNull(listing); @@ -1219,7 +1487,8 @@ internal sealed class InMemoryEphemeralRendezvousStore : IEphemeralRendezvousSto public bool IsCancelled { get; set; } } - private readonly record struct AttemptExpiry(JoinAttemptId AttemptId, TimeSpan Deadline); + private readonly record struct DeadlineEntry(TKey Key, TimeSpan Deadline) + where TKey : notnull; private sealed class OutcomeReportEntry( SessionListingId listingId, diff --git a/tests/FinalFactory.Rendezvous.Capacity/CapacityOptions.cs b/tests/FinalFactory.Rendezvous.Capacity/CapacityOptions.cs new file mode 100644 index 0000000..57b1d97 --- /dev/null +++ b/tests/FinalFactory.Rendezvous.Capacity/CapacityOptions.cs @@ -0,0 +1,91 @@ +namespace FinalFactory.Rendezvous.Capacity; + +internal sealed record CapacityOptions +{ + public required string Profile { get; init; } + public required int Listings { get; init; } + public required int Attempts { get; init; } + public required int Samples { get; init; } + public required int SoakCycles { get; init; } + public required int SoakSeconds { get; init; } + public string? OutputPath { get; init; } + + public static CapacityOptions Parse(string[] args) + { + Dictionary values = ParseArguments(args); + string profile = values.GetValueOrDefault("--profile") ?? "quick"; + (int listings, int attempts, int samples, int soakCycles, int soakSeconds) = profile switch + { + "quick" => (1_000, 500, 100, 20, 0), + "candidate" => (25_000, 10_000, 1_000, 1_000, 300), + _ => throw new ArgumentException("--profile must be 'quick' or 'candidate'."), + }; + + return new() + { + Profile = profile, + Listings = ParsePositive(values.GetValueOrDefault("--listings"), listings, "--listings"), + Attempts = ParsePositive(values.GetValueOrDefault("--attempts"), attempts, "--attempts"), + Samples = ParsePositive(values.GetValueOrDefault("--samples"), samples, "--samples"), + SoakCycles = ParsePositive( + values.GetValueOrDefault("--soak-cycles"), + soakCycles, + "--soak-cycles"), + SoakSeconds = ParseNonNegative( + values.GetValueOrDefault("--soak-seconds"), + soakSeconds, + "--soak-seconds"), + OutputPath = values.GetValueOrDefault("--output"), + }; + } + + private static Dictionary ParseArguments(string[] args) + { + HashSet allowed = + [ + "--profile", + "--listings", + "--attempts", + "--samples", + "--soak-cycles", + "--soak-seconds", + "--output", + ]; + Dictionary values = new(StringComparer.Ordinal); + for (int index = 0; index < args.Length; index += 2) + { + string option = args[index]; + if (!allowed.Contains(option)) + { + throw new ArgumentException($"Unknown option: {option}."); + } + + if (index == args.Length - 1 + || args[index + 1].StartsWith("--", StringComparison.Ordinal)) + { + throw new ArgumentException($"{option} requires a value."); + } + + if (!values.TryAdd(option, args[index + 1])) + { + throw new ArgumentException($"{option} may be supplied only once."); + } + } + + return values; + } + + private static int ParsePositive(string? value, int fallback, string option) => + value is null + ? fallback + : int.TryParse(value, out int parsed) && parsed > 0 + ? parsed + : throw new ArgumentException($"{option} must be a positive integer."); + + private static int ParseNonNegative(string? value, int fallback, string option) => + value is null + ? fallback + : int.TryParse(value, out int parsed) && parsed >= 0 + ? parsed + : throw new ArgumentException($"{option} must be a non-negative integer."); +} diff --git a/tests/FinalFactory.Rendezvous.Capacity/CapacityReport.cs b/tests/FinalFactory.Rendezvous.Capacity/CapacityReport.cs new file mode 100644 index 0000000..c49ccd5 --- /dev/null +++ b/tests/FinalFactory.Rendezvous.Capacity/CapacityReport.cs @@ -0,0 +1,76 @@ +namespace FinalFactory.Rendezvous.Capacity; + +internal sealed record CapacityReport +{ + public required int SchemaVersion { get; init; } + public required string EvidenceVersion { get; init; } + public required DateTimeOffset GeneratedAt { get; init; } + public required string Profile { get; init; } + public required RuntimeEvidence Runtime { get; init; } + public required CapacityTargets Targets { get; init; } + public required IReadOnlyList Measurements { get; init; } + public required StateEvidence State { get; init; } + public required IReadOnlyList Failures { get; init; } + public required bool Passed { get; init; } +} + +internal sealed record RuntimeEvidence( + string Framework, + string OperatingSystem, + string Kernel, + string Architecture, + string CpuModel, + int ProcessorCount, + string CpuAffinity, + string CpuQuota, + string MemoryLimit, + string GarbageCollector, + string CommitSha, + string TreeState, + string Command, + string ImageDigest, + string WorkloadSeed, + double CapacityPhaseAverageCpuPercent, + long PeakWorkingSetBytes, + long ManagedBytesAfterCleanup); + +internal sealed record CapacityTargets( + int VisibleListings, + int ActiveJoinAttempts, + int CoreControlOperationsPerSecond, + int CoreMediationOperationsPerSecond, + double CoreControlP95Milliseconds, + double CoreMediationP95Milliseconds, + double MaximumAverageCpuPercent, + long MaximumWorkingSetBytes, + int SoakCycles, + int SoakDurationSeconds); + +internal sealed record CapacityMeasurement( + string Operation, + int Samples, + double P50Milliseconds, + double P95Milliseconds, + double P99Milliseconds, + double OperationsPerSecond, + double MinimumOperationsPerSecond, + double BudgetMilliseconds, + bool Passed); + +internal sealed record StateEvidence( + int PeakListings, + int PeakAttempts, + int PeakReplayMarkers, + int FinalListings, + int FinalAttempts, + int FinalReplayMarkers, + long ExpiryChurn, + long MaintenanceSweeps, + int SoakCyclesCompleted, + double SoakDurationSeconds, + int SoakPeakScheduledExpiryEntries, + long SoakManagedGrowthBytes, + int SoakHandleGrowth, + bool RestartStartedEmpty, + bool OverloadWasTyped, + bool RecoverySucceeded); diff --git a/tests/FinalFactory.Rendezvous.Capacity/CapacityRunner.cs b/tests/FinalFactory.Rendezvous.Capacity/CapacityRunner.cs new file mode 100644 index 0000000..e9573ae --- /dev/null +++ b/tests/FinalFactory.Rendezvous.Capacity/CapacityRunner.cs @@ -0,0 +1,580 @@ +using System.Collections.Concurrent; +using System.Diagnostics; +using System.Runtime; +using System.Runtime.InteropServices; +using FinalFactory.Rendezvous.Contracts; +using FinalFactory.Rendezvous.Server.Observability; +using FinalFactory.Rendezvous.Server.State; + +namespace FinalFactory.Rendezvous.Capacity; + +internal static class CapacityRunner +{ + private static readonly TenantScope Scope = new(new("space-game"), new("production")); + private const uint ProtocolVersion = 1; + + public static Task RunAsync(CapacityOptions options) + { + ArgumentNullException.ThrowIfNull(options); + Process process = Process.GetCurrentProcess(); + TimeSpan cpuBefore = process.TotalProcessorTime; + Stopwatch capacityPhaseTime = Stopwatch.StartNew(); + List failures = []; + List measurements = []; + ManualClock clock = new(); + InMemoryEphemeralRendezvousStore store = CreateStore(options, clock); + List listings = new(options.Listings); + List attempts = new(options.Attempts); + int registrationSamples = Math.Min(options.Samples, options.Listings); + int attemptSamples = Math.Min(options.Samples, options.Attempts); + + for (int index = 0; index < options.Listings - registrationSamples; index++) + { + listings.Add(CreateVisibleListing(store, index)); + } + + measurements.Add(Measure( + "registration-and-presence", + registrationSamples, + budgetMilliseconds: 200, + minimumOperationsPerSecond: 200, + index => listings.Add(CreateVisibleListing( + store, + options.Listings - registrationSamples + index)))); + + measurements.Add(Measure( + "lease-renewal", + registrationSamples, + budgetMilliseconds: 200, + minimumOperationsPerSecond: 200, + index => + { + StoredListing listing = listings[index]; + StoreResult renewed = store.RenewLease(new( + listing.Definition.ListingId, + listing.Definition.LeaseId, + listing.Definition.LeaseFingerprint, + listing.Definition.OwnerSubject, + listing.Version)); + RequireSuccess(renewed, "renewal"); + listings[index] = renewed.Value!; + })); + + int browseSamples = Math.Min(options.Samples, 250); + measurements.Add(Measure( + "visible-session-browse", + browseSamples, + budgetMilliseconds: 200, + minimumOperationsPerSecond: 200, + _ => RequireSuccess( + store.BrowseVisibleListings(new( + Scope, + ProtocolVersion, + new RegionId("eu-central"), + ContractLimits.BrowserPageMaxItems, + ExcludeFull: true)), + "browse"))); + + for (int index = 0; index < options.Attempts - attemptSamples; index++) + { + CreateJoinAttemptCommand command = CreateAttempt(index, listings[index % listings.Count]); + RequireSuccess(store.CreateJoinAttempt(command), "join issuance"); + attempts.Add(command); + } + + measurements.Add(Measure( + "join-attempt-issuance", + attemptSamples, + budgetMilliseconds: 200, + minimumOperationsPerSecond: 200, + index => + { + int sequence = options.Attempts - attemptSamples + index; + CreateJoinAttemptCommand command = CreateAttempt( + sequence, + listings[sequence % listings.Count]); + RequireSuccess(store.CreateJoinAttempt(command), "join issuance"); + attempts.Add(command); + })); + + int punchSamples = Math.Min(options.Samples, attempts.Count); + measurements.Add(MeasureConcurrentPunch(store, attempts, punchSamples)); + EphemeralStoreSnapshot peak = store.GetSnapshot(); + + StoreResult overloaded = store.CreateJoinAttempt( + CreateAttempt(options.Attempts + 1, listings[^1])); + bool overloadWasTyped = peak.ActiveJoinAttempts == options.Attempts + && overloaded.Code == StoreResultCode.CapacityExceeded; + if (!overloadWasTyped) + { + failures.Add( + $"Expected typed CapacityExceeded at {options.Attempts} active attempts, " + + $"observed count={peak.ActiveJoinAttempts}, result={overloaded.Code}."); + } + + int revocationSamples = Math.Min(Math.Max(1, options.Samples / 20), listings.Count / 2); + measurements.Add(Measure( + "principal-revocation", + revocationSamples, + budgetMilliseconds: 200, + minimumOperationsPerSecond: 50, + index => RequireSuccess( + store.RevokePrincipal( + listings[index].Definition.OwnerSubject, + TimeSpan.FromMinutes(1)), + "principal revocation"))); + + using (RendezvousTelemetry telemetry = new(store)) + { + measurements.Add(Measure( + "telemetry-recording", + Math.Max(100, options.Samples), + budgetMilliseconds: 1, + minimumOperationsPerSecond: 10_000, + _ => + { + telemetry.RecordHttp("browse", 200, 1); + telemetry.RecordUdp("contribution", "accepted", 1); + telemetry.RecordPairingLatency(2); + })); + } + + clock.Advance(TimeSpan.FromSeconds(61)); + EphemeralStoreSnapshot? afterCoincidentExpiry = null; + measurements.Add(Measure( + "coincident-listing-attempt-expiry", + samples: 1, + budgetMilliseconds: 200, + minimumOperationsPerSecond: 0, + _ => afterCoincidentExpiry = store.GetSnapshot())); + if (afterCoincidentExpiry!.ActiveJoinAttempts != 0 + || afterCoincidentExpiry.ActiveListings != 0) + { + failures.Add("Coincident 60-second cleanup retained expired listings or join attempts."); + } + + clock.Advance(TimeSpan.FromSeconds(90)); + _ = store.GetSnapshot(); + StoredListing recoveryListing = CreateVisibleListing(store, options.Listings + 1); + StoreResult recovered = store.CreateJoinAttempt( + CreateAttempt(options.Attempts + 2, recoveryListing)); + bool recoverySucceeded = recovered.Succeeded; + if (!recoverySucceeded) + { + failures.Add($"Store did not recover after attempt expiry: {recovered.Code}."); + } + + clock.Advance(TimeSpan.FromSeconds(151)); + EphemeralStoreSnapshot final = store.GetSnapshot(); + if (final.ActiveListings != 0 + || final.ActiveJoinAttempts != 0 + || final.ReplayMarkers != 0 + || final.IdempotencyEntries != 0 + || final.RetainedOutcomeReports != 0) + { + failures.Add("Expiry cleanup left active or retained state after every configured deadline."); + } + + capacityPhaseTime.Stop(); + process.Refresh(); + double capacityPhaseCpuPercent = 100 + * (process.TotalProcessorTime - cpuBefore).TotalSeconds + / Math.Max(capacityPhaseTime.Elapsed.TotalSeconds * Environment.ProcessorCount, 0.000_001); + if (options.Profile == "candidate" && capacityPhaseCpuPercent > 70) + { + failures.Add( + $"Capacity-phase CPU {capacityPhaseCpuPercent:F1}% exceeded the 70% candidate budget."); + } + + SoakEvidence soak = RunAcceleratedSoak(options, failures); + ManualClock restartClock = new(); + EphemeralStoreSnapshot restarted = CreateStore(options, restartClock).GetSnapshot(); + bool restartStartedEmpty = restarted.ActiveListings == 0 + && restarted.ActiveJoinAttempts == 0 + && restarted.ReplayMarkers == 0; + if (!restartStartedEmpty) + { + failures.Add("A restarted store did not begin empty."); + } + + foreach (CapacityMeasurement measurement in measurements.Where(static item => !item.Passed)) + { + failures.Add( + $"{measurement.Operation} missed its budget: p95={measurement.P95Milliseconds:F3} ms, " + + $"rate={measurement.OperationsPerSecond:F1}/s."); + } + + GC.Collect(); + GC.WaitForPendingFinalizers(); + GC.Collect(); + process.Refresh(); + long managedAfterCleanup = GC.GetTotalMemory(forceFullCollection: true); + long memoryBudget = 1_610_612_736; + if (process.PeakWorkingSet64 > memoryBudget) + { + failures.Add( + $"Peak working set {process.PeakWorkingSet64} exceeded the 1.5 GiB profile budget."); + } + + if (options.Profile == "candidate" && Environment.ProcessorCount != 2) + { + failures.Add( + $"Candidate evidence must expose exactly two CPUs; runtime exposed " + + $"{Environment.ProcessorCount}."); + } + + CapacityReport report = new() + { + SchemaVersion = 2, + EvidenceVersion = "v2", + GeneratedAt = DateTimeOffset.UtcNow, + Profile = options.Profile, + Runtime = new( + RuntimeInformation.FrameworkDescription, + RuntimeInformation.OSDescription, + Environment.OSVersion.VersionString, + RuntimeInformation.ProcessArchitecture.ToString(), + ReadCpuModel(), + Environment.ProcessorCount, + Environment.GetEnvironmentVariable("RENDEZVOUS_EVIDENCE_CPUSET") ?? "unrestricted", + ReadCgroupValue("/sys/fs/cgroup/cpu.max"), + ReadCgroupValue("/sys/fs/cgroup/memory.max"), + GCSettings.IsServerGC ? "server" : "workstation", + Environment.GetEnvironmentVariable("RENDEZVOUS_EVIDENCE_COMMIT") ?? "unrecorded", + Environment.GetEnvironmentVariable("RENDEZVOUS_EVIDENCE_TREE_STATE") ?? "unrecorded", + Environment.GetEnvironmentVariable("RENDEZVOUS_EVIDENCE_COMMAND") ?? "unrecorded", + Environment.GetEnvironmentVariable("RENDEZVOUS_EVIDENCE_IMAGE_DIGEST") ?? "not-containerized", + "fixed-sequences-random-identifiers", + capacityPhaseCpuPercent, + process.PeakWorkingSet64, + managedAfterCleanup), + Targets = new( + options.Listings, + options.Attempts, + 200, + 2_000, + 200, + 100, + 70, + memoryBudget, + options.SoakCycles, + options.SoakSeconds), + Measurements = measurements, + State = new( + peak.ActiveListings, + peak.ActiveJoinAttempts, + peak.ReplayMarkers, + final.ActiveListings, + final.ActiveJoinAttempts, + final.ReplayMarkers, + final.ExpiryChurn, + final.MaintenanceSweeps, + soak.Cycles, + soak.Duration.TotalSeconds, + soak.PeakScheduledExpiryEntries, + soak.ManagedGrowthBytes, + soak.HandleGrowth, + restartStartedEmpty, + overloadWasTyped, + recoverySucceeded), + Failures = failures, + Passed = failures.Count == 0, + }; + return Task.FromResult(report); + } + + private static InMemoryEphemeralRendezvousStore CreateStore( + CapacityOptions options, + ManualClock clock) => new( + new EphemeralStoreOptions + { + MaxListings = options.Listings, + MaxPresenceBindings = options.Listings, + MaxJoinAttempts = options.Attempts, + MaxOutcomeReports = options.Attempts, + MaxIdempotencyEntries = options.Listings + options.Attempts + 1, + }, + clock, + clock); + + private static StoredListing CreateVisibleListing( + InMemoryEphemeralRendezvousStore store, + int sequence) + { + string owner = $"publisher-{sequence}"; + SecretFingerprint leaseFingerprint = new($"lease-{sequence}"); + SecretFingerprint presenceFingerprint = new($"presence-{sequence}"); + ListingDefinition definition = new() + { + ListingId = new(Guid.NewGuid()), + LeaseId = new(Guid.NewGuid()), + Scope = Scope, + OwnerSubject = owner, + RegionId = new("eu-central"), + ProtocolVersion = ProtocolVersion, + BuildVersion = "1.0.0", + DisplayName = $"Capacity host {sequence}", + Visibility = ListingVisibility.Public, + TrustMode = PublisherTrustMode.ManagedDedicated, + CurrentPlayers = 1, + MaximumPlayers = 8, + Metadata = new Dictionary(StringComparer.Ordinal), + LeaseFingerprint = leaseFingerprint, + HostPresenceHandle = new(Guid.NewGuid()), + HostPresenceFingerprint = presenceFingerprint, + CapabilityDerivationSalt = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", + }; + StoreResult created = store.CreateListing(new( + $"register-{sequence}", + $"register-request-{sequence}", + definition)); + RequireSuccess(created, "registration"); + StoreResult bound = store.BindHostPresence(new( + definition.HostPresenceHandle, + presenceFingerprint, + PublicEndpoint(10_000 + sequence % 50_000), + null)); + RequireSuccess(bound, "host presence"); + return bound.Value!; + } + + private static CreateJoinAttemptCommand CreateAttempt(int sequence, StoredListing listing) => new() + { + IdempotencyOwner = $"client-{sequence}", + IdempotencyKey = $"join-{sequence}", + RequestFingerprint = $"join-request-{sequence}", + ClientSubject = $"client-{sequence}", + AttemptId = new(Guid.NewGuid()), + MediationHandle = new(Guid.NewGuid()), + Scope = Scope, + ListingId = listing.Definition.ListingId, + ProtocolVersion = ProtocolVersion, + HostCapabilityFingerprint = new($"host-capability-{sequence}"), + ClientCapabilityFingerprint = new($"client-capability-{sequence}"), + ConnectionTicketFingerprint = new($"ticket-{sequence}"), + CapabilityDerivationSalt = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", + }; + + private static CapacityMeasurement MeasureConcurrentPunch( + InMemoryEphemeralRendezvousStore store, + List attempts, + int samples) + { + ConcurrentBag latencies = []; + Stopwatch total = Stopwatch.StartNew(); + Parallel.ForEach( + Enumerable.Range(0, samples), + new ParallelOptions { MaxDegreeOfParallelism = Math.Min(64, Environment.ProcessorCount * 4) }, + index => + { + CreateJoinAttemptCommand attempt = attempts[index]; + Stopwatch elapsed = Stopwatch.StartNew(); + RequireSuccess(store.BindAttemptEndpoint(new( + attempt.MediationHandle, + AttemptPeerRole.Host, + attempt.HostCapabilityFingerprint, + PublicEndpoint(20_000 + index % 20_000), + null)), "host punch"); + RequireSuccess(store.BindAttemptEndpoint(new( + attempt.MediationHandle, + AttemptPeerRole.Client, + attempt.ClientCapabilityFingerprint, + PublicEndpoint(40_000 + index % 20_000), + null)), "client punch"); + RequireSuccess(store.ConsumeIntroduction(attempt.MediationHandle), "introduction"); + latencies.Add(elapsed.Elapsed.TotalMilliseconds); + }); + total.Stop(); + return BuildMeasurement( + "simultaneous-punch-pairing", + latencies.ToArray(), + total.Elapsed, + budgetMilliseconds: 100, + minimumOperationsPerSecond: 2_000); + } + + private static CapacityMeasurement Measure( + string operation, + int samples, + double budgetMilliseconds, + double minimumOperationsPerSecond, + Action action) + { + double[] latencies = new double[samples]; + Stopwatch total = Stopwatch.StartNew(); + for (int index = 0; index < samples; index++) + { + long started = Stopwatch.GetTimestamp(); + action(index); + latencies[index] = Stopwatch.GetElapsedTime(started).TotalMilliseconds; + } + + total.Stop(); + return BuildMeasurement( + operation, + latencies, + total.Elapsed, + budgetMilliseconds, + minimumOperationsPerSecond); + } + + private static CapacityMeasurement BuildMeasurement( + string operation, + double[] latencies, + TimeSpan elapsed, + double budgetMilliseconds, + double minimumOperationsPerSecond) + { + Array.Sort(latencies); + double operationsPerSecond = latencies.Length / Math.Max(elapsed.TotalSeconds, 0.000_001); + double p95 = Percentile(latencies, 0.95); + return new( + operation, + latencies.Length, + Percentile(latencies, 0.50), + p95, + Percentile(latencies, 0.99), + operationsPerSecond, + minimumOperationsPerSecond, + budgetMilliseconds, + p95 <= budgetMilliseconds && operationsPerSecond >= minimumOperationsPerSecond); + } + + private static double Percentile(double[] sorted, double percentile) + { + int index = Math.Clamp((int)Math.Ceiling(sorted.Length * percentile) - 1, 0, sorted.Length - 1); + return sorted[index]; + } + + private static SoakEvidence RunAcceleratedSoak( + CapacityOptions options, + List failures) + { + GC.Collect(); + GC.WaitForPendingFinalizers(); + GC.Collect(); + long managedBefore = GC.GetTotalMemory(forceFullCollection: true); + int handlesBefore = Process.GetCurrentProcess().HandleCount; + ManualClock clock = new(); + CapacityOptions soakOptions = options with { Listings = 100, Attempts = 100 }; + InMemoryEphemeralRendezvousStore store = CreateStore(soakOptions, clock); + Stopwatch elapsed = Stopwatch.StartNew(); + int cycle = 0; + int peakScheduledExpiryEntries = 0; + while (cycle < options.SoakCycles + || elapsed.Elapsed < TimeSpan.FromSeconds(options.SoakSeconds)) + { + StoredListing listing = CreateVisibleListing(store, cycle); + for (int refresh = 0; refresh < 10; refresh++) + { + clock.Advance(TimeSpan.FromTicks(1)); + listing = RequireSuccess(store.RenewLease(new( + listing.Definition.ListingId, + listing.Definition.LeaseId, + listing.Definition.LeaseFingerprint, + listing.Definition.OwnerSubject, + listing.Version)), "soak lease refresh"); + listing = RequireSuccess(store.BindHostPresence(new( + listing.Definition.HostPresenceHandle, + listing.Definition.HostPresenceFingerprint, + PublicEndpoint(10_000 + cycle % 50_000), + null)), "soak presence refresh"); + } + + CreateJoinAttemptCommand attempt = CreateAttempt(cycle, listing); + RequireSuccess(store.CreateJoinAttempt(attempt), "soak join issuance"); + RequireSuccess(store.ConsumeReplay(new("capacity-soak", $"replay-{cycle}")), "soak replay"); + peakScheduledExpiryEntries = Math.Max( + peakScheduledExpiryEntries, + store.ScheduledExpiryEntryCount); + if (store.ScheduledExpiryEntryCount > 7) + { + failures.Add( + $"Mutable deadline refresh grew the expiry queue to " + + $"{store.ScheduledExpiryEntryCount} entries for one lifecycle."); + break; + } + + clock.Advance(TimeSpan.FromSeconds(151)); + EphemeralStoreSnapshot snapshot = store.GetSnapshot(); + if (snapshot.ActiveListings != 0 + || snapshot.ActiveJoinAttempts != 0 + || snapshot.ReplayMarkers != 0 + || snapshot.IdempotencyEntries != 0 + || snapshot.RetainedOutcomeReports != 0) + { + failures.Add($"Accelerated soak retained state after cycle {cycle}."); + break; + } + + cycle++; + } + + elapsed.Stop(); + GC.Collect(); + GC.WaitForPendingFinalizers(); + GC.Collect(); + long managedGrowth = GC.GetTotalMemory(forceFullCollection: true) - managedBefore; + int handleGrowth = Process.GetCurrentProcess().HandleCount - handlesBefore; + if (managedGrowth > 67_108_864) + { + failures.Add($"Soak retained {managedGrowth} managed bytes; budget is 64 MiB."); + } + + if (handleGrowth > 8) + { + failures.Add($"Soak retained {handleGrowth} process handles; budget is 8."); + } + + return new(cycle, elapsed.Elapsed, peakScheduledExpiryEntries, managedGrowth, handleGrowth); + } + + private static ObservedEndpoint PublicEndpoint(int port) => + new(AddressFamilyKind.Ipv4, "203.0.113.10", port); + + private static T RequireSuccess(StoreResult result, string operation) + { + if (!result.Succeeded) + { + throw new InvalidOperationException($"{operation} failed with {result.Code}."); + } + + return result.Value!; + } + + private static string ReadCpuModel() + { + const string cpuInfoPath = "/proc/cpuinfo"; + if (!File.Exists(cpuInfoPath)) + { + return "unavailable"; + } + + string? model = File.ReadLines(cpuInfoPath) + .FirstOrDefault(static line => line.StartsWith("model name", StringComparison.Ordinal)); + int separator = model?.IndexOf(':') ?? -1; + return separator >= 0 ? model![(separator + 1)..].Trim() : "unavailable"; + } + + private static string ReadCgroupValue(string path) => + File.Exists(path) ? File.ReadAllText(path).Trim() : "not-enforced"; + + private sealed class ManualClock : IWallClock, IMonotonicClock + { + public DateTimeOffset UtcNow { get; private set; } = DateTimeOffset.UtcNow; + public TimeSpan Elapsed { get; private set; } + + public void Advance(TimeSpan duration) + { + UtcNow += duration; + Elapsed += duration; + } + } + + private readonly record struct SoakEvidence( + int Cycles, + TimeSpan Duration, + int PeakScheduledExpiryEntries, + long ManagedGrowthBytes, + int HandleGrowth); +} diff --git a/tests/FinalFactory.Rendezvous.Capacity/FinalFactory.Rendezvous.Capacity.csproj b/tests/FinalFactory.Rendezvous.Capacity/FinalFactory.Rendezvous.Capacity.csproj new file mode 100644 index 0000000..ea56899 --- /dev/null +++ b/tests/FinalFactory.Rendezvous.Capacity/FinalFactory.Rendezvous.Capacity.csproj @@ -0,0 +1,13 @@ + + + Exe + net10.0 + FinalFactory.Rendezvous.Capacity + FinalFactory.Rendezvous.Capacity + false + + + + + + diff --git a/tests/FinalFactory.Rendezvous.Capacity/Program.cs b/tests/FinalFactory.Rendezvous.Capacity/Program.cs new file mode 100644 index 0000000..e289a17 --- /dev/null +++ b/tests/FinalFactory.Rendezvous.Capacity/Program.cs @@ -0,0 +1,37 @@ +using System.Text.Json; + +namespace FinalFactory.Rendezvous.Capacity; + +internal static class Program +{ + private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web) + { + WriteIndented = true, + }; + + public static async Task Main(string[] args) + { + CapacityOptions options; + try + { + options = CapacityOptions.Parse(args); + } + catch (ArgumentException exception) + { + Console.Error.WriteLine(exception.Message); + return 2; + } + + CapacityReport report = await CapacityRunner.RunAsync(options).ConfigureAwait(false); + string json = JsonSerializer.Serialize(report, JsonOptions); + Console.WriteLine(json); + if (options.OutputPath is not null) + { + string fullPath = Path.GetFullPath(options.OutputPath); + Directory.CreateDirectory(Path.GetDirectoryName(fullPath)!); + await File.WriteAllTextAsync(fullPath, json + Environment.NewLine).ConfigureAwait(false); + } + + return report.Passed ? 0 : 1; + } +} diff --git a/tests/FinalFactory.Rendezvous.Capacity/packages.lock.json b/tests/FinalFactory.Rendezvous.Capacity/packages.lock.json new file mode 100644 index 0000000..14ff028 --- /dev/null +++ b/tests/FinalFactory.Rendezvous.Capacity/packages.lock.json @@ -0,0 +1,39 @@ +{ + "version": 2, + "dependencies": { + "net10.0": { + "finalfactory.rendezvous.contracts": { + "type": "Project" + }, + "finalfactory.rendezvous.server": { + "type": "Project", + "dependencies": { + "FinalFactory.Rendezvous.Contracts": "[1.0.0, )", + "LiteNetLib": "[2.1.4, )", + "Microsoft.AspNetCore.OpenApi": "[10.0.9, )" + } + }, + "LiteNetLib": { + "type": "CentralTransitive", + "requested": "[2.1.4, )", + "resolved": "2.1.4", + "contentHash": "KWlxvMw3Urpqj9joD96LRiK+LC62pQNs/zkXRJc+rHnxgkGp+vV703xzDrxRmv+V1YhCFfIGzs5nrVWtREIlyA==" + }, + "Microsoft.AspNetCore.OpenApi": { + "type": "CentralTransitive", + "requested": "[10.0.9, )", + "resolved": "10.0.9", + "contentHash": "1ihb8FO9cGgEK1/m3CTtT/SfnynwmiZib0W2pcDVj3KSWk/Sca4VOXEtaptKQc582zpFrzTFiwkGRCglt6H+WQ==", + "dependencies": { + "Microsoft.OpenApi": "2.0.0" + } + }, + "Microsoft.OpenApi": { + "type": "CentralTransitive", + "requested": "[2.7.5, )", + "resolved": "2.7.5", + "contentHash": "0FA67RSnRM4tcBKqiqVu/HPdZ9+QOKbmeRjxRUGTCjPU4C0bmUhd97Dso7Yild5P7nOV6GxJ2xrK0Kv/O9xp0w==" + } + } + } +} \ No newline at end of file diff --git a/tests/FinalFactory.Rendezvous.Tests/Client/RendezvousClientIntegrationTests.cs b/tests/FinalFactory.Rendezvous.Tests/Client/RendezvousClientIntegrationTests.cs index ce14313..445f5dd 100644 --- a/tests/FinalFactory.Rendezvous.Tests/Client/RendezvousClientIntegrationTests.cs +++ b/tests/FinalFactory.Rendezvous.Tests/Client/RendezvousClientIntegrationTests.cs @@ -1,3 +1,6 @@ +using System.Diagnostics; +using System.Net; +using System.Net.Sockets; using FinalFactory.Rendezvous.Client; using FinalFactory.Rendezvous.Contracts; using FinalFactory.Rendezvous.Server.Abuse; @@ -19,6 +22,49 @@ namespace FinalFactory.Rendezvous.Tests.Client; public sealed class RendezvousClientIntegrationTests { + [Fact] + public async Task RestartReturnsTypedUnavailabilityThenAllowsHostReregistration() + { + int port = ReserveTcpPort(); + string address = $"http://127.0.0.1:{port}"; + using HttpClient client = new() { BaseAddress = new Uri(address) }; + RendezvousClientOptions noRetry = new() + { + MaximumSafeRetries = 0, + RequestTimeout = TimeSpan.FromSeconds(1), + }; + ClientTestHost first = await ClientTestHost.StartAsync(address); + RendezvousPublisherClient publisher = new(client, noRetry); + RendezvousClientResult registered = await publisher.RegisterAsync( + CreateRegistration(100), + first.PublisherCredential); + PublishedSession initialSession = AssertSuccess(registered); + BindPresence(first, initialSession, 41_100); + RendezvousSessionBrowserClient browser = new(client, noRetry); + BrowseSessionsResponse beforeRestart = AssertSuccess(await browser.BrowseAsync(BrowseRequest())); + Assert.Equal(initialSession.ListingId, Assert.Single(beforeRestart.Items).ListingId); + Stopwatch restart = Stopwatch.StartNew(); + await first.DisposeAsync(); + + RendezvousClientResult unavailable = await browser.BrowseAsync(BrowseRequest()); + Assert.False(unavailable.IsSuccess); + Assert.Equal(RendezvousErrorCode.ServiceUnavailable, unavailable.Error); + + await using ClientTestHost second = await ClientTestHost.StartAsync(address); + RendezvousClientResult reregistered = await publisher.RegisterAsync( + CreateRegistration(101), + second.PublisherCredential); + PublishedSession replacementSession = AssertSuccess(reregistered); + Assert.NotEqual(initialSession.ListingId, replacementSession.ListingId); + BindPresence(second, replacementSession, 41_101); + BrowseSessionsResponse afterRestart = AssertSuccess(await browser.BrowseAsync(BrowseRequest())); + Assert.Equal(replacementSession.ListingId, Assert.Single(afterRestart.Items).ListingId); + Assert.DoesNotContain(afterRestart.Items, item => item.ListingId == initialSession.ListingId); + Assert.True( + restart.Elapsed < TimeSpan.FromSeconds(5), + $"Local restart and host re-registration took {restart.Elapsed}."); + } + [Fact] public async Task PublisherAndBrowserClientsCompleteTheRealSessionLifecycleAndPaging() { @@ -102,6 +148,27 @@ public sealed class RendezvousClientIntegrationTests return Assert.IsAssignableFrom(result.Value); } + private static void BindPresence(ClientTestHost host, PublishedSession session, int port) + { + Assert.True(host.Capabilities.TryFingerprint( + session.HostPresenceCapability, + out SecretFingerprint fingerprint)); + Assert.Equal(StoreResultCode.Success, host.Store.BindHostPresence(new( + session.HostPresenceHandle, + fingerprint, + new(AddressFamilyKind.Ipv4, "203.0.113.80", port), + null)).Code); + } + + private static BrowseSessionsRequest BrowseRequest() => new() + { + GameId = new("space-game"), + EnvironmentId = new("production"), + RegionId = new("eu-central"), + ProtocolVersion = 7, + PageSize = 10, + }; + private static RegisterSessionRequest CreateRegistration(int index) => new() { IdempotencyKey = $"sdk-integration-{index}", @@ -139,7 +206,7 @@ public sealed class RendezvousClientIntegrationTests internal EphemeralCapabilityIssuer Capabilities { get; } internal string PublisherCredential { get; } - internal static async Task StartAsync() + internal static async Task StartAsync(string? bindAddress = null) { ManualRendezvousClock clock = new(ProvisioningTestData.Now); EphemeralStoreOptions stateOptions = new(); @@ -153,7 +220,7 @@ public sealed class RendezvousClientIntegrationTests string credential = provisioning.Credentials.Issue(principal, clock.UtcNow); WebApplicationBuilder builder = WebApplication.CreateBuilder(); - builder.WebHost.UseUrls("http://127.0.0.1:0"); + builder.WebHost.UseUrls(bindAddress ?? "http://127.0.0.1:0"); builder.Services.ConfigureHttpJsonOptions(static options => ContractJson.Configure(options.SerializerOptions)); builder.Services.Configure(static options => @@ -180,10 +247,10 @@ public sealed class RendezvousClientIntegrationTests app.MapRendezvousContractEndpoints(); await app.StartAsync(); IServer server = app.Services.GetRequiredService(); - string address = Assert.Single(server.Features.Get()!.Addresses); + string serviceAddress = Assert.Single(server.Features.Get()!.Addresses); return new( app, - new HttpClient { BaseAddress = new Uri(address) }, + new HttpClient { BaseAddress = new Uri(serviceAddress) }, store, capabilities, credential); @@ -196,4 +263,13 @@ public sealed class RendezvousClientIntegrationTests await _application.DisposeAsync(); } } + + private static int ReserveTcpPort() + { + TcpListener listener = new(IPAddress.Loopback, 0); + listener.Start(); + int port = ((IPEndPoint)listener.LocalEndpoint).Port; + listener.Stop(); + return port; + } } diff --git a/tests/FinalFactory.Rendezvous.Tests/Deployment/ProductionProcessTests.cs b/tests/FinalFactory.Rendezvous.Tests/Deployment/ProductionProcessTests.cs index 46e3f9c..82191b5 100644 --- a/tests/FinalFactory.Rendezvous.Tests/Deployment/ProductionProcessTests.cs +++ b/tests/FinalFactory.Rendezvous.Tests/Deployment/ProductionProcessTests.cs @@ -61,6 +61,32 @@ public sealed class ProductionProcessTests Assert.InRange(shutdown.Elapsed, TimeSpan.FromMilliseconds(700), TimeSpan.FromSeconds(5)); AssertTcpPortIsReleased(httpPort); AssertUdpPortIsReleased(udpPort); + + process.Dispose(); + process = null; + Stopwatch replacementReady = Stopwatch.StartNew(); + ProcessStartInfo replacementInfo = CreateStartInfo(httpPort, udpPort, secretPath); + process = Process.Start(replacementInfo) + ?? throw new InvalidOperationException("The replacement production process did not start."); + Task replacementOutput = process.StandardOutput.ReadToEndAsync(); + Task replacementError = process.StandardError.ReadToEndAsync(); + await WaitForReadyAsync(httpPort, process, TimeSpan.FromSeconds(15)); + Assert.True( + replacementReady.Elapsed < TimeSpan.FromSeconds(15), + $"Replacement readiness took {replacementReady.Elapsed}."); + AssertUdpPortIsBound(udpPort); + + await SendSigtermAsync(process); + using CancellationTokenSource replacementTimeout = new(TimeSpan.FromSeconds(6)); + await process.WaitForExitAsync(replacementTimeout.Token); + string replacementFinalOutput = await replacementOutput; + string replacementFinalError = await replacementError; + Assert.True( + process.ExitCode == 0, + $"Replacement exited with {process.ExitCode}. " + + $"stdout: {replacementFinalOutput} stderr: {replacementFinalError}"); + AssertTcpPortIsReleased(httpPort); + AssertUdpPortIsReleased(udpPort); } finally { @@ -79,6 +105,97 @@ public sealed class ProductionProcessTests } } + [Fact] + public async Task ProductionTransportSoakKeepsHandlesMemoryAndSocketsBounded() + { + if (!OperatingSystem.IsLinux()) + { + return; + } + + int httpPort = ReserveTcpPort(); + int udpPort = ReserveUdpPort(); + string secretPath = Path.Combine( + Path.GetTempPath(), + $"rendezvous-transport-soak-secret-{Guid.NewGuid():N}"); + await File.WriteAllBytesAsync(secretPath, RandomNumberGenerator.GetBytes(32)); + Process? process = null; + try + { + process = Process.Start(CreateStartInfo(httpPort, udpPort, secretPath)) + ?? throw new InvalidOperationException("The production soak process did not start."); + Task standardOutput = process.StandardOutput.ReadToEndAsync(); + Task standardError = process.StandardError.ReadToEndAsync(); + await WaitForReadyAsync(httpPort, process, TimeSpan.FromSeconds(10)); + process.Refresh(); + int baselineHandles = process.HandleCount; + long baselineWorkingSet = process.WorkingSet64; + int peakHandles = baselineHandles; + byte[] invalidDatagram = RandomNumberGenerator.GetBytes(64); + IPEndPoint udpEndpoint = new(IPAddress.Loopback, udpPort); + using HttpClient client = new() { Timeout = TimeSpan.FromSeconds(1) }; + using UdpClient udp = new(); + Stopwatch soak = Stopwatch.StartNew(); + int cycles = 0; + int accepted = 0; + int shed = 0; + while (soak.Elapsed < TimeSpan.FromSeconds(10)) + { + using HttpResponseMessage response = await client.GetAsync( + $"http://127.0.0.1:{httpPort}/health/live"); + if (response.StatusCode == HttpStatusCode.OK) + { + accepted++; + } + else + { + Assert.Equal(HttpStatusCode.TooManyRequests, response.StatusCode); + shed++; + } + + await udp.SendAsync(invalidDatagram, udpEndpoint); + cycles++; + if (cycles % 100 == 0) + { + process.Refresh(); + peakHandles = Math.Max(peakHandles, process.HandleCount); + } + } + + Assert.True(cycles >= 100, $"Transport soak completed only {cycles} cycles."); + Assert.True(accepted > 0, "Transport soak never admitted a health request."); + Assert.True(shed > 0, "Transport soak never exercised typed HTTP load shedding."); + await Task.Delay(TimeSpan.FromSeconds(2)); + using (HttpResponseMessage recovered = await client.GetAsync( + $"http://127.0.0.1:{httpPort}/health/live")) + { + Assert.Equal(HttpStatusCode.OK, recovered.StatusCode); + } + + process.Refresh(); + Assert.InRange(peakHandles, 0, baselineHandles + 32); + Assert.InRange(process.HandleCount, 0, baselineHandles + 16); + Assert.InRange(process.WorkingSet64, 0, baselineWorkingSet + 67_108_864); + AssertUdpPortIsBound(udpPort); + + await SendSigtermAsync(process); + using CancellationTokenSource shutdownTimeout = new(TimeSpan.FromSeconds(6)); + await process.WaitForExitAsync(shutdownTimeout.Token); + string output = await standardOutput; + string error = await standardError; + Assert.True( + process.ExitCode == 0, + $"Transport soak process failed. stdout: {output} stderr: {error}"); + AssertTcpPortIsReleased(httpPort); + AssertUdpPortIsReleased(udpPort); + } + finally + { + await StopProcessTreeAsync(process); + File.Delete(secretPath); + } + } + [Fact] public async Task DocumentedSmokeScriptReachesHttpAndAuthenticatedUdpFlow() { diff --git a/tests/FinalFactory.Rendezvous.Tests/State/InMemoryEphemeralRendezvousStoreTests.cs b/tests/FinalFactory.Rendezvous.Tests/State/InMemoryEphemeralRendezvousStoreTests.cs index 147f627..e09c046 100644 --- a/tests/FinalFactory.Rendezvous.Tests/State/InMemoryEphemeralRendezvousStoreTests.cs +++ b/tests/FinalFactory.Rendezvous.Tests/State/InMemoryEphemeralRendezvousStoreTests.cs @@ -411,6 +411,92 @@ public sealed class InMemoryEphemeralRendezvousStoreTests Assert.True(fixture.Store.CreateJoinAttempt(attempt).Succeeded); Assert.Equal(StoreResultCode.CapacityExceeded, fixture.Store.CreateJoinAttempt( fixture.AttemptCommand(listing, "client-quota-2") with { ScopeAttemptLimit = 1 }).Code); + + fixture.Clock.Advance(TimeSpan.FromSeconds(30)); + Assert.True(fixture.Store.BindHostPresence(new( + first.Listing.HostPresenceHandle, + first.Listing.HostPresenceFingerprint, + EphemeralStateFixture.PublicEndpoint(42_101), + null)).Succeeded); + Assert.True(fixture.Store.CreateJoinAttempt( + fixture.AttemptCommand(listing, "client-quota-3") with { ScopeAttemptLimit = 1 }).Succeeded); + } + + [Fact] + public void RemovingAListingReleasesItsConstantTimeOwnerQuota() + { + EphemeralStateFixture fixture = new(); + CreateListingCommand first = fixture.ListingCommand(owner: "publisher-quota") with + { + OwnerListingLimit = 1, + }; + CreateListingCommand second = fixture.ListingCommand(owner: "publisher-quota") with + { + OwnerListingLimit = 1, + }; + StoreResult created = fixture.Store.CreateListing(first); + Assert.True(created.Succeeded); + Assert.Equal(StoreResultCode.CapacityExceeded, fixture.Store.CreateListing(second).Code); + + Assert.True(fixture.Store.DeleteListing(new( + first.Listing.ListingId, + first.Listing.LeaseId, + first.Listing.LeaseFingerprint, + first.Listing.OwnerSubject)).Succeeded); + Assert.True(fixture.Store.CreateListing(second).Succeeded); + } + + [Fact] + public void RepeatedMutableDeadlineRefreshesKeepOneScheduledEntryPerKey() + { + EphemeralStateFixture fixture = new(); + StoredListing listing = fixture.CreateVisibleListing(out _); + + for (int iteration = 0; iteration < 10_000; iteration++) + { + fixture.Clock.Advance(TimeSpan.FromTicks(1)); + StoreResult renewed = fixture.Store.RenewLease(new( + listing.Definition.ListingId, + listing.Definition.LeaseId, + listing.Definition.LeaseFingerprint, + listing.Definition.OwnerSubject, + listing.Version)); + Assert.True(renewed.Succeeded, $"Renewal {iteration} failed with {renewed.Code}."); + listing = renewed.Value!; + StoreResult presence = fixture.Store.BindHostPresence(new( + listing.Definition.HostPresenceHandle, + listing.Definition.HostPresenceFingerprint, + EphemeralStateFixture.PublicEndpoint(42_200), + null)); + Assert.True(presence.Succeeded, $"Presence {iteration} failed with {presence.Code}."); + StoreResult revoked = fixture.Store.RevokePrincipal( + "repeated-revocation", + TimeSpan.FromMinutes(1)); + Assert.True(revoked.Succeeded, $"Revocation {iteration} failed with {revoked.Code}."); + } + + Assert.Equal(4, fixture.Store.ScheduledExpiryEntryCount); + fixture.Clock.Advance(TimeSpan.FromSeconds(19)); + Assert.True(fixture.Store.GetListing(listing.Definition.ListingId, true).Succeeded); + Assert.Equal(4, fixture.Store.ScheduledExpiryEntryCount); + } + + [Fact] + public void RepeatedPrincipalRevocationCanExtendButCannotShortenProtection() + { + EphemeralStateFixture fixture = new(); + const string subject = "protected-publisher"; + + Assert.True(fixture.Store.RevokePrincipal(subject, TimeSpan.FromMinutes(10)).Succeeded); + fixture.Clock.Advance(TimeSpan.FromSeconds(1)); + Assert.True(fixture.Store.RevokePrincipal(subject, TimeSpan.FromSeconds(1)).Succeeded); + fixture.Clock.Advance(TimeSpan.FromSeconds(2)); + Assert.Equal( + StoreResultCode.Revoked, + fixture.Store.CreateListing(fixture.ListingCommand(owner: subject)).Code); + + fixture.Clock.Advance(TimeSpan.FromSeconds(597)); + Assert.True(fixture.Store.CreateListing(fixture.ListingCommand(owner: subject)).Succeeded); } [Fact]