test(integration): add deterministic NAT topology harness (#14)

.gitea/workflows/ci.yml

@@ -35,3 +35,56 @@ jobs:
 
       - name: Test
         run: dotnet test Rendezvous.slnx --configuration Release --no-build
+
+      - name: Test privileged Linux namespace topology when available
+        shell: bash
+        run: |
+          set -euo pipefail
+          probe="rendezvous-probe-$$"
+          suffix="$(( $$ % 100000 ))"
+          bridge="rvb${suffix}"
+          veth_root="rvr${suffix}"
+          veth_peer="rvp${suffix}"
+          cleanup_probe() {
+            if [[ -n "$veth_root" ]]; then
+              ip link delete "$veth_root" >/dev/null 2>&1 || true
+            fi
+            if [[ -n "$bridge" ]]; then
+              ip link delete "$bridge" >/dev/null 2>&1 || true
+            fi
+            if [[ -n "$probe" ]]; then
+              ip netns delete "$probe" >/dev/null 2>&1 || true
+            fi
+          }
+          trap cleanup_probe EXIT
+          if command -v ip >/dev/null 2>&1 \
+            && command -v iptables >/dev/null 2>&1 \
+            && command -v sysctl >/dev/null 2>&1 \
+            && ip netns add "$probe" 2>/dev/null \
+            && ip link add "$bridge" type bridge \
+            && ip link add "$veth_root" type veth peer name "$veth_peer" \
+            && ip link set "$veth_root" master "$bridge" \
+            && ip link set "$veth_peer" netns "$probe" \
+            && ip netns exec "$probe" sysctl -q -w net.ipv4.ip_forward=1 \
+            && ip netns exec "$probe" iptables -t nat -A POSTROUTING -o "$veth_peer" -j MASQUERADE \
+            && ip netns exec "$probe" iptables -A FORWARD -i "$veth_peer" -o lo \
+              -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT; then
+            ip link delete "$veth_root"
+            veth_root=""
+            ip link delete "$bridge"
+            bridge=""
+            ip netns delete "$probe"
+            probe=""
+            results="${RUNNER_TEMP:-/tmp}/rendezvous-netns-results"
+            mkdir -p "$results"
+            RENDEZVOUS_RUN_NETNS_TESTS=1 dotnet test Rendezvous.slnx \
+              --configuration Release \
+              --no-build \
+              --filter FullyQualifiedName~PrivilegedLinuxNatNamespacesCompleteDirectTrafficAcrossSeparateObservedEndpoints \
+              --logger "trx;LogFileName=netns.trx" \
+              --results-directory "$results"
+            grep -q 'testName="[^"]*\.PrivilegedLinuxNatNamespacesCompleteDirectTrafficAcrossSeparateObservedEndpoints"' \
+              "$results/netns.trx"
+          else
+            echo "Network namespaces/NAT tooling unavailable; deterministic loopback topology remains the required gate."
+          fi

README.md

@@ -91,6 +91,9 @@ Tenant policy, publisher/operator principals, and production key custody are
 defined in [game provisioning and signing-key lifecycle](docs/security/provisioning.md).
 The scriptable host/browser/join diagnostic and its stable automation contract are
 documented in the [TestClient integration guide](docs/integration/test-client.md).
+The always-on three-party scenarios, optional Linux namespace topology, and
+simulation limits are documented in the
+[deterministic topology harness](docs/integration/topology-harness.md).
 
 ## Development
 

docs/integration/test-client.md

@@ -7,6 +7,9 @@ It exists for integration development, CI smoke checks, deployment verification,
 and operator diagnosis. It is intentionally not a production game client, game
 server, matchmaking UI, or relay.
 
+The automated scenario matrix, privileged Linux namespace run, and topology
+limitations are documented in the [deterministic topology harness](topology-harness.md).
+
 ## Prerequisites
 
 Start a configured Rendezvous service and note both its HTTP base URL and UDP

docs/integration/topology-harness.md

@@ -0,0 +1,91 @@
+# Deterministic topology harness
+
+Issue #14 is verified at three layers. The layers are deliberately separate so
+the always-on gate remains deterministic while privileged CI workers can add a
+stronger operating-system topology without overstating what local emulation
+proves about the public Internet.
+
+## Always-on public-process gate
+
+`TestClientProcessIntegrationTests` launches the built server and the same
+`FinalFactory.Rendezvous.TestClient` executable shipped to operators. Every
+child process uses `--script --json`, dynamic HTTP and UDP ports, bounded
+state-driven waits, and enforced process-tree cleanup.
+
+The suite proves:
+
+| Scenario | Required observation |
+| --- | --- |
+| Three-party happy path | register, presence-ready, browse, authorize, punch, authenticated LiteNetLib connection, direct ping/echo/ack/completion traffic, outcome report, disconnect, deregister |
+| Same-LAN candidate | the connected peer is reported as `loopback` or `private`, never inferred merely from an introduction callback |
+| Empty and missing selection | browse exits `11`; exact missing lookup exits `10` |
+| Wrong tenant/protocol | no listing is returned for an incompatible protocol; exact joins with either mismatch fail before `join.punch` |
+| Traversal timeout | an unreachable mediator produces typed `PunchTimedOut`, exits `12`, advertises the configured dedicated fallback, and never connects to it |
+| Caller cancellation | POSIX `SIGINT` exits `130`, deregisters the listing, and removes it from public lookup |
+| Abrupt host loss | the listing disappears after the presence window and before its lease expires; public exact lookup intentionally reports `NotFound` |
+| Bounded host without a peer | exits `13` and still deregisters |
+
+Captured output is parsed as the stable JSON v1 event schema. Publisher
+credentials and signing-key material are checked against all captured output.
+The direct traffic payload is handled only by the caller-owned host and client
+LiteNetLib managers; the HTTP service and mediator do not implement or observe
+the echo protocol.
+
+Run the always-on scenarios with:
+
+```bash
+dotnet test Rendezvous.slnx --configuration Release --no-build \
+  --filter FullyQualifiedName~TestClientProcessIntegrationTests
+```
+
+## Deterministic protocol and adverse-state gate
+
+The following real service-boundary tests cover conditions that a public CLI
+cannot safely manufacture by accepting raw capabilities or tickets:
+
+| Scenario | Test evidence |
+| --- | --- |
+| Same-NAT private candidates | `NatMediationProcessorTests.MatchedPeersReceiveOneIntroductionAndSameNatPrivateCandidates` |
+| Separate observed endpoints | `NatMediationProcessorTests.DifferentNatsAndInvalidLocalClaimsExposeOnlyObservedPublicEndpoints` |
+| One-time introduction and replay | `InMemoryEphemeralRendezvousStoreTests.AttemptCapabilitiesAndIntroductionAreOneTime` |
+| Direct ticket replay | `RendezvousCoordinatorIntegrationTests.CallerOwnedManagersCompleteAuthenticatedDirectConnectionAndRejectTicketReplay` |
+| Wrong tenant/protocol and stale presence | `InMemoryEphemeralRendezvousStoreTests.JoinRequiresExactScopeProtocolAndFreshHostPresence` |
+| Cancellation and late callbacks | `RendezvousCoordinatorBehaviorTests.CancellationCompletesExactlyOnceAndLateCallbacksCannotReopenTheAttempt` |
+| Mediator restart | both cases of `UdpMediatorServiceTests.NativeLiteNetLibRequestsIntroduceTheAuthorizedPair`; the restarted case rebinds the same UDP port and completes a native LiteNetLib introduction |
+
+These tests use fake monotonic clocks or state predicates where expiry and race
+ordering matter. They do not use fixed sleeps as proof of state.
+
+## Privileged Linux namespace gate
+
+When a Linux CI worker can create network namespaces, the workflow sets
+`RENDEZVOUS_RUN_NETNS_TESTS=1` and reruns
+`PrivilegedLinuxNatNamespacesCompleteDirectTrafficAcrossSeparateObservedEndpoints`.
+The test creates a temporary WAN bridge, an isolated service namespace, two NAT
+router namespaces, and isolated host/client LAN namespaces. Each NAT has its own
+inside subnet and WAN address. Linux forwarding plus per-router MASQUERADE rules
+force the service to observe separate translated endpoints; the public TestClient
+processes must then complete authenticated direct traffic through those mappings
+using the public candidate. Namespaces, rules, veth pairs, bridge, processes, and
+sockets are removed in bounded async-disposal paths. A cleanup failure fails the
+test.
+
+If `ip netns add`/`iptables` is unavailable or the worker lacks `CAP_NET_ADMIN`,
+CI records the limitation and keeps the always-on loopback suite as the required gate.
+To request the privileged run explicitly:
+
+```bash
+RENDEZVOUS_RUN_NETNS_TESTS=1 dotnet test Rendezvous.slnx \
+  --configuration Release --no-build \
+  --filter FullyQualifiedName~PrivilegedLinuxNatNamespacesCompleteDirectTrafficAcrossSeparateObservedEndpoints
+```
+
+## What this does not prove
+
+Loopback, MASQUERADE, and namespace routing cannot reproduce every consumer router,
+carrier-grade NAT, firewall, IPv6 transition mechanism, symmetric NAT mapping,
+or real-world packet-loss pattern. The separate-observed-endpoint processor
+test proves that untrusted private claims are excluded and public candidates are
+selected; it is not presented as universal Internet traversal proof. Real
+network canaries and measured production readiness remain the scope of issue
+#23.

tests/FinalFactory.Rendezvous.Tests/Server/UdpMediatorServiceTests.cs

@@ -86,8 +86,10 @@ public sealed class UdpMediatorServiceTests
         await service.StopAsync(timeout.Token);
     }
 
-    [Fact]
-    public async Task NativeLiteNetLibRequestsIntroduceTheAuthorizedPair()
+    [Theory]
+    [InlineData(false)]
+    [InlineData(true)]
+    public async Task NativeLiteNetLibRequestsIntroduceTheAuthorizedPair(bool restartMediator)
     {
         using CancellationTokenSource timeout = new(TimeSpan.FromSeconds(5));
         using JoinAttemptFixture fixture = new();
@@ -103,18 +105,6 @@ public sealed class UdpMediatorServiceTests
             fixture.Sessions.Store,
             fixture.Sessions.Capabilities,
             fixture.Service);
-        using UdpMediatorService service = new(
-            Options.Create(new UdpMediatorOptions
-            {
-                ListenAddress = IPAddress.Loopback.ToString(),
-                Port = 0,
-                MaxDatagramsPerPoll = 8,
-                PollIntervalMilliseconds = 1,
-            }),
-            NullLogger<UdpMediatorService>.Instance,
-            processor);
-        await service.StartAsync(timeout.Token);
-
         EventBasedNetListener hostListener = new();
         EventBasedNetListener clientListener = new();
         NetManager host = new(hostListener) { NatPunchEnabled = true };
@@ -127,53 +117,141 @@ public sealed class UdpMediatorServiceTests
         clientPunch.NatIntroductionSuccess += (_, _, ticket) => clientTickets.Add(ticket);
         host.NatPunchModule.Init(hostPunch);
         client.NatPunchModule.Init(clientPunch);
+        UdpMediatorService? service = null;
+        bool serviceStarted = false;
 
         try
         {
+            service = CreateMediator(processor, port: 0);
+            await service.StartAsync(timeout.Token);
+            serviceStarted = true;
             Assert.True(host.Start(0));
             Assert.True(client.Start(0));
-            IPEndPoint mediator = Assert.IsType<IPEndPoint>(service.LocalEndpoint);
-            host.NatPunchModule.SendNatIntroduceRequest(
-                mediator,
-                NatPunchRequestTokenCodec.Encode(
-                    NatPunchPeerRole.Host,
-                    created.MediationHandle,
-                    hostAttempt.HostPunchCapability));
-            client.NatPunchModule.SendNatIntroduceRequest(
-                mediator,
-                NatPunchRequestTokenCodec.Encode(
-                    NatPunchPeerRole.Client,
-                    created.MediationHandle,
-                    created.ClientPunchCapability));
-
-            while ((hostTickets.Count == 0 || clientTickets.Count == 0)
-                   && !timeout.IsCancellationRequested)
+            if (restartMediator)
             {
-                host.PollEvents();
-                host.NatPunchModule.PollEvents();
-                client.PollEvents();
-                client.NatPunchModule.PollEvents();
-                await Task.Delay(5, timeout.Token);
+                await AssertNativeIntroductionAsync(
+                    service,
+                    host,
+                    client,
+                    hostTickets,
+                    clientTickets,
+                    created,
+                    hostAttempt,
+                    expectedCount: 1,
+                    cancellationToken: timeout.Token);
+                created = fixture.Create(registration.ListingId, "native-litenet-after-restart");
+                hostAttempt = fixture.Service.BrowseForHost(
+                    registration.ListingId,
+                    ContractLimits.ContractVersion,
+                    registration.LeaseToken,
+                    ContractLimits.BrowserPageMaxItems,
+                    null).Value!.Items.Single(item => item.AttemptId == created.AttemptId);
+                int boundPort = Assert.IsType<IPEndPoint>(service.LocalEndpoint).Port;
+                await service.StopAsync(timeout.Token);
+                serviceStarted = false;
+                service.Dispose();
+                service = null;
+                service = CreateMediator(processor, boundPort);
+                await service.StartAsync(timeout.Token);
+                serviceStarted = true;
+                Assert.Equal(boundPort, Assert.IsType<IPEndPoint>(service.LocalEndpoint).Port);
             }
 
-            string hostTicket = Assert.Single(hostTickets.Distinct(StringComparer.Ordinal));
-            string clientTicket = Assert.Single(clientTickets.Distinct(StringComparer.Ordinal));
-            Assert.Equal(hostTicket, clientTicket);
-            Assert.True(NatIntroductionTokenCodec.TryDecode(
-                hostTicket,
-                out NatIntroductionToken? introduction));
-            Assert.NotNull(introduction);
-            Assert.Equal(created.AttemptId, introduction.AttemptId);
-            Assert.Equal(43, introduction.ConnectionTicket.Length);
+            await AssertNativeIntroductionAsync(
+                service,
+                host,
+                client,
+                hostTickets,
+                clientTickets,
+                created,
+                hostAttempt,
+                expectedCount: restartMediator ? 2 : 1,
+                cancellationToken: timeout.Token);
         }
         finally
         {
             host.Stop();
             client.Stop();
-            await service.StopAsync(CancellationToken.None);
+            if (service is not null)
+            {
+                try
+                {
+                    if (serviceStarted)
+                    {
+                        using CancellationTokenSource cleanup = new(TimeSpan.FromSeconds(5));
+                        await service.StopAsync(cleanup.Token);
+                    }
+                }
+                finally
+                {
+                    service.Dispose();
+                }
+            }
         }
     }
 
+    private static async Task AssertNativeIntroductionAsync(
+        UdpMediatorService service,
+        NetManager host,
+        NetManager client,
+        List<string> hostTickets,
+        List<string> clientTickets,
+        CreateJoinAttemptResponse created,
+        HostJoinAttempt hostAttempt,
+        int expectedCount,
+        CancellationToken cancellationToken)
+    {
+        IPEndPoint mediator = Assert.IsType<IPEndPoint>(service.LocalEndpoint);
+        host.NatPunchModule.SendNatIntroduceRequest(
+            mediator,
+            NatPunchRequestTokenCodec.Encode(
+                NatPunchPeerRole.Host,
+                created.MediationHandle,
+                hostAttempt.HostPunchCapability));
+        client.NatPunchModule.SendNatIntroduceRequest(
+            mediator,
+            NatPunchRequestTokenCodec.Encode(
+                NatPunchPeerRole.Client,
+                created.MediationHandle,
+                created.ClientPunchCapability));
+
+        while ((hostTickets.Distinct(StringComparer.Ordinal).Count() < expectedCount
+                || clientTickets.Distinct(StringComparer.Ordinal).Count() < expectedCount)
+               && !cancellationToken.IsCancellationRequested)
+        {
+            host.PollEvents();
+            host.NatPunchModule.PollEvents();
+            client.PollEvents();
+            client.NatPunchModule.PollEvents();
+            await Task.Delay(5, cancellationToken);
+        }
+
+        List<string> distinctHostTickets = hostTickets.Distinct(StringComparer.Ordinal).ToList();
+        List<string> distinctClientTickets = clientTickets.Distinct(StringComparer.Ordinal).ToList();
+        Assert.Equal(expectedCount, distinctHostTickets.Count);
+        Assert.Equal(expectedCount, distinctClientTickets.Count);
+        string hostTicket = distinctHostTickets[^1];
+        string clientTicket = distinctClientTickets[^1];
+        Assert.Equal(hostTicket, clientTicket);
+        Assert.True(NatIntroductionTokenCodec.TryDecode(
+            hostTicket,
+            out NatIntroductionToken? introduction));
+        Assert.NotNull(introduction);
+        Assert.Equal(created.AttemptId, introduction.AttemptId);
+        Assert.Equal(43, introduction.ConnectionTicket.Length);
+    }
+
+    private static UdpMediatorService CreateMediator(NatMediationProcessor processor, int port) => new(
+        Options.Create(new UdpMediatorOptions
+        {
+            ListenAddress = IPAddress.Loopback.ToString(),
+            Port = port,
+            MaxDatagramsPerPoll = 8,
+            PollIntervalMilliseconds = 1,
+        }),
+        NullLogger<UdpMediatorService>.Instance,
+        processor);
+
     [Fact]
     public async Task FrozenV1EnvelopeIsConsumedOnTheLiteNetSocketWithinAmplificationBudget()
     {