M3: live ingestion (MQTT/Tasmota + Home Assistant)

- PayloadExtractor: dot-path value/time extraction (Tasmota ENERGY.Total, bare scalars). - MqttTopicMatcher: standard +/# wildcard matching. - IngestionService: scale/offset, idempotent upsert on (meter_id, time), and a spurious- decrease guard for monotonic registers (allowed only with a reset/swap event) + source last-seen status. - MqttMessageRouter + MqttIngestionWorker (MQTTnet 5): per-endpoint persistent connections, topic subscription, graceful degradation; secrets resolved by env-var reference. - Home Assistant: HaStateClient (REST /api/states parse) + HomeAssistantWorker polling on each source's interval. HA-via-MQTT also works through the MQTT path. - Ingestion workers gated by MeterVault:EnableLiveIngestion (off in tests). 85 tests green (53 Core + 32 integration): Tasmota payload → reading verified end to end. Follow-up (polish): HA WebSocket push (state_changed) as an alternative to REST poll; source-topic index caching in the router. Claude-Session: https://claude.ai/code/session_01WujdMtMJPbxDpDnMeK22rr
2026-07-13 11:45:20 +02:00
parent 5977c81002
commit 4b0cad67df
17 changed files with 1060 additions and 0 deletions
@@ -0,0 +1,134 @@
+using System.Text.Json;
+using MeterVault.Core.Domain;
+using MeterVault.Infrastructure.Ingestion;
+using MeterVault.Infrastructure.Persistence;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.Extensions.Logging.Abstractions;
+
+namespace MeterVault.Integration.Tests.Ingestion;
+
+[Collection("Timescale")]
+public sealed class IngestionServiceTests(TimescaleFixture fx)
+{
+    private static readonly DateTimeOffset T0 = new(2024, 1, 1, 0, 0, 0, TimeSpan.Zero);
+
+    [Fact]
+    public async Task Writes_and_updates_idempotently_with_scale_and_offset()
+    {
+        await using var db = fx.CreateContext();
+        var (meterId, sourceId) = await SetupAsync(db, MeterMode.CumulativeCounter, scale: 0.001, offset: 0);
+        var service = new IngestionService(db);
+
+        // 1000 raw × 0.001 = 1.0.
+        Assert.Equal(IngestionOutcome.Written, await service.IngestAsync(sourceId, T0, 1000));
+        Assert.Equal(IngestionOutcome.Updated, await service.IngestAsync(sourceId, T0, 2000)); // same time → update
+
+        var reading = await db.Readings.SingleAsync(r => r.MeterId == meterId && r.Time == T0);
+        Assert.Equal(2.0, reading.Value, 6);
+
+        var source = await db.MeterSources.SingleAsync(s => s.Id == sourceId);
+        Assert.Equal("ok", source.LastStatus);
+        Assert.Equal(2.0, source.LastValue!.Value, 6);
+
+        await CleanupAsync(db, meterId);
+    }
+
+    [Fact]
+    public async Task Rejects_spurious_decrease_on_a_cumulative_register()
+    {
+        await using var db = fx.CreateContext();
+        var (meterId, sourceId) = await SetupAsync(db, MeterMode.CumulativeCounter);
+        var service = new IngestionService(db);
+
+        await service.IngestAsync(sourceId, T0, 500);
+        var outcome = await service.IngestAsync(sourceId, T0.AddHours(1), 400); // decrease, no event
+
+        Assert.Equal(IngestionOutcome.RejectedDecrease, outcome);
+        Assert.False(await db.Readings.AnyAsync(r => r.MeterId == meterId && r.Time == T0.AddHours(1)));
+
+        await CleanupAsync(db, meterId);
+    }
+
+    [Fact]
+    public async Task Allows_decrease_when_a_swap_event_explains_it()
+    {
+        await using var db = fx.CreateContext();
+        var (meterId, sourceId) = await SetupAsync(db, MeterMode.CumulativeCounter);
+        var service = new IngestionService(db);
+
+        await service.IngestAsync(sourceId, T0, 500);
+        db.MeterEvents.Add(new MeterEvent
+        {
+            MeterId = meterId,
+            Time = T0.AddMinutes(30),
+            EventType = MeterEventType.MeterSwap,
+            PrevValue = 500,
+            NewValue = 0,
+        });
+        await db.SaveChangesAsync();
+
+        var outcome = await service.IngestAsync(sourceId, T0.AddHours(1), 20); // new meter reads low
+
+        Assert.Equal(IngestionOutcome.Written, outcome);
+
+        await CleanupAsync(db, meterId);
+    }
+
+    [Fact]
+    public async Task Mqtt_router_ingests_a_tasmota_payload()
+    {
+        await using var db = fx.CreateContext();
+        var (meterId, _) = await SetupAsync(db, MeterMode.CumulativeCounter, topic: "tele/plug7/SENSOR");
+        var router = new MqttMessageRouter(db, new IngestionService(db), NullLogger<MqttMessageRouter>.Instance);
+
+        var routed = await router.RouteAsync(
+            "tele/plug7/SENSOR",
+            """{"Time":"2024-03-01T10:00:00","ENERGY":{"Total":8421.0}}""");
+
+        Assert.Equal(1, routed);
+        var reading = await db.Readings.SingleAsync(r => r.MeterId == meterId);
+        Assert.Equal(8421.0, reading.Value, 3);
+        Assert.Equal(new DateTimeOffset(2024, 3, 1, 10, 0, 0, TimeSpan.Zero), reading.Time);
+
+        await CleanupAsync(db, meterId);
+    }
+
+    private static async Task<(int MeterId, int SourceId)> SetupAsync(
+        MeterVaultDbContext db, MeterMode mode, double scale = 1, double offset = 0,
+        string topic = "tele/x/SENSOR", string? path = "ENERGY.Total")
+    {
+        await DatabaseSeeder.SeedAsync(db);
+        var type = await db.EnergyTypes.FirstAsync(t => t.Key == "electricity");
+
+        var meter = new Meter
+        {
+            Name = $"ingest-{Guid.NewGuid():N}",
+            EnergyTypeId = type.Id,
+            Mode = mode,
+            Unit = "kWh",
+        };
+        db.Meters.Add(meter);
+        await db.SaveChangesAsync();
+
+        var source = new MeterSource
+        {
+            MeterId = meter.Id,
+            SourceType = SourceType.Tasmota,
+            ValueKind = SourceValueKind.Register,
+            Scale = scale,
+            Offset = offset,
+            Config = JsonSerializer.Serialize(new { topic, path }),
+        };
+        db.MeterSources.Add(source);
+        await db.SaveChangesAsync();
+
+        return (meter.Id, source.Id);
+    }
+
+    private static async Task CleanupAsync(MeterVaultDbContext db, int meterId)
+    {
+        await db.Readings.Where(r => r.MeterId == meterId).ExecuteDeleteAsync();
+        await db.MeterEvents.Where(e => e.MeterId == meterId).ExecuteDeleteAsync();
+        await db.Meters.Where(m => m.Id == meterId).ExecuteDeleteAsync();
+    }
+}