M3: live ingestion (MQTT/Tasmota + Home Assistant)

- PayloadExtractor: dot-path value/time extraction (Tasmota ENERGY.Total, bare scalars). - MqttTopicMatcher: standard +/# wildcard matching. - IngestionService: scale/offset, idempotent upsert on (meter_id, time), and a spurious- decrease guard for monotonic registers (allowed only with a reset/swap event) + source last-seen status. - MqttMessageRouter + MqttIngestionWorker (MQTTnet 5): per-endpoint persistent connections, topic subscription, graceful degradation; secrets resolved by env-var reference. - Home Assistant: HaStateClient (REST /api/states parse) + HomeAssistantWorker polling on each source's interval. HA-via-MQTT also works through the MQTT path. - Ingestion workers gated by MeterVault:EnableLiveIngestion (off in tests). 85 tests green (53 Core + 32 integration): Tasmota payload → reading verified end to end. Follow-up (polish): HA WebSocket push (state_changed) as an alternative to REST poll; source-topic index caching in the router. Claude-Session: https://claude.ai/code/session_01WujdMtMJPbxDpDnMeK22rr
2026-07-13 11:45:20 +02:00
parent 5977c81002
commit 4b0cad67df
17 changed files with 1060 additions and 0 deletions
@@ -0,0 +1,134 @@
+using System.Text.Json;
+using MeterVault.Core.Domain;
+using MeterVault.Infrastructure.Ingestion;
+using MeterVault.Infrastructure.Persistence;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.Extensions.Logging.Abstractions;
+
+namespace MeterVault.Integration.Tests.Ingestion;
+
+[Collection("Timescale")]
+public sealed class IngestionServiceTests(TimescaleFixture fx)
+{
+    private static readonly DateTimeOffset T0 = new(2024, 1, 1, 0, 0, 0, TimeSpan.Zero);
+
+    [Fact]
+    public async Task Writes_and_updates_idempotently_with_scale_and_offset()
+    {
+        await using var db = fx.CreateContext();
+        var (meterId, sourceId) = await SetupAsync(db, MeterMode.CumulativeCounter, scale: 0.001, offset: 0);
+        var service = new IngestionService(db);
+
+        // 1000 raw × 0.001 = 1.0.
+        Assert.Equal(IngestionOutcome.Written, await service.IngestAsync(sourceId, T0, 1000));
+        Assert.Equal(IngestionOutcome.Updated, await service.IngestAsync(sourceId, T0, 2000)); // same time → update
+
+        var reading = await db.Readings.SingleAsync(r => r.MeterId == meterId && r.Time == T0);
+        Assert.Equal(2.0, reading.Value, 6);
+
+        var source = await db.MeterSources.SingleAsync(s => s.Id == sourceId);
+        Assert.Equal("ok", source.LastStatus);
+        Assert.Equal(2.0, source.LastValue!.Value, 6);
+
+        await CleanupAsync(db, meterId);
+    }
+
+    [Fact]
+    public async Task Rejects_spurious_decrease_on_a_cumulative_register()
+    {
+        await using var db = fx.CreateContext();
+        var (meterId, sourceId) = await SetupAsync(db, MeterMode.CumulativeCounter);
+        var service = new IngestionService(db);
+
+        await service.IngestAsync(sourceId, T0, 500);
+        var outcome = await service.IngestAsync(sourceId, T0.AddHours(1), 400); // decrease, no event
+
+        Assert.Equal(IngestionOutcome.RejectedDecrease, outcome);
+        Assert.False(await db.Readings.AnyAsync(r => r.MeterId == meterId && r.Time == T0.AddHours(1)));
+
+        await CleanupAsync(db, meterId);
+    }
+
+    [Fact]
+    public async Task Allows_decrease_when_a_swap_event_explains_it()
+    {
+        await using var db = fx.CreateContext();
+        var (meterId, sourceId) = await SetupAsync(db, MeterMode.CumulativeCounter);
+        var service = new IngestionService(db);
+
+        await service.IngestAsync(sourceId, T0, 500);
+        db.MeterEvents.Add(new MeterEvent
+        {
+            MeterId = meterId,
+            Time = T0.AddMinutes(30),
+            EventType = MeterEventType.MeterSwap,
+            PrevValue = 500,
+            NewValue = 0,
+        });
+        await db.SaveChangesAsync();
+
+        var outcome = await service.IngestAsync(sourceId, T0.AddHours(1), 20); // new meter reads low
+
+        Assert.Equal(IngestionOutcome.Written, outcome);
+
+        await CleanupAsync(db, meterId);
+    }
+
+    [Fact]
+    public async Task Mqtt_router_ingests_a_tasmota_payload()
+    {
+        await using var db = fx.CreateContext();
+        var (meterId, _) = await SetupAsync(db, MeterMode.CumulativeCounter, topic: "tele/plug7/SENSOR");
+        var router = new MqttMessageRouter(db, new IngestionService(db), NullLogger<MqttMessageRouter>.Instance);
+
+        var routed = await router.RouteAsync(
+            "tele/plug7/SENSOR",
+            """{"Time":"2024-03-01T10:00:00","ENERGY":{"Total":8421.0}}""");
+
+        Assert.Equal(1, routed);
+        var reading = await db.Readings.SingleAsync(r => r.MeterId == meterId);
+        Assert.Equal(8421.0, reading.Value, 3);
+        Assert.Equal(new DateTimeOffset(2024, 3, 1, 10, 0, 0, TimeSpan.Zero), reading.Time);
+
+        await CleanupAsync(db, meterId);
+    }
+
+    private static async Task<(int MeterId, int SourceId)> SetupAsync(
+        MeterVaultDbContext db, MeterMode mode, double scale = 1, double offset = 0,
+        string topic = "tele/x/SENSOR", string? path = "ENERGY.Total")
+    {
+        await DatabaseSeeder.SeedAsync(db);
+        var type = await db.EnergyTypes.FirstAsync(t => t.Key == "electricity");
+
+        var meter = new Meter
+        {
+            Name = $"ingest-{Guid.NewGuid():N}",
+            EnergyTypeId = type.Id,
+            Mode = mode,
+            Unit = "kWh",
+        };
+        db.Meters.Add(meter);
+        await db.SaveChangesAsync();
+
+        var source = new MeterSource
+        {
+            MeterId = meter.Id,
+            SourceType = SourceType.Tasmota,
+            ValueKind = SourceValueKind.Register,
+            Scale = scale,
+            Offset = offset,
+            Config = JsonSerializer.Serialize(new { topic, path }),
+        };
+        db.MeterSources.Add(source);
+        await db.SaveChangesAsync();
+
+        return (meter.Id, source.Id);
+    }
+
+    private static async Task CleanupAsync(MeterVaultDbContext db, int meterId)
+    {
+        await db.Readings.Where(r => r.MeterId == meterId).ExecuteDeleteAsync();
+        await db.MeterEvents.Where(e => e.MeterId == meterId).ExecuteDeleteAsync();
+        await db.Meters.Where(m => m.Id == meterId).ExecuteDeleteAsync();
+    }
+}
@@ -0,0 +1,73 @@
+using System.Text.Json;
+using MeterVault.Infrastructure.Ingestion;
+
+namespace MeterVault.Integration.Tests.Ingestion;
+
+/// <summary>Pure ingestion helpers — no broker or database needed.</summary>
+public sealed class PayloadExtractionTests
+{
+    private const string TasmotaSensor =
+        """{"Time":"2024-01-15T12:30:00","ENERGY":{"Total":1234.56,"Today":1.2,"Power":50}}""";
+
+    [Fact]
+    public void Extracts_tasmota_energy_total()
+    {
+        Assert.True(PayloadExtractor.TryExtractValue(TasmotaSensor, "ENERGY.Total", out var value));
+        Assert.Equal(1234.56, value, 3);
+    }
+
+    [Fact]
+    public void Extracts_tasmota_time()
+    {
+        Assert.True(PayloadExtractor.TryExtractTime(TasmotaSensor, "Time", out var time));
+        Assert.Equal(new DateTimeOffset(2024, 1, 15, 12, 30, 0, TimeSpan.Zero), time);
+    }
+
+    [Fact]
+    public void Extracts_bare_scalar_when_no_path()
+    {
+        Assert.True(PayloadExtractor.TryExtractValue("42.7", null, out var value));
+        Assert.Equal(42.7, value, 3);
+    }
+
+    [Fact]
+    public void Missing_path_yields_false()
+    {
+        Assert.False(PayloadExtractor.TryExtractValue(TasmotaSensor, "ENERGY.Nope", out _));
+    }
+
+    [Theory]
+    [InlineData("tele/+/SENSOR", "tele/plug1/SENSOR", true)]
+    [InlineData("tele/plug1/SENSOR", "tele/plug1/SENSOR", true)]
+    [InlineData("tele/#", "tele/plug1/SENSOR", true)]
+    [InlineData("tele/+/SENSOR", "tele/plug1/STATE", false)]
+    [InlineData("tele/+/SENSOR", "tele/a/b/SENSOR", false)]
+    [InlineData("home/#", "office/x", false)]
+    public void Topic_matcher_follows_mqtt_wildcards(string filter, string topic, bool expected)
+    {
+        Assert.Equal(expected, MqttTopicMatcher.Matches(filter, topic));
+    }
+
+    [Fact]
+    public void Ha_state_client_parses_numeric_state()
+    {
+        using var doc = JsonDocument.Parse(
+            """{"entity_id":"sensor.p","state":"987.6","attributes":{"unit":"kWh"},"last_updated":"2024-02-01T08:00:00+00:00"}""");
+
+        var state = HaStateClient.ParseState(doc, attribute: null);
+
+        Assert.NotNull(state);
+        Assert.Equal(987.6, state!.Value.Value, 3);
+        Assert.Equal(new DateTimeOffset(2024, 2, 1, 8, 0, 0, TimeSpan.Zero), state.Value.Time);
+    }
+
+    [Fact]
+    public void Ha_state_client_parses_attribute_and_rejects_unavailable()
+    {
+        using var doc = JsonDocument.Parse(
+            """{"entity_id":"sensor.p","state":"unavailable","attributes":{"power":123.4}}""");
+
+        Assert.Null(HaStateClient.ParseState(doc, attribute: null));       // non-numeric state
+        Assert.Equal(123.4, HaStateClient.ParseState(doc, "power")!.Value.Value, 3);
+    }
+}
@@ -14,5 +14,6 @@ public sealed class MeterVaultAppFactory(string connectionString) : WebApplicati
        builder.UseEnvironment("Testing");
        builder.UseSetting("ConnectionStrings:Default", connectionString);
        builder.UseSetting("MeterVault:RunMigrationsAtStartup", "false");
+        builder.UseSetting("MeterVault:EnableLiveIngestion", "false");
    }
 }