FinalFactory/MeterVault
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Ingestion: derive consumption on ingest, and poll HA in minutes not seconds
ci / build-test (push) Successful in 1m13s
Details

Browse Source 
Live ingestion wrote the raw reading and stopped there. Import, the REST push
endpoint and the meter editor all recompute afterwards; the MQTT/Tasmota/HA
path was the one that did not, so a polled reading landed in `reading` and
every derived figure stayed frozen at the last import. Observed on a
GenerationCounter: 45 readings, 44 consumption rows, generation pinned to the
register value of the last imported reading.

Recompute inline rather than behind a debounce. Normalizing a whole meter is
cheap at metering cadence and a background dirty-set worker is machinery this
does not yet need; the remark on RenormalizeAsync records when it would.

Fixes a latent bug this surfaced in NormalizationService: ExecuteDelete drops
the consumption rows in the database but leaves them in the change tracker, so
a second recompute on the same context threw an identity conflict on
(meter, time, kind). One worker scope ingesting two readings was enough to hit
it. Detach the stale entries after the delete.

Poll interval is now minutes, default 60, replacing seconds/60. A meter answers
"how much this month, what will it cost" — an hourly sample answers that
exactly as well as a per-second one, with far less raw volume (SDD §5.5). The
`pollSeconds` key no longer binds, so existing sources fall back to the 60
default and move from every-60-seconds to hourly, which is the intent. A source
that had deliberately set e.g. 300 seconds also lands on 60 minutes.

Two test cleanups now delete consumption before the meter: live ingestion never
produced any before, so the FK had nothing to trip on.

Claude-Session: https://claude.ai/code/session_01V6joyergfvVLFEizH1hJLd
This commit is contained in:
Florian Schmidt
2026-07-18 18:45:57 +02:00
parent e23df37a3f
commit 62d102c335
9 changed files with 114 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/Integration.Tests/Ingestion/IngestionServiceTests.cs
+38 -7
View File
@@ -1,4 +1,4 @@
using System.Text.Json;
using System.Text.Json;
using MeterVault.Core.Domain;
using MeterVault.Infrastructure.Ingestion;
using MeterVault.Infrastructure.Persistence;
@@ -17,7 +17,7 @@ public sealed class IngestionServiceTests(TimescaleFixture fx)
{
await using var db = fx.CreateContext();
var (meterId, sourceId) = await SetupAsync(db, MeterMode.CumulativeCounter, scale: 0.001, offset: 0);
var service = new IngestionService(db);
var service = NewIngestion(db);
// 1000 raw × 0.001 = 1.0.
Assert.Equal(IngestionOutcome.Written, await service.IngestAsync(sourceId, T0, 1000));
@@ -38,7 +38,7 @@ public sealed class IngestionServiceTests(TimescaleFixture fx)
{
await using var db = fx.CreateContext();
var (meterId, sourceId) = await SetupAsync(db, MeterMode.CumulativeCounter);
var service = new IngestionService(db);
var service = NewIngestion(db);
await service.IngestAsync(sourceId, T0, 500);
var outcome = await service.IngestAsync(sourceId, T0.AddHours(1), 400); // decrease, no event
@@ -54,7 +54,7 @@ public sealed class IngestionServiceTests(TimescaleFixture fx)
{
await using var db = fx.CreateContext();
var (meterId, sourceId) = await SetupAsync(db, MeterMode.CumulativeCounter);
var service = new IngestionService(db);
var service = NewIngestion(db);
// A reset early on explains an early decrease...
await service.IngestAsync(sourceId, T0, 100);
@@ -76,7 +76,7 @@ public sealed class IngestionServiceTests(TimescaleFixture fx)
{
await using var db = fx.CreateContext();
var (meterId, sourceId) = await SetupAsync(db, MeterMode.CumulativeCounter);
var service = new IngestionService(db);
var service = NewIngestion(db);
await service.IngestAsync(sourceId, T0, 500);
db.MeterEvents.Add(new MeterEvent
@@ -103,7 +103,7 @@ public sealed class IngestionServiceTests(TimescaleFixture fx)
var brokerId = await CreateBrokerAsync(db);
var (meterId, _) = await SetupAsync(
db, MeterMode.CumulativeCounter, topic: "tele/plug7/SENSOR", endpointId: brokerId);
var router = new MqttMessageRouter(db, new IngestionService(db), NullLogger<MqttMessageRouter>.Instance);
var router = new MqttMessageRouter(db, NewIngestion(db), NullLogger<MqttMessageRouter>.Instance);
var routed = await router.RouteAsync(
brokerId,
@@ -129,7 +129,7 @@ public sealed class IngestionServiceTests(TimescaleFixture fx)
// separating them.
var (meterId, _) = await SetupAsync(
db, MeterMode.CumulativeCounter, topic: "tele/+/SENSOR", endpointId: brokerB);
var router = new MqttMessageRouter(db, new IngestionService(db), NullLogger<MqttMessageRouter>.Instance);
var router = new MqttMessageRouter(db, NewIngestion(db), NullLogger<MqttMessageRouter>.Instance);
var routed = await router.RouteAsync(
brokerA,
@@ -148,6 +148,36 @@ public sealed class IngestionServiceTests(TimescaleFixture fx)
await CleanupAsync(db, meterId);
}
[Fact]
public async Task Ingesting_a_reading_derives_consumption_without_a_separate_recompute()
{
// Regression: live ingestion used to write only the raw reading, so consumption/generation
// stayed frozen at the last import until something else recomputed the meter.
await using var db = fx.CreateContext();
var (meterId, sourceId) = await SetupAsync(db, MeterMode.CumulativeCounter);
var service = NewIngestion(db);
await service.IngestAsync(sourceId, T0, 1000);
await service.IngestAsync(sourceId, T0.AddHours(1), 1250);
var consumption = await db.Consumption.AsNoTracking()
.Where(c => c.MeterId == meterId)
.OrderBy(c => c.Time)
.ToListAsync();
// The first reading is anchored against the meter's baseline (0), so it contributes 1000;
// what proves the fix is the second reading's 250 delta being there at all.
Assert.Equal(2, consumption.Count);
Assert.Equal(250d, consumption.Single(c => c.Time == T0.AddHours(1)).Amount, 3);
Assert.Equal(1250d, consumption.Sum(c => c.Amount), 3);
await CleanupAsync(db, meterId);
}
private static IngestionService NewIngestion(MeterVaultDbContext db) =>
new(db, new MeterVault.Infrastructure.Normalization.NormalizationService(
db, MeterVault.Core.Normalization.NormalizationEngine.CreateDefault()));
private static async Task<(int MeterId, int SourceId)> SetupAsync(
MeterVaultDbContext db, MeterMode mode, double scale = 1, double offset = 0,
string topic = "tele/x/SENSOR", string? path = "ENERGY.Total", int? endpointId = null)
@@ -198,6 +228,7 @@ public sealed class IngestionServiceTests(TimescaleFixture fx)
{
await db.Readings.Where(r => r.MeterId == meterId).ExecuteDeleteAsync();
await db.MeterEvents.Where(e => e.MeterId == meterId).ExecuteDeleteAsync();
await db.Consumption.Where(c => c.MeterId == meterId).ExecuteDeleteAsync();
await db.Meters.Where(m => m.Id == meterId).ExecuteDeleteAsync();
await db.IngestionEndpoints.Where(e => e.Name.StartsWith("broker-")).ExecuteDeleteAsync();
}