8fe5f4411b
ci / build-test (push) Successful in 1m16s
An audit of this session's commits found several real problems, three of which
lose or expose data. Ordered by severity.
Live recompute was not atomic. RecomputeMeterAsync clears a meter's series with
ExecuteDelete, which commits by itself when no transaction is ambient, and only
then adds the rebuilt rows. Between the two the meter had *no* consumption:
a dashboard read reported zero, and a crash or cancelled request made the loss
permanent, for data the SDD treats as the long-term source of truth (§5.5).
Import and the events API already wrapped their recomputes; live ingestion,
which I added this session, did not. Now shares one transaction, joining an
ambient one rather than nesting.
The MQTT backfill migration counted brokers without regard to is_enabled. One
live broker plus a disabled leftover counted two, declined to backfill, and left
those sources unbound — which under endpoint-scoped routing means silently and
permanently dead. The "two or more is ambiguous" reasoning did not hold there:
the worker only ever connected to enabled endpoints. Corrected by a follow-up
migration rather than an edit, since the original may already have run; it
touches only rows still NULL, so hand-made bindings are safe.
A mapping edited after a dry run committed the *old* staged rows under the
*new* mapping. Readings went to the previous meter while the batch recorded the
current mapping — wrong data, provenance contradicting it, no exception. The
earlier fix re-validated but did not detect staleness. Commit now compares the
mapping against the one the preview was staged under and refuses.
"Test connection" sent a stored token to whatever Base URL was in the dialog.
Encrypting secrets at rest means the UI can decrypt what the operator can no
longer read, so this turned the button into an exfiltration primitive: point it
at any host, the token arrives as a Bearer header. A stored token now only goes
to the origin it was saved for; testing elsewhere requires typing it again.
A source that cannot ingest looked identical to a healthy one. Endpoint-scoped
routing made unbound and mis-bound sources silently dead, while the Sources tab
showed no connector at all and the delete dialog still promised sources would be
"unlinked". Added a Connector column that names the fault, stopped offering
disabled connectors (both workers filter on IsEnabled), and made the delete
warning say ingestion stops.
Virtual meters rendered four zero tiles: they evaluate on read and only
materialize when a cost category references them (§14.1), so summing
consumption is a confident lie about a working meter. They now report nothing
and the page explains why.
Re-importing an overlapping file failed at the database with EF's "An error
occurred while saving the entity changes", naming neither meter nor date — the
diagnosis problem a3af483 set out to fix, via the path its guard could not see.
Checked up front now, bounded by each meter's staged range.
The LXC updater left the service stopped on any failure. set -e plus an
explicit stop means Restart=always does not apply, so an OOM-killed publish or
a brief Gitea outage took MeterVault down until someone noticed. An EXIT trap
restarts the previous build and says so.
Claude-Session: https://claude.ai/code/session_01V6joyergfvVLFEizH1hJLd
140 lines
5.4 KiB
C#
140 lines
5.4 KiB
C#
using MeterVault.Core.Domain;
|
|
using MeterVault.Core.Normalization;
|
|
using MeterVault.Infrastructure.Import;
|
|
using MeterVault.Infrastructure.Normalization;
|
|
using MeterVault.Infrastructure.Persistence;
|
|
using Microsoft.EntityFrameworkCore;
|
|
using static MeterVault.Integration.Tests.Reconciliation.ReconciliationSupport;
|
|
|
|
namespace MeterVault.Integration.Tests;
|
|
|
|
/// <summary>
|
|
/// End-to-end persistence: stage the water CSV → commit as a batch → normalized consumption lands
|
|
/// in the hypertable → revert removes everything and rebases consumption (SDD §6.3, FR-6).
|
|
/// </summary>
|
|
[Collection("Timescale")]
|
|
public sealed class ImportRoundTripTests(TimescaleFixture fx)
|
|
{
|
|
[Fact]
|
|
public async Task Commit_persists_consumption_and_revert_removes_it()
|
|
{
|
|
await using var db = fx.CreateContext();
|
|
await DatabaseSeeder.SeedAsync(db);
|
|
|
|
var waterType = await db.EnergyTypes.FirstAsync(t => t.Key == "water");
|
|
var meter = new Meter
|
|
{
|
|
Name = "Zähler Wasser (round-trip test)",
|
|
EnergyTypeId = waterType.Id,
|
|
Mode = MeterMode.CumulativeCounter,
|
|
Unit = "m3",
|
|
InitialBaseline = 820,
|
|
};
|
|
db.Meters.Add(meter);
|
|
await db.SaveChangesAsync();
|
|
|
|
var profile = new MappingProfile
|
|
{
|
|
Name = "test-water",
|
|
DateColumn = 0,
|
|
DateKind = DateKind.MonthName,
|
|
FirstDataRowIndex = 1,
|
|
DetectCumulativeSwaps = true,
|
|
Columns =
|
|
[
|
|
new ColumnMapping
|
|
{
|
|
Index = 1,
|
|
Role = MappingRole.Reading,
|
|
MeterId = meter.Id,
|
|
Unit = "m3",
|
|
SwapConsumptionColumn = 2,
|
|
},
|
|
],
|
|
};
|
|
|
|
StagedImport staged;
|
|
using (var reader = new StreamReader(FixturePath(Water)))
|
|
{
|
|
staged = new CsvImporter().Stage(profile, reader);
|
|
}
|
|
|
|
var normalization = new NormalizationService(db, NormalizationEngine.CreateDefault());
|
|
var service = new ImportService(db, normalization);
|
|
|
|
// Commit.
|
|
var batchId = await service.CommitAsync(staged, sourceName: "Wasser.csv", mappingJson: null);
|
|
|
|
Assert.True(await db.Readings.AnyAsync(r => r.MeterId == meter.Id));
|
|
Assert.True(await db.MeterEvents.AnyAsync(e => e.MeterId == meter.Id && e.EventType == MeterEventType.MeterSwap));
|
|
|
|
var maerz = await db.Consumption.SingleAsync(c =>
|
|
c.MeterId == meter.Id && c.Time == new DateTimeOffset(2023, 3, 1, 0, 0, 0, TimeSpan.Zero));
|
|
Assert.Equal(12d, maerz.Amount, 3);
|
|
|
|
// Revert.
|
|
await service.RevertAsync(batchId);
|
|
|
|
Assert.False(await db.Readings.AnyAsync(r => r.MeterId == meter.Id));
|
|
Assert.False(await db.MeterEvents.AnyAsync(e => e.MeterId == meter.Id));
|
|
Assert.False(await db.Consumption.AnyAsync(c => c.MeterId == meter.Id));
|
|
var batch = await db.ImportBatches.SingleAsync(b => b.Id == batchId);
|
|
Assert.NotNull(batch.RevertedAt);
|
|
|
|
// Cleanup so the shared container stays tidy for other tests. ExecuteDelete bypasses the
|
|
// change tracker (which holds stale entries after the revert's set-based deletes).
|
|
await db.Meters.Where(m => m.Id == meter.Id).ExecuteDeleteAsync();
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Re_importing_the_same_file_is_refused_by_meter_and_date()
|
|
{
|
|
// The commonest real duplicate: import a file, then import an overlapping one. The in-batch
|
|
// guard cannot see it (that set is internally unique), so left to the database it surfaced as
|
|
// EF's "An error occurred while saving the entity changes", naming neither meter nor date.
|
|
await using var db = fx.CreateContext();
|
|
await DatabaseSeeder.SeedAsync(db);
|
|
|
|
var type = await db.EnergyTypes.FirstAsync(t => t.Key == "electricity");
|
|
var meter = new Meter
|
|
{
|
|
Name = $"reimport-{Guid.NewGuid():N}",
|
|
EnergyTypeId = type.Id,
|
|
Mode = MeterMode.CumulativeCounter,
|
|
Unit = "kWh",
|
|
};
|
|
db.Meters.Add(meter);
|
|
await db.SaveChangesAsync();
|
|
|
|
var service = new ImportService(db, new NormalizationService(db, NormalizationEngine.CreateDefault()));
|
|
|
|
StagedImport Stage() => Staged(meter.Id, new DateTimeOffset(2024, 5, 1, 0, 0, 0, TimeSpan.Zero), 1200);
|
|
|
|
await service.CommitAsync(Stage(), "first.csv", mappingJson: null);
|
|
|
|
var error = await Assert.ThrowsAsync<InvalidOperationException>(
|
|
() => service.CommitAsync(Stage(), "again.csv", mappingJson: null));
|
|
|
|
Assert.Contains("already exist", error.Message, StringComparison.OrdinalIgnoreCase);
|
|
Assert.Contains("2024-05-01", error.Message, StringComparison.Ordinal);
|
|
Assert.Contains($"meter {meter.Id}", error.Message, StringComparison.Ordinal);
|
|
|
|
await db.Consumption.Where(c => c.MeterId == meter.Id).ExecuteDeleteAsync();
|
|
await db.Readings.Where(r => r.MeterId == meter.Id).ExecuteDeleteAsync();
|
|
await db.Meters.Where(m => m.Id == meter.Id).ExecuteDeleteAsync();
|
|
}
|
|
|
|
private static StagedImport Staged(int meterId, DateTimeOffset time, double value)
|
|
{
|
|
var staged = new StagedImport();
|
|
staged.Readings.Add(new Reading
|
|
{
|
|
MeterId = meterId,
|
|
Time = time,
|
|
Value = value,
|
|
Quality = ReadingQuality.Imported,
|
|
});
|
|
return staged;
|
|
}
|
|
}
|