M2: German CSV importer + reconciliation of all 4 fixtures
- German-dialect scalar parsers in Core (GermanNumber/Money/Date, ValueCell): decimal comma, thousands dot, unit suffixes, € currency, both date shapes. - Declarative MappingProfile + RowClassifier (skip summary/blank/all-zero rows) + CsvImporter (CsvHelper) staging readings/events/manual-costs, with auto swap detection on register decreases and month-end anchoring for interleaved oil dates. - Four built-in ReferenceProfiles (Strom/Wasser/Heizöl/Kosten). - ImportService: commit as revertible import_batch + wholesale consumption recompute per affected meter (NormalizationService/MeterConfigFactory), revert by batch. - Reconciliation tests: all 4 CSVs match the sheet's own columns within tolerance (electricity 5 meters + Netz Einsparung, water swap→12, oil tank incl. deliveries + burner hours, cost category totals). Commit/revert round-trip verified on Timescale. 69 tests green (53 Core + 16 integration). Known follow-up (polish): historical imports can contend with the 30-day compression policy's background job; tests pause it. Consider retry-on-deadlock or deferred compression for large historical imports in production. Claude-Session: https://claude.ai/code/session_01WujdMtMJPbxDpDnMeK22rr
This commit is contained in:
@@ -0,0 +1,88 @@
|
||||
using System.Text.Json;
|
||||
using MeterVault.Core.Domain;
|
||||
using MeterVault.Core.Normalization;
|
||||
|
||||
namespace MeterVault.Infrastructure.Normalization;
|
||||
|
||||
/// <summary>
|
||||
/// Projects a persisted <see cref="Meter"/> (and optional <see cref="Tank"/>) into the
|
||||
/// infrastructure-free <see cref="MeterConfig"/> the normalization engine consumes.
|
||||
/// </summary>
|
||||
public static class MeterConfigFactory
|
||||
{
|
||||
public static MeterConfig FromMeter(Meter meter, Tank? tank)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(meter);
|
||||
|
||||
return new MeterConfig
|
||||
{
|
||||
MeterId = meter.Id,
|
||||
Mode = meter.Mode,
|
||||
Unit = meter.Unit,
|
||||
InitialBaseline = meter.InitialBaseline,
|
||||
Tank = tank is null ? null : new TankConfig
|
||||
{
|
||||
Capacity = tank.Capacity,
|
||||
RateMode = tank.RateMode,
|
||||
FixedRate = tank.FixedRate,
|
||||
Calibration = ParseCalibration(tank.Calibration),
|
||||
},
|
||||
Virtual = ParseVirtual(meter),
|
||||
};
|
||||
}
|
||||
|
||||
private static CalibrationCurve? ParseCalibration(string? json)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(json))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
using var doc = JsonDocument.Parse(json);
|
||||
var root = doc.RootElement;
|
||||
if (!root.TryGetProperty("volumePerUnit", out var perUnit))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var offset = root.TryGetProperty("offset", out var off) ? off.GetDouble() : 0d;
|
||||
return new CalibrationCurve(perUnit.GetDouble(), offset);
|
||||
}
|
||||
catch (JsonException)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private static VirtualSpec? ParseVirtual(Meter meter)
|
||||
{
|
||||
if (meter.Mode != MeterMode.Virtual || string.IsNullOrWhiteSpace(meter.Meta))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
using var doc = JsonDocument.Parse(meter.Meta);
|
||||
var root = doc.RootElement;
|
||||
if (!root.TryGetProperty("expression", out var expression))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var referenced = new List<int>();
|
||||
if (root.TryGetProperty("referencedMeterIds", out var ids) && ids.ValueKind == JsonValueKind.Array)
|
||||
{
|
||||
referenced.AddRange(ids.EnumerateArray().Select(e => e.GetInt32()));
|
||||
}
|
||||
|
||||
return new VirtualSpec { Expression = expression.GetString() ?? string.Empty, ReferencedMeterIds = referenced };
|
||||
}
|
||||
catch (JsonException)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
using MeterVault.Core.Domain;
|
||||
using MeterVault.Core.Normalization;
|
||||
using MeterVault.Infrastructure.Persistence;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
|
||||
namespace MeterVault.Infrastructure.Normalization;
|
||||
|
||||
/// <summary>
|
||||
/// Bridges persisted readings/events to the pure engine: loads a meter's inputs, recomputes its
|
||||
/// consumption wholesale (consumption is a pure function of readings + events), and replaces the
|
||||
/// stored rows. Virtual meters are skipped here — they are computed on read (SDD §14.1).
|
||||
/// </summary>
|
||||
public sealed class NormalizationService(MeterVaultDbContext db, INormalizationEngine engine)
|
||||
{
|
||||
private readonly MeterVaultDbContext _db = db;
|
||||
private readonly INormalizationEngine _engine = engine;
|
||||
|
||||
/// <summary>
|
||||
/// Recomputes and replaces the consumption series for one meter from all its current readings
|
||||
/// and events. Tags new rows with <paramref name="batchId"/> for provenance. Does not save.
|
||||
/// </summary>
|
||||
public async Task RecomputeMeterAsync(int meterId, int? batchId, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var meter = await _db.Meters.FirstOrDefaultAsync(m => m.Id == meterId, cancellationToken).ConfigureAwait(false);
|
||||
if (meter is null || meter.Mode == MeterMode.Virtual)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var tank = await _db.Tanks.FirstOrDefaultAsync(t => t.MeterId == meterId, cancellationToken).ConfigureAwait(false);
|
||||
var config = MeterConfigFactory.FromMeter(meter, tank);
|
||||
|
||||
var readings = await _db.Readings
|
||||
.Where(r => r.MeterId == meterId)
|
||||
.OrderBy(r => r.Time)
|
||||
.ToListAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var events = await _db.MeterEvents
|
||||
.Where(e => e.MeterId == meterId)
|
||||
.OrderBy(e => e.Time)
|
||||
.ToListAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var context = new NormalizationContext { Meter = config, Readings = readings, Events = events };
|
||||
var consumption = _engine.Normalize(context);
|
||||
|
||||
await _db.Consumption.Where(c => c.MeterId == meterId)
|
||||
.ExecuteDeleteAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
foreach (var row in consumption)
|
||||
{
|
||||
row.ImportBatchId = batchId;
|
||||
_db.Consumption.Add(row);
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user