M2: German CSV importer + reconciliation of all 4 fixtures
- German-dialect scalar parsers in Core (GermanNumber/Money/Date, ValueCell): decimal comma, thousands dot, unit suffixes, € currency, both date shapes. - Declarative MappingProfile + RowClassifier (skip summary/blank/all-zero rows) + CsvImporter (CsvHelper) staging readings/events/manual-costs, with auto swap detection on register decreases and month-end anchoring for interleaved oil dates. - Four built-in ReferenceProfiles (Strom/Wasser/Heizöl/Kosten). - ImportService: commit as revertible import_batch + wholesale consumption recompute per affected meter (NormalizationService/MeterConfigFactory), revert by batch. - Reconciliation tests: all 4 CSVs match the sheet's own columns within tolerance (electricity 5 meters + Netz Einsparung, water swap→12, oil tank incl. deliveries + burner hours, cost category totals). Commit/revert round-trip verified on Timescale. 69 tests green (53 Core + 16 integration). Known follow-up (polish): historical imports can contend with the 30-day compression policy's background job; tests pause it. Consider retry-on-deadlock or deferred compression for large historical imports in production. Claude-Session: https://claude.ai/code/session_01WujdMtMJPbxDpDnMeK22rr
This commit is contained in:
@@ -21,7 +21,10 @@ public sealed class ConsumableBalanceNormalizer : IMeterNormalizer
|
||||
var calibration = context.Meter.Tank?.Calibration;
|
||||
var events = context.Events
|
||||
.Where(e => e.EventType is MeterEventType.Delivery or MeterEventType.TankLevel)
|
||||
// At the same instant a delivery is applied before the level is read, otherwise a
|
||||
// refill reads as a negative draw and gets clamped away.
|
||||
.OrderBy(e => e.Time)
|
||||
.ThenBy(e => e.EventType == MeterEventType.Delivery ? 0 : 1)
|
||||
.ToList();
|
||||
|
||||
double? lastLevelVolume = null;
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
using System.Globalization;
|
||||
|
||||
namespace MeterVault.Core.Parsing;
|
||||
|
||||
/// <summary>
|
||||
/// Parses the two date shapes in the reference data (SDD Appendix A): German month tables
|
||||
/// (<c>September 2022</c>) and event rows (<c>10.06.1997</c>, DD.MM.YYYY). Month tables are
|
||||
/// anchored to the first of the month.
|
||||
/// </summary>
|
||||
public static class GermanDate
|
||||
{
|
||||
private static readonly Dictionary<string, int> MonthNames = new(StringComparer.OrdinalIgnoreCase)
|
||||
{
|
||||
["Januar"] = 1,
|
||||
["Februar"] = 2,
|
||||
["März"] = 3,
|
||||
["Maerz"] = 3,
|
||||
["April"] = 4,
|
||||
["Mai"] = 5,
|
||||
["Juni"] = 6,
|
||||
["Juli"] = 7,
|
||||
["August"] = 8,
|
||||
["September"] = 9,
|
||||
["Oktober"] = 10,
|
||||
["November"] = 11,
|
||||
["Dezember"] = 12,
|
||||
};
|
||||
|
||||
public static bool TryParseMonth(string? raw, out DateOnly firstOfMonth)
|
||||
{
|
||||
firstOfMonth = default;
|
||||
if (string.IsNullOrWhiteSpace(raw))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
var parts = raw.Trim().Split(' ', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
|
||||
if (parts.Length != 2
|
||||
|| !MonthNames.TryGetValue(parts[0], out var month)
|
||||
|| !int.TryParse(parts[1], NumberStyles.Integer, CultureInfo.InvariantCulture, out var year))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
firstOfMonth = new DateOnly(year, month, 1);
|
||||
return true;
|
||||
}
|
||||
|
||||
public static bool TryParseDay(string? raw, out DateOnly date) =>
|
||||
DateOnly.TryParseExact(raw?.Trim(), "dd.MM.yyyy", CultureInfo.InvariantCulture, DateTimeStyles.None, out date);
|
||||
|
||||
/// <summary>Tries day format first, then month-name format (SDD: oil switches mid-file).</summary>
|
||||
public static bool TryParse(string? raw, out DateOnly date)
|
||||
{
|
||||
if (TryParseDay(raw, out date))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
if (TryParseMonth(raw, out date))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
date = default;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
using System.Globalization;
|
||||
|
||||
namespace MeterVault.Core.Parsing;
|
||||
|
||||
/// <summary>
|
||||
/// Parses German currency cells (SDD Appendix A): trailing € with a space, decimal comma,
|
||||
/// thousands dot (<c>120,00 €</c> → (120.00, "EUR"), <c>2.940,19 €</c> → (2940.19, "EUR"),
|
||||
/// <c>70 €</c> → (70, "EUR")). Amount is a decimal; currency defaults to EUR.
|
||||
/// </summary>
|
||||
public static class GermanMoney
|
||||
{
|
||||
public static bool TryParse(ReadOnlySpan<char> raw, out decimal amount, out string currency)
|
||||
{
|
||||
amount = 0m;
|
||||
currency = "EUR";
|
||||
raw = raw.Trim();
|
||||
if (raw.IsEmpty)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
var sign = 1;
|
||||
var index = 0;
|
||||
if (raw[0] is '-' or '+')
|
||||
{
|
||||
sign = raw[0] == '-' ? -1 : 1;
|
||||
index = 1;
|
||||
}
|
||||
|
||||
var start = index;
|
||||
while (index < raw.Length && (char.IsDigit(raw[index]) || raw[index] is '.' or ','))
|
||||
{
|
||||
index++;
|
||||
}
|
||||
|
||||
var numeric = raw[start..index];
|
||||
if (numeric.IsEmpty)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
Span<char> buffer = stackalloc char[numeric.Length];
|
||||
var length = 0;
|
||||
foreach (var c in numeric)
|
||||
{
|
||||
if (c == '.')
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
buffer[length++] = c == ',' ? '.' : c;
|
||||
}
|
||||
|
||||
if (!decimal.TryParse(buffer[..length], NumberStyles.Float, CultureInfo.InvariantCulture, out var parsed))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
amount = sign * parsed;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
using System.Globalization;
|
||||
|
||||
namespace MeterVault.Core.Parsing;
|
||||
|
||||
/// <summary>
|
||||
/// Parses the reference spreadsheet's German numeric dialect (SDD Appendix A): decimal comma,
|
||||
/// thousands dot, optional sign, and an optional trailing unit/currency suffix which is ignored
|
||||
/// (<c>411kWh</c> → 411, <c>2.940,19 €</c> → 2940.19, <c>-90kWh</c> → -90, <c>180,8244706</c> →
|
||||
/// 180.82…). Done by explicit transform rather than a de-DE NumberStyles parse because of the
|
||||
/// mixed unit suffixes.
|
||||
/// </summary>
|
||||
public static class GermanNumber
|
||||
{
|
||||
public static bool TryParse(ReadOnlySpan<char> raw, out double value)
|
||||
{
|
||||
value = 0;
|
||||
raw = raw.Trim();
|
||||
if (raw.IsEmpty)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
var sign = 1;
|
||||
var index = 0;
|
||||
if (raw[0] is '-' or '+')
|
||||
{
|
||||
sign = raw[0] == '-' ? -1 : 1;
|
||||
index = 1;
|
||||
}
|
||||
|
||||
var start = index;
|
||||
while (index < raw.Length && (char.IsDigit(raw[index]) || raw[index] is '.' or ','))
|
||||
{
|
||||
index++;
|
||||
}
|
||||
|
||||
var numeric = raw[start..index];
|
||||
if (numeric.IsEmpty)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
// Strip thousands dots, turn the decimal comma into a point, then parse invariant.
|
||||
Span<char> buffer = stackalloc char[numeric.Length];
|
||||
var length = 0;
|
||||
foreach (var c in numeric)
|
||||
{
|
||||
if (c == '.')
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
buffer[length++] = c == ',' ? '.' : c;
|
||||
}
|
||||
|
||||
if (!double.TryParse(buffer[..length], NumberStyles.Float, CultureInfo.InvariantCulture, out var parsed))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
value = sign * parsed;
|
||||
return true;
|
||||
}
|
||||
|
||||
public static double? Parse(ReadOnlySpan<char> raw) => TryParse(raw, out var value) ? value : null;
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
namespace MeterVault.Core.Parsing;
|
||||
|
||||
/// <summary>
|
||||
/// Splits a value cell that carries a trailing unit into its number and unit
|
||||
/// (SDD Appendix A): <c>2287L</c> → (2287, "L"), <c>411kWh</c> → (411, "kWh"), <c>49</c> →
|
||||
/// (49, null). The unit is validated by the caller against the meter's expected unit.
|
||||
/// </summary>
|
||||
public static class ValueCell
|
||||
{
|
||||
public static (double Value, string? Unit)? Split(ReadOnlySpan<char> raw)
|
||||
{
|
||||
raw = raw.Trim();
|
||||
if (raw.IsEmpty)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var sign = raw[0] is '-' or '+' ? 1 : 0;
|
||||
var index = sign;
|
||||
while (index < raw.Length && (char.IsDigit(raw[index]) || raw[index] is '.' or ','))
|
||||
{
|
||||
index++;
|
||||
}
|
||||
|
||||
if (!GermanNumber.TryParse(raw[..index], out var value))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var unit = raw[index..].Trim();
|
||||
return (value, unit.IsEmpty ? null : unit.ToString());
|
||||
}
|
||||
}
|
||||
@@ -1,3 +1,6 @@
|
||||
using MeterVault.Core.Normalization;
|
||||
using MeterVault.Infrastructure.Import;
|
||||
using MeterVault.Infrastructure.Normalization;
|
||||
using MeterVault.Infrastructure.Persistence;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
@@ -17,6 +20,11 @@ public static class DependencyInjection
|
||||
npgsql.MigrationsAssembly(typeof(MeterVaultDbContext).Assembly.FullName))
|
||||
.UseSnakeCaseNamingConvention());
|
||||
|
||||
services.AddSingleton<INormalizationEngine>(_ => NormalizationEngine.CreateDefault());
|
||||
services.AddScoped<NormalizationService>();
|
||||
services.AddScoped<CsvImporter>();
|
||||
services.AddScoped<ImportService>();
|
||||
|
||||
return services;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,244 @@
|
||||
using System.Globalization;
|
||||
using CsvHelper;
|
||||
using CsvHelper.Configuration;
|
||||
using MeterVault.Core.Domain;
|
||||
using MeterVault.Core.Parsing;
|
||||
|
||||
namespace MeterVault.Infrastructure.Import;
|
||||
|
||||
/// <summary>
|
||||
/// Reads a CSV in the reference German dialect and stages readings/events/manual-costs per a
|
||||
/// <see cref="MappingProfile"/> (SDD §6.3). Pure transformation — no persistence; feed the result
|
||||
/// to <see cref="ImportService"/> to commit as a revertible batch, or to the normalization engine
|
||||
/// for a dry-run preview.
|
||||
/// </summary>
|
||||
public sealed class CsvImporter
|
||||
{
|
||||
public StagedImport Stage(MappingProfile profile, TextReader reader)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(profile);
|
||||
ArgumentNullException.ThrowIfNull(reader);
|
||||
|
||||
var rows = ReadRows(reader);
|
||||
var staged = new StagedImport();
|
||||
var previousByMeter = new Dictionary<int, double>();
|
||||
|
||||
for (var r = profile.FirstDataRowIndex; r < rows.Count; r++)
|
||||
{
|
||||
var row = rows[r];
|
||||
if (RowClassifier.ShouldSkip(row, profile, out var reason))
|
||||
{
|
||||
staged.SkippedRows++;
|
||||
if (!reason.StartsWith("blank", StringComparison.Ordinal))
|
||||
{
|
||||
staged.Warnings.Add($"Row {r + 1}: skipped ({reason}).");
|
||||
}
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!TryGetDate(row, profile, out var time))
|
||||
{
|
||||
staged.SkippedRows++;
|
||||
staged.Warnings.Add($"Row {r + 1}: unparseable date, skipped.");
|
||||
continue;
|
||||
}
|
||||
|
||||
foreach (var column in profile.Columns)
|
||||
{
|
||||
StageColumn(column, row, time, r, staged, previousByMeter);
|
||||
}
|
||||
}
|
||||
|
||||
return staged;
|
||||
}
|
||||
|
||||
private static void StageColumn(ColumnMapping column, IReadOnlyList<string> row, DateTimeOffset time,
|
||||
int rowIndex, StagedImport staged, Dictionary<int, double> previousByMeter)
|
||||
{
|
||||
if (column.Role == MappingRole.Ignore || column.Index >= row.Count)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var cell = row[column.Index];
|
||||
if (string.IsNullOrWhiteSpace(cell))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
switch (column.Role)
|
||||
{
|
||||
case MappingRole.Reading:
|
||||
StageReading(column, row, cell, time, rowIndex, staged, previousByMeter);
|
||||
break;
|
||||
|
||||
case MappingRole.Delivery:
|
||||
if (GermanNumber.TryParse(cell, out var litres) && litres > 0)
|
||||
{
|
||||
staged.Events.Add(new MeterEvent
|
||||
{
|
||||
MeterId = column.MeterId!.Value,
|
||||
Time = time,
|
||||
EventType = MeterEventType.Delivery,
|
||||
Amount = litres,
|
||||
Unit = column.Unit ?? "L",
|
||||
});
|
||||
}
|
||||
|
||||
break;
|
||||
|
||||
case MappingRole.TankLevel:
|
||||
var split = ValueCell.Split(cell);
|
||||
if (split is not null)
|
||||
{
|
||||
staged.Events.Add(new MeterEvent
|
||||
{
|
||||
MeterId = column.MeterId!.Value,
|
||||
Time = time,
|
||||
EventType = MeterEventType.TankLevel,
|
||||
Amount = split.Value.Value,
|
||||
Unit = column.Unit ?? split.Value.Unit,
|
||||
});
|
||||
}
|
||||
|
||||
break;
|
||||
|
||||
case MappingRole.ManualCost:
|
||||
if (GermanMoney.TryParse(cell, out var amount, out var currency))
|
||||
{
|
||||
staged.ManualCosts.Add(new ManualCost
|
||||
{
|
||||
CategoryId = column.CategoryId,
|
||||
PeriodStart = DateOnly.FromDateTime(time.UtcDateTime),
|
||||
PeriodEnd = DateOnly.FromDateTime(time.UtcDateTime.AddMonths(1).AddDays(-1)),
|
||||
Amount = (double)amount,
|
||||
Currency = currency,
|
||||
});
|
||||
}
|
||||
|
||||
break;
|
||||
|
||||
case MappingRole.Ignore:
|
||||
default:
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
private static void StageReading(ColumnMapping column, IReadOnlyList<string> row, string cell,
|
||||
DateTimeOffset time, int rowIndex, StagedImport staged, Dictionary<int, double> previousByMeter)
|
||||
{
|
||||
var split = ValueCell.Split(cell);
|
||||
if (split is null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var meterId = column.MeterId!.Value;
|
||||
var value = split.Value.Value;
|
||||
|
||||
if (column.Unit is not null && split.Value.Unit is not null
|
||||
&& !string.Equals(column.Unit, split.Value.Unit, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
staged.Warnings.Add(
|
||||
$"Row {rowIndex + 1}: expected unit '{column.Unit}' but found '{split.Value.Unit}'.");
|
||||
}
|
||||
|
||||
if (previousByMeter.TryGetValue(meterId, out var previous) && value < previous)
|
||||
{
|
||||
var override_ = ReadSwapOverride(column, row);
|
||||
staged.Events.Add(new MeterEvent
|
||||
{
|
||||
MeterId = meterId,
|
||||
Time = time,
|
||||
EventType = MeterEventType.MeterSwap,
|
||||
PrevValue = previous,
|
||||
NewValue = value,
|
||||
Amount = override_,
|
||||
Notes = "Auto-detected register decrease (meter swap / reset candidate).",
|
||||
});
|
||||
staged.Warnings.Add(
|
||||
$"Row {rowIndex + 1}: register for meter {meterId} dropped {previous}→{value}; " +
|
||||
(override_ is null ? "swap event created (verify)." : $"swap consumption set to {override_}."));
|
||||
}
|
||||
|
||||
staged.Readings.Add(new Reading
|
||||
{
|
||||
MeterId = meterId,
|
||||
Time = time,
|
||||
Value = value,
|
||||
Quality = ReadingQuality.Imported,
|
||||
});
|
||||
previousByMeter[meterId] = value;
|
||||
}
|
||||
|
||||
private static double? ReadSwapOverride(ColumnMapping column, IReadOnlyList<string> row)
|
||||
{
|
||||
if (column.SwapConsumptionColumn is not { } source || source >= row.Count)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
return GermanNumber.TryParse(row[source], out var value) ? value : null;
|
||||
}
|
||||
|
||||
private static bool TryGetDate(IReadOnlyList<string> row, MappingProfile profile, out DateTimeOffset time)
|
||||
{
|
||||
time = default;
|
||||
if (profile.DateColumn >= row.Count)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
var raw = row[profile.DateColumn];
|
||||
DateOnly date;
|
||||
var parsed = profile.DateKind switch
|
||||
{
|
||||
DateKind.MonthName => TryParseMonthAnchored(raw, profile.AnchorMonthsToEnd, out date),
|
||||
DateKind.DayDotMonthYear => GermanDate.TryParseDay(raw, out date),
|
||||
_ => ImportDate.TryResolve(raw, profile.AnchorMonthsToEnd, out date),
|
||||
};
|
||||
|
||||
if (!parsed)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
time = new DateTimeOffset(date.Year, date.Month, date.Day, 0, 0, 0, TimeSpan.Zero);
|
||||
return true;
|
||||
}
|
||||
|
||||
private static bool TryParseMonthAnchored(string? raw, bool anchorToEnd, out DateOnly date)
|
||||
{
|
||||
if (!GermanDate.TryParseMonth(raw, out var month))
|
||||
{
|
||||
date = default;
|
||||
return false;
|
||||
}
|
||||
|
||||
date = anchorToEnd
|
||||
? new DateOnly(month.Year, month.Month, DateTime.DaysInMonth(month.Year, month.Month))
|
||||
: month;
|
||||
return true;
|
||||
}
|
||||
|
||||
internal static List<string[]> ReadRows(TextReader reader)
|
||||
{
|
||||
var config = new CsvConfiguration(CultureInfo.InvariantCulture)
|
||||
{
|
||||
HasHeaderRecord = false,
|
||||
DetectColumnCountChanges = false,
|
||||
MissingFieldFound = null,
|
||||
BadDataFound = null,
|
||||
};
|
||||
|
||||
using var parser = new CsvParser(reader, config);
|
||||
var rows = new List<string[]>();
|
||||
while (parser.Read())
|
||||
{
|
||||
rows.Add(parser.Record ?? []);
|
||||
}
|
||||
|
||||
return rows;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
using MeterVault.Core.Parsing;
|
||||
|
||||
namespace MeterVault.Infrastructure.Import;
|
||||
|
||||
/// <summary>
|
||||
/// Resolves a cell to a date for import. Day-dated rows keep their day; month-name rows anchor to
|
||||
/// the first or last day of the month. Month-end anchoring matters when a sheet interleaves
|
||||
/// day-dated event rows with monthly rows (heating oil): a "Dezember 2022" snapshot must sort
|
||||
/// after a "05.12.2022" delivery reading, not before it.
|
||||
/// </summary>
|
||||
public static class ImportDate
|
||||
{
|
||||
public static bool TryResolve(string? raw, bool anchorMonthsToEnd, out DateOnly date)
|
||||
{
|
||||
if (GermanDate.TryParseDay(raw, out date))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
if (GermanDate.TryParseMonth(raw, out var month))
|
||||
{
|
||||
date = anchorMonthsToEnd
|
||||
? new DateOnly(month.Year, month.Month, DateTime.DaysInMonth(month.Year, month.Month))
|
||||
: month;
|
||||
return true;
|
||||
}
|
||||
|
||||
date = default;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
using MeterVault.Infrastructure.Normalization;
|
||||
using MeterVault.Infrastructure.Persistence;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
|
||||
namespace MeterVault.Infrastructure.Import;
|
||||
|
||||
/// <summary>
|
||||
/// Persists a <see cref="StagedImport"/> as a revertible <c>import_batch</c> and recomputes the
|
||||
/// affected meters' consumption (SDD §6.3, FR-6). Every row it writes is tagged with the batch id
|
||||
/// so the whole import can be undone. Runs in a single transaction.
|
||||
/// </summary>
|
||||
public sealed class ImportService(MeterVaultDbContext db, NormalizationService normalization)
|
||||
{
|
||||
private readonly MeterVaultDbContext _db = db;
|
||||
private readonly NormalizationService _normalization = normalization;
|
||||
|
||||
public async Task<int> CommitAsync(
|
||||
StagedImport staged, string? sourceName, string? mappingJson, CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(staged);
|
||||
|
||||
await using var tx = await _db.Database.BeginTransactionAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var batch = new Core.Domain.ImportBatch
|
||||
{
|
||||
SourceName = sourceName,
|
||||
Mapping = mappingJson,
|
||||
RowCount = staged.TotalRows,
|
||||
};
|
||||
_db.ImportBatches.Add(batch);
|
||||
await _db.SaveChangesAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
foreach (var reading in staged.Readings)
|
||||
{
|
||||
reading.ImportBatchId = batch.Id;
|
||||
_db.Readings.Add(reading);
|
||||
}
|
||||
|
||||
foreach (var meterEvent in staged.Events)
|
||||
{
|
||||
meterEvent.ImportBatchId = batch.Id;
|
||||
_db.MeterEvents.Add(meterEvent);
|
||||
}
|
||||
|
||||
foreach (var manualCost in staged.ManualCosts)
|
||||
{
|
||||
manualCost.ImportBatchId = batch.Id;
|
||||
_db.ManualCosts.Add(manualCost);
|
||||
}
|
||||
|
||||
await _db.SaveChangesAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
foreach (var meterId in AffectedMeters(staged))
|
||||
{
|
||||
await _normalization.RecomputeMeterAsync(meterId, batch.Id, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
await _db.SaveChangesAsync(cancellationToken).ConfigureAwait(false);
|
||||
await tx.CommitAsync(cancellationToken).ConfigureAwait(false);
|
||||
return batch.Id;
|
||||
}
|
||||
|
||||
public async Task RevertAsync(int batchId, CancellationToken cancellationToken = default)
|
||||
{
|
||||
await using var tx = await _db.Database.BeginTransactionAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var affected = await _db.Readings.Where(r => r.ImportBatchId == batchId).Select(r => r.MeterId)
|
||||
.Union(_db.MeterEvents.Where(e => e.ImportBatchId == batchId).Select(e => e.MeterId))
|
||||
.Distinct()
|
||||
.ToListAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
await _db.Consumption.Where(c => c.ImportBatchId == batchId).ExecuteDeleteAsync(cancellationToken).ConfigureAwait(false);
|
||||
await _db.Readings.Where(r => r.ImportBatchId == batchId).ExecuteDeleteAsync(cancellationToken).ConfigureAwait(false);
|
||||
await _db.MeterEvents.Where(e => e.ImportBatchId == batchId).ExecuteDeleteAsync(cancellationToken).ConfigureAwait(false);
|
||||
await _db.ManualCosts.Where(m => m.ImportBatchId == batchId).ExecuteDeleteAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
foreach (var meterId in affected)
|
||||
{
|
||||
await _normalization.RecomputeMeterAsync(meterId, null, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
var batch = await _db.ImportBatches.FirstOrDefaultAsync(b => b.Id == batchId, cancellationToken).ConfigureAwait(false);
|
||||
if (batch is not null)
|
||||
{
|
||||
batch.RevertedAt = DateTimeOffset.UtcNow;
|
||||
}
|
||||
|
||||
await _db.SaveChangesAsync(cancellationToken).ConfigureAwait(false);
|
||||
await tx.CommitAsync(cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private static IEnumerable<int> AffectedMeters(StagedImport staged) =>
|
||||
staged.Readings.Select(r => r.MeterId)
|
||||
.Concat(staged.Events.Select(e => e.MeterId))
|
||||
.Distinct();
|
||||
}
|
||||
@@ -0,0 +1,89 @@
|
||||
namespace MeterVault.Infrastructure.Import;
|
||||
|
||||
/// <summary>How a mapped column's cells are interpreted.</summary>
|
||||
public enum MappingRole
|
||||
{
|
||||
/// <summary>Column carries derived/oracle values the importer ignores.</summary>
|
||||
Ignore,
|
||||
|
||||
/// <summary>Cumulative/generation register or runtime hours → a <c>reading</c>.</summary>
|
||||
Reading,
|
||||
|
||||
/// <summary>Delivery quantity → a <c>delivery</c> meter event.</summary>
|
||||
Delivery,
|
||||
|
||||
/// <summary>Physical tank level (cm or L) → a <c>tank_level</c> meter event.</summary>
|
||||
TankLevel,
|
||||
|
||||
/// <summary>A pre-computed monthly cost → a <c>manual_cost</c> row for a category.</summary>
|
||||
ManualCost,
|
||||
}
|
||||
|
||||
/// <summary>How the date column is parsed.</summary>
|
||||
public enum DateKind
|
||||
{
|
||||
Auto,
|
||||
MonthName,
|
||||
DayDotMonthYear,
|
||||
}
|
||||
|
||||
/// <summary>Maps a single spreadsheet column (by index) to a meter/category and a role.</summary>
|
||||
public sealed class ColumnMapping
|
||||
{
|
||||
public required int Index { get; init; }
|
||||
|
||||
public MappingRole Role { get; init; }
|
||||
|
||||
/// <summary>Target meter for Reading/Delivery/TankLevel roles.</summary>
|
||||
public int? MeterId { get; init; }
|
||||
|
||||
/// <summary>Target category for ManualCost roles.</summary>
|
||||
public int? CategoryId { get; init; }
|
||||
|
||||
/// <summary>Expected unit; validated against the cell's unit suffix when present.</summary>
|
||||
public string? Unit { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// For a cumulative Reading column that can swap: the column whose value supplies the
|
||||
/// swap-month consumption override (the sheet's own consumption column). SDD "spreadsheet wins".
|
||||
/// </summary>
|
||||
public int? SwapConsumptionColumn { get; init; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A declarative description of one spreadsheet shape (SDD FR-6, Appendix A). The four reference
|
||||
/// sheets ship as built-in profiles; the import wizard produces more. Columns are referenced by
|
||||
/// index because the sheets pack blank spacer and duplicated derived headers side by side.
|
||||
/// </summary>
|
||||
public sealed class MappingProfile
|
||||
{
|
||||
public static readonly IReadOnlyList<string> DefaultSkipMarkers =
|
||||
["Total", "Heute", "Seitbeginn", "Seit ", "Gesammt"];
|
||||
|
||||
public required string Name { get; init; }
|
||||
|
||||
/// <summary>0-based index of the data header row (Heizöl's real header is not row 0).</summary>
|
||||
public int HeaderRowIndex { get; init; }
|
||||
|
||||
/// <summary>0-based index of the first data row.</summary>
|
||||
public int FirstDataRowIndex { get; init; }
|
||||
|
||||
public required int DateColumn { get; init; }
|
||||
|
||||
public DateKind DateKind { get; init; } = DateKind.Auto;
|
||||
|
||||
public IReadOnlyList<ColumnMapping> Columns { get; init; } = [];
|
||||
|
||||
public IReadOnlyList<string> SkipRowMarkers { get; init; } = DefaultSkipMarkers;
|
||||
|
||||
public bool SkipAllZeroRows { get; init; } = true;
|
||||
|
||||
/// <summary>Emit a meter swap when a cumulative register drops without an explaining event.</summary>
|
||||
public bool DetectCumulativeSwaps { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Anchor month-name rows to the last day of the month instead of the first. Needed when
|
||||
/// day-dated event rows are interleaved with monthly rows (heating oil).
|
||||
/// </summary>
|
||||
public bool AnchorMonthsToEnd { get; init; }
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
namespace MeterVault.Infrastructure.Import;
|
||||
|
||||
/// <summary>
|
||||
/// Built-in mapping profiles for the four reference <em>Energiebilanz</em> sheets (SDD §6.3).
|
||||
/// They ship as example imports and as the golden reconciliation fixtures. Meter and category ids
|
||||
/// are fixed conventions for the reference data; the import wizard maps to real ids at runtime.
|
||||
/// </summary>
|
||||
public static class ReferenceProfiles
|
||||
{
|
||||
// Reference meter ids.
|
||||
public const int Haus = 1;
|
||||
public const int Netz = 2;
|
||||
public const int Auto = 3;
|
||||
public const int Solar1 = 4;
|
||||
public const int Solar2 = 5;
|
||||
public const int Wasser = 10;
|
||||
public const int OilTank = 20;
|
||||
public const int Burner = 21;
|
||||
|
||||
// Reference category ids (match DatabaseSeeder order).
|
||||
public const int CategoryHeizung = 1;
|
||||
public const int CategoryStrom = 2;
|
||||
public const int CategoryWasser = 3;
|
||||
public const int CategoryPool = 4;
|
||||
|
||||
/// <summary>Linear heating-oil tank calibration: 7000 L / 150 cm ≈ 46.667 L/cm.</summary>
|
||||
public const double OilLitresPerCm = 7000d / 150d;
|
||||
|
||||
public static MappingProfile Electricity() => new()
|
||||
{
|
||||
Name = "Energiebilanz — Strom",
|
||||
DateColumn = 0,
|
||||
DateKind = DateKind.MonthName,
|
||||
FirstDataRowIndex = 1,
|
||||
Columns =
|
||||
[
|
||||
new ColumnMapping { Index = 1, Role = MappingRole.Reading, MeterId = Haus, Unit = "kWh" },
|
||||
new ColumnMapping { Index = 2, Role = MappingRole.Reading, MeterId = Netz, Unit = "kWh" },
|
||||
// Index 3 is a blank spacer column — left unmapped (Ignore).
|
||||
new ColumnMapping { Index = 4, Role = MappingRole.Reading, MeterId = Auto, Unit = "kWh" },
|
||||
new ColumnMapping { Index = 5, Role = MappingRole.Reading, MeterId = Solar1, Unit = "kWh" },
|
||||
new ColumnMapping { Index = 6, Role = MappingRole.Reading, MeterId = Solar2, Unit = "kWh" },
|
||||
],
|
||||
};
|
||||
|
||||
public static MappingProfile Water() => new()
|
||||
{
|
||||
Name = "Energiebilanz — Wasser",
|
||||
DateColumn = 0,
|
||||
DateKind = DateKind.MonthName,
|
||||
FirstDataRowIndex = 1,
|
||||
DetectCumulativeSwaps = true,
|
||||
Columns =
|
||||
[
|
||||
new ColumnMapping
|
||||
{
|
||||
Index = 1,
|
||||
Role = MappingRole.Reading,
|
||||
MeterId = Wasser,
|
||||
Unit = "m3",
|
||||
SwapConsumptionColumn = 2, // Wasserverbrauch supplies the swap-month consumption.
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
public static MappingProfile HeatingOil() => new()
|
||||
{
|
||||
Name = "Energiebilanz — Heizöl",
|
||||
DateColumn = 0,
|
||||
DateKind = DateKind.Auto, // early rows DD.MM.YYYY, later rows month names.
|
||||
AnchorMonthsToEnd = true, // a monthly snapshot sorts after same-month day-dated readings.
|
||||
HeaderRowIndex = 3,
|
||||
FirstDataRowIndex = 4,
|
||||
Columns =
|
||||
[
|
||||
new ColumnMapping { Index = 1, Role = MappingRole.Reading, MeterId = Burner, Unit = "h" },
|
||||
new ColumnMapping { Index = 4, Role = MappingRole.TankLevel, MeterId = OilTank, Unit = "cm" },
|
||||
new ColumnMapping { Index = 7, Role = MappingRole.Delivery, MeterId = OilTank, Unit = "L" },
|
||||
],
|
||||
};
|
||||
|
||||
public static MappingProfile Costs() => new()
|
||||
{
|
||||
Name = "Energiebilanz — Kosten",
|
||||
DateColumn = 0,
|
||||
DateKind = DateKind.MonthName,
|
||||
FirstDataRowIndex = 1,
|
||||
Columns =
|
||||
[
|
||||
new ColumnMapping { Index = 3, Role = MappingRole.ManualCost, CategoryId = CategoryHeizung },
|
||||
new ColumnMapping { Index = 4, Role = MappingRole.ManualCost, CategoryId = CategoryStrom },
|
||||
new ColumnMapping { Index = 5, Role = MappingRole.ManualCost, CategoryId = CategoryWasser },
|
||||
new ColumnMapping { Index = 6, Role = MappingRole.ManualCost, CategoryId = CategoryPool },
|
||||
],
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
using MeterVault.Core.Parsing;
|
||||
|
||||
namespace MeterVault.Infrastructure.Import;
|
||||
|
||||
/// <summary>
|
||||
/// Decides which CSV rows to ingest (SDD Appendix A): skips blank/separator rows, inline summary
|
||||
/// rows (Total / Heute / Seitbeginn / Seit … / Gesammt), and all-zero future placeholders.
|
||||
/// </summary>
|
||||
public static class RowClassifier
|
||||
{
|
||||
public static bool ShouldSkip(IReadOnlyList<string> row, MappingProfile profile, out string reason)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(row);
|
||||
ArgumentNullException.ThrowIfNull(profile);
|
||||
|
||||
if (row.Count == 0 || row.All(string.IsNullOrWhiteSpace))
|
||||
{
|
||||
reason = "blank/separator row";
|
||||
return true;
|
||||
}
|
||||
|
||||
var first = (row.Count > 0 ? row[0] : string.Empty).Trim();
|
||||
foreach (var marker in profile.SkipRowMarkers)
|
||||
{
|
||||
if (first.StartsWith(marker, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
reason = $"summary row '{first}'";
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
if (profile.SkipAllZeroRows && IsAllZeroData(row, profile))
|
||||
{
|
||||
reason = "all-zero placeholder";
|
||||
return true;
|
||||
}
|
||||
|
||||
reason = string.Empty;
|
||||
return false;
|
||||
}
|
||||
|
||||
private static bool IsAllZeroData(IReadOnlyList<string> row, MappingProfile profile)
|
||||
{
|
||||
var sawValueColumn = false;
|
||||
foreach (var column in profile.Columns)
|
||||
{
|
||||
if (column.Role == MappingRole.Ignore || column.Index >= row.Count)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
sawValueColumn = true;
|
||||
var cell = row[column.Index];
|
||||
if (string.IsNullOrWhiteSpace(cell))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
// Any non-zero numeric in a value column means the row carries data.
|
||||
if (GermanNumber.TryParse(cell, out var value) && value != 0)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!GermanNumber.TryParse(cell, out _) && GermanMoney.TryParse(cell, out var money, out _) && money != 0)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return sawValueColumn;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
using MeterVault.Core.Domain;
|
||||
|
||||
namespace MeterVault.Infrastructure.Import;
|
||||
|
||||
/// <summary>
|
||||
/// The result of staging a CSV against a profile: the readings/events/manual-costs it would
|
||||
/// create, plus warnings (detected swaps, anomalies, unit mismatches) and a skipped-row count.
|
||||
/// Nothing is persisted until <see cref="ImportService"/> commits it.
|
||||
/// </summary>
|
||||
public sealed class StagedImport
|
||||
{
|
||||
public List<Reading> Readings { get; } = [];
|
||||
|
||||
public List<MeterEvent> Events { get; } = [];
|
||||
|
||||
public List<ManualCost> ManualCosts { get; } = [];
|
||||
|
||||
public List<string> Warnings { get; } = [];
|
||||
|
||||
public int SkippedRows { get; set; }
|
||||
|
||||
public int TotalRows => Readings.Count + Events.Count + ManualCosts.Count;
|
||||
}
|
||||
@@ -0,0 +1,88 @@
|
||||
using System.Text.Json;
|
||||
using MeterVault.Core.Domain;
|
||||
using MeterVault.Core.Normalization;
|
||||
|
||||
namespace MeterVault.Infrastructure.Normalization;
|
||||
|
||||
/// <summary>
|
||||
/// Projects a persisted <see cref="Meter"/> (and optional <see cref="Tank"/>) into the
|
||||
/// infrastructure-free <see cref="MeterConfig"/> the normalization engine consumes.
|
||||
/// </summary>
|
||||
public static class MeterConfigFactory
|
||||
{
|
||||
public static MeterConfig FromMeter(Meter meter, Tank? tank)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(meter);
|
||||
|
||||
return new MeterConfig
|
||||
{
|
||||
MeterId = meter.Id,
|
||||
Mode = meter.Mode,
|
||||
Unit = meter.Unit,
|
||||
InitialBaseline = meter.InitialBaseline,
|
||||
Tank = tank is null ? null : new TankConfig
|
||||
{
|
||||
Capacity = tank.Capacity,
|
||||
RateMode = tank.RateMode,
|
||||
FixedRate = tank.FixedRate,
|
||||
Calibration = ParseCalibration(tank.Calibration),
|
||||
},
|
||||
Virtual = ParseVirtual(meter),
|
||||
};
|
||||
}
|
||||
|
||||
private static CalibrationCurve? ParseCalibration(string? json)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(json))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
using var doc = JsonDocument.Parse(json);
|
||||
var root = doc.RootElement;
|
||||
if (!root.TryGetProperty("volumePerUnit", out var perUnit))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var offset = root.TryGetProperty("offset", out var off) ? off.GetDouble() : 0d;
|
||||
return new CalibrationCurve(perUnit.GetDouble(), offset);
|
||||
}
|
||||
catch (JsonException)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private static VirtualSpec? ParseVirtual(Meter meter)
|
||||
{
|
||||
if (meter.Mode != MeterMode.Virtual || string.IsNullOrWhiteSpace(meter.Meta))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
using var doc = JsonDocument.Parse(meter.Meta);
|
||||
var root = doc.RootElement;
|
||||
if (!root.TryGetProperty("expression", out var expression))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var referenced = new List<int>();
|
||||
if (root.TryGetProperty("referencedMeterIds", out var ids) && ids.ValueKind == JsonValueKind.Array)
|
||||
{
|
||||
referenced.AddRange(ids.EnumerateArray().Select(e => e.GetInt32()));
|
||||
}
|
||||
|
||||
return new VirtualSpec { Expression = expression.GetString() ?? string.Empty, ReferencedMeterIds = referenced };
|
||||
}
|
||||
catch (JsonException)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
using MeterVault.Core.Domain;
|
||||
using MeterVault.Core.Normalization;
|
||||
using MeterVault.Infrastructure.Persistence;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
|
||||
namespace MeterVault.Infrastructure.Normalization;
|
||||
|
||||
/// <summary>
|
||||
/// Bridges persisted readings/events to the pure engine: loads a meter's inputs, recomputes its
|
||||
/// consumption wholesale (consumption is a pure function of readings + events), and replaces the
|
||||
/// stored rows. Virtual meters are skipped here — they are computed on read (SDD §14.1).
|
||||
/// </summary>
|
||||
public sealed class NormalizationService(MeterVaultDbContext db, INormalizationEngine engine)
|
||||
{
|
||||
private readonly MeterVaultDbContext _db = db;
|
||||
private readonly INormalizationEngine _engine = engine;
|
||||
|
||||
/// <summary>
|
||||
/// Recomputes and replaces the consumption series for one meter from all its current readings
|
||||
/// and events. Tags new rows with <paramref name="batchId"/> for provenance. Does not save.
|
||||
/// </summary>
|
||||
public async Task RecomputeMeterAsync(int meterId, int? batchId, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var meter = await _db.Meters.FirstOrDefaultAsync(m => m.Id == meterId, cancellationToken).ConfigureAwait(false);
|
||||
if (meter is null || meter.Mode == MeterMode.Virtual)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var tank = await _db.Tanks.FirstOrDefaultAsync(t => t.MeterId == meterId, cancellationToken).ConfigureAwait(false);
|
||||
var config = MeterConfigFactory.FromMeter(meter, tank);
|
||||
|
||||
var readings = await _db.Readings
|
||||
.Where(r => r.MeterId == meterId)
|
||||
.OrderBy(r => r.Time)
|
||||
.ToListAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var events = await _db.MeterEvents
|
||||
.Where(e => e.MeterId == meterId)
|
||||
.OrderBy(e => e.Time)
|
||||
.ToListAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var context = new NormalizationContext { Meter = config, Readings = readings, Events = events };
|
||||
var consumption = _engine.Normalize(context);
|
||||
|
||||
await _db.Consumption.Where(c => c.MeterId == meterId)
|
||||
.ExecuteDeleteAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
foreach (var row in consumption)
|
||||
{
|
||||
row.ImportBatchId = batchId;
|
||||
_db.Consumption.Add(row);
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user