M2: German CSV importer + reconciliation of all 4 fixtures
- German-dialect scalar parsers in Core (GermanNumber/Money/Date, ValueCell): decimal comma, thousands dot, unit suffixes, € currency, both date shapes. - Declarative MappingProfile + RowClassifier (skip summary/blank/all-zero rows) + CsvImporter (CsvHelper) staging readings/events/manual-costs, with auto swap detection on register decreases and month-end anchoring for interleaved oil dates. - Four built-in ReferenceProfiles (Strom/Wasser/Heizöl/Kosten). - ImportService: commit as revertible import_batch + wholesale consumption recompute per affected meter (NormalizationService/MeterConfigFactory), revert by batch. - Reconciliation tests: all 4 CSVs match the sheet's own columns within tolerance (electricity 5 meters + Netz Einsparung, water swap→12, oil tank incl. deliveries + burner hours, cost category totals). Commit/revert round-trip verified on Timescale. 69 tests green (53 Core + 16 integration). Known follow-up (polish): historical imports can contend with the 30-day compression policy's background job; tests pause it. Consider retry-on-deadlock or deferred compression for large historical imports in production. Claude-Session: https://claude.ai/code/session_01WujdMtMJPbxDpDnMeK22rr
This commit is contained in:
@@ -0,0 +1,244 @@
|
||||
using System.Globalization;
|
||||
using CsvHelper;
|
||||
using CsvHelper.Configuration;
|
||||
using MeterVault.Core.Domain;
|
||||
using MeterVault.Core.Parsing;
|
||||
|
||||
namespace MeterVault.Infrastructure.Import;
|
||||
|
||||
/// <summary>
|
||||
/// Reads a CSV in the reference German dialect and stages readings/events/manual-costs per a
|
||||
/// <see cref="MappingProfile"/> (SDD §6.3). Pure transformation — no persistence; feed the result
|
||||
/// to <see cref="ImportService"/> to commit as a revertible batch, or to the normalization engine
|
||||
/// for a dry-run preview.
|
||||
/// </summary>
|
||||
public sealed class CsvImporter
|
||||
{
|
||||
public StagedImport Stage(MappingProfile profile, TextReader reader)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(profile);
|
||||
ArgumentNullException.ThrowIfNull(reader);
|
||||
|
||||
var rows = ReadRows(reader);
|
||||
var staged = new StagedImport();
|
||||
var previousByMeter = new Dictionary<int, double>();
|
||||
|
||||
for (var r = profile.FirstDataRowIndex; r < rows.Count; r++)
|
||||
{
|
||||
var row = rows[r];
|
||||
if (RowClassifier.ShouldSkip(row, profile, out var reason))
|
||||
{
|
||||
staged.SkippedRows++;
|
||||
if (!reason.StartsWith("blank", StringComparison.Ordinal))
|
||||
{
|
||||
staged.Warnings.Add($"Row {r + 1}: skipped ({reason}).");
|
||||
}
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!TryGetDate(row, profile, out var time))
|
||||
{
|
||||
staged.SkippedRows++;
|
||||
staged.Warnings.Add($"Row {r + 1}: unparseable date, skipped.");
|
||||
continue;
|
||||
}
|
||||
|
||||
foreach (var column in profile.Columns)
|
||||
{
|
||||
StageColumn(column, row, time, r, staged, previousByMeter);
|
||||
}
|
||||
}
|
||||
|
||||
return staged;
|
||||
}
|
||||
|
||||
private static void StageColumn(ColumnMapping column, IReadOnlyList<string> row, DateTimeOffset time,
|
||||
int rowIndex, StagedImport staged, Dictionary<int, double> previousByMeter)
|
||||
{
|
||||
if (column.Role == MappingRole.Ignore || column.Index >= row.Count)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var cell = row[column.Index];
|
||||
if (string.IsNullOrWhiteSpace(cell))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
switch (column.Role)
|
||||
{
|
||||
case MappingRole.Reading:
|
||||
StageReading(column, row, cell, time, rowIndex, staged, previousByMeter);
|
||||
break;
|
||||
|
||||
case MappingRole.Delivery:
|
||||
if (GermanNumber.TryParse(cell, out var litres) && litres > 0)
|
||||
{
|
||||
staged.Events.Add(new MeterEvent
|
||||
{
|
||||
MeterId = column.MeterId!.Value,
|
||||
Time = time,
|
||||
EventType = MeterEventType.Delivery,
|
||||
Amount = litres,
|
||||
Unit = column.Unit ?? "L",
|
||||
});
|
||||
}
|
||||
|
||||
break;
|
||||
|
||||
case MappingRole.TankLevel:
|
||||
var split = ValueCell.Split(cell);
|
||||
if (split is not null)
|
||||
{
|
||||
staged.Events.Add(new MeterEvent
|
||||
{
|
||||
MeterId = column.MeterId!.Value,
|
||||
Time = time,
|
||||
EventType = MeterEventType.TankLevel,
|
||||
Amount = split.Value.Value,
|
||||
Unit = column.Unit ?? split.Value.Unit,
|
||||
});
|
||||
}
|
||||
|
||||
break;
|
||||
|
||||
case MappingRole.ManualCost:
|
||||
if (GermanMoney.TryParse(cell, out var amount, out var currency))
|
||||
{
|
||||
staged.ManualCosts.Add(new ManualCost
|
||||
{
|
||||
CategoryId = column.CategoryId,
|
||||
PeriodStart = DateOnly.FromDateTime(time.UtcDateTime),
|
||||
PeriodEnd = DateOnly.FromDateTime(time.UtcDateTime.AddMonths(1).AddDays(-1)),
|
||||
Amount = (double)amount,
|
||||
Currency = currency,
|
||||
});
|
||||
}
|
||||
|
||||
break;
|
||||
|
||||
case MappingRole.Ignore:
|
||||
default:
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
private static void StageReading(ColumnMapping column, IReadOnlyList<string> row, string cell,
|
||||
DateTimeOffset time, int rowIndex, StagedImport staged, Dictionary<int, double> previousByMeter)
|
||||
{
|
||||
var split = ValueCell.Split(cell);
|
||||
if (split is null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var meterId = column.MeterId!.Value;
|
||||
var value = split.Value.Value;
|
||||
|
||||
if (column.Unit is not null && split.Value.Unit is not null
|
||||
&& !string.Equals(column.Unit, split.Value.Unit, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
staged.Warnings.Add(
|
||||
$"Row {rowIndex + 1}: expected unit '{column.Unit}' but found '{split.Value.Unit}'.");
|
||||
}
|
||||
|
||||
if (previousByMeter.TryGetValue(meterId, out var previous) && value < previous)
|
||||
{
|
||||
var override_ = ReadSwapOverride(column, row);
|
||||
staged.Events.Add(new MeterEvent
|
||||
{
|
||||
MeterId = meterId,
|
||||
Time = time,
|
||||
EventType = MeterEventType.MeterSwap,
|
||||
PrevValue = previous,
|
||||
NewValue = value,
|
||||
Amount = override_,
|
||||
Notes = "Auto-detected register decrease (meter swap / reset candidate).",
|
||||
});
|
||||
staged.Warnings.Add(
|
||||
$"Row {rowIndex + 1}: register for meter {meterId} dropped {previous}→{value}; " +
|
||||
(override_ is null ? "swap event created (verify)." : $"swap consumption set to {override_}."));
|
||||
}
|
||||
|
||||
staged.Readings.Add(new Reading
|
||||
{
|
||||
MeterId = meterId,
|
||||
Time = time,
|
||||
Value = value,
|
||||
Quality = ReadingQuality.Imported,
|
||||
});
|
||||
previousByMeter[meterId] = value;
|
||||
}
|
||||
|
||||
private static double? ReadSwapOverride(ColumnMapping column, IReadOnlyList<string> row)
|
||||
{
|
||||
if (column.SwapConsumptionColumn is not { } source || source >= row.Count)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
return GermanNumber.TryParse(row[source], out var value) ? value : null;
|
||||
}
|
||||
|
||||
private static bool TryGetDate(IReadOnlyList<string> row, MappingProfile profile, out DateTimeOffset time)
|
||||
{
|
||||
time = default;
|
||||
if (profile.DateColumn >= row.Count)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
var raw = row[profile.DateColumn];
|
||||
DateOnly date;
|
||||
var parsed = profile.DateKind switch
|
||||
{
|
||||
DateKind.MonthName => TryParseMonthAnchored(raw, profile.AnchorMonthsToEnd, out date),
|
||||
DateKind.DayDotMonthYear => GermanDate.TryParseDay(raw, out date),
|
||||
_ => ImportDate.TryResolve(raw, profile.AnchorMonthsToEnd, out date),
|
||||
};
|
||||
|
||||
if (!parsed)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
time = new DateTimeOffset(date.Year, date.Month, date.Day, 0, 0, 0, TimeSpan.Zero);
|
||||
return true;
|
||||
}
|
||||
|
||||
private static bool TryParseMonthAnchored(string? raw, bool anchorToEnd, out DateOnly date)
|
||||
{
|
||||
if (!GermanDate.TryParseMonth(raw, out var month))
|
||||
{
|
||||
date = default;
|
||||
return false;
|
||||
}
|
||||
|
||||
date = anchorToEnd
|
||||
? new DateOnly(month.Year, month.Month, DateTime.DaysInMonth(month.Year, month.Month))
|
||||
: month;
|
||||
return true;
|
||||
}
|
||||
|
||||
internal static List<string[]> ReadRows(TextReader reader)
|
||||
{
|
||||
var config = new CsvConfiguration(CultureInfo.InvariantCulture)
|
||||
{
|
||||
HasHeaderRecord = false,
|
||||
DetectColumnCountChanges = false,
|
||||
MissingFieldFound = null,
|
||||
BadDataFound = null,
|
||||
};
|
||||
|
||||
using var parser = new CsvParser(reader, config);
|
||||
var rows = new List<string[]>();
|
||||
while (parser.Read())
|
||||
{
|
||||
rows.Add(parser.Record ?? []);
|
||||
}
|
||||
|
||||
return rows;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user