FinalFactory/MeterVault
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Import: CSV mapping wizard with revertible batches

Browse Source 
New /import/wizard: upload an arbitrary CSV, preview the column grid, map each column to a role + target meter/category + unit, dry-run, then commit as a revertible import_batch. The /import page now lists recent imports with one-click revert. Adds CsvImporter.ReadRawRows (raw preview) and Confirm.ConfirmAsync (generic confirm). Verified end-to-end in a real browser + direct DB checks.

Claude-Session: https://claude.ai/code/session_01Kib2MniVFbD95fkgLgBBnB
This commit is contained in:
Florian Schmidt
2026-07-17 11:05:01 +02:00
parent b3b8b92520
commit 14930bc3d8
4 changed files with 507 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/App/Components/Pages/Import.razor
+100 -6
View File
@@ -1,8 +1,13 @@
@page "/import"
@inject MeterVault.Infrastructure.Import.ReferenceDataImporter ReferenceImporter
@inject MeterVault.Infrastructure.Import.CsvImporter CsvImporter
@inject MeterVault.Infrastructure.Import.ImportService ImportService
@inject Microsoft.EntityFrameworkCore.IDbContextFactory<MeterVault.Infrastructure.Persistence.MeterVaultDbContext> DbFactory
@inject IDialogService Dialogs
@inject ISnackbar Snackbar
@using MeterVault.Infrastructure.Import
@using MeterVault.Infrastructure.Persistence
@using Microsoft.EntityFrameworkCore
<PageTitle>MeterVault — Import</PageTitle>
@@ -29,18 +34,23 @@
<MudItem xs="12" md="6">
<MudPaper Class="pa-4" Elevation="2" Style="height:100%">
<MudText Typo="Typo.h6">Dry-run a CSV</MudText>
<MudText Typo="Typo.body2" Color="Color.Secondary" Class="mb-3">
Upload a sheet and preview what would be staged (no changes are made).
<div class="d-flex align-center justify-space-between">
<MudText Typo="Typo.h6">Your own CSV</MudText>
<MudButton Variant="Variant.Filled" Color="Color.Secondary" Href="/import/wizard"
StartIcon="@Icons.Material.Filled.AutoFixHigh">Mapping wizard</MudButton>
</div>
<MudText Typo="Typo.body2" Color="Color.Secondary" Class="mb-3 mt-1">
Map an arbitrary sheet's columns to your meters/categories, preview and commit it as a
revertible import. Or dry-run against one of the built-in reference profiles below.
</MudText>
<MudSelect T="string" @bind-Value="_profileName" Label="Sheet type" Dense="true" Class="mb-2">
<MudSelect T="string" @bind-Value="_profileName" Label="Reference profile" Dense="true" Class="mb-2">
<MudSelectItem T="string" Value="@("Strom")">Electricity (Strom)</MudSelectItem>
<MudSelectItem T="string" Value="@("Wasser")">Water (Wasser)</MudSelectItem>
<MudSelectItem T="string" Value="@("Heizöl")">Heating oil (Heizöl)</MudSelectItem>
<MudSelectItem T="string" Value="@("Kosten")">Costs (Kosten)</MudSelectItem>
</MudSelect>
<MudButton HtmlTag="label" Variant="Variant.Outlined" StartIcon="@Icons.Material.Filled.UploadFile" for="csvUpload">
Choose CSV
Dry-run a reference sheet
</MudButton>
<InputFile id="csvUpload" OnChange="PreviewAsync" accept=".csv" style="display:none" />
</MudPaper>
@@ -71,6 +81,49 @@
</MudPaper>
</MudItem>
}
<MudItem xs="12">
<MudPaper Class="pa-4" Elevation="2">
<MudText Typo="Typo.h6" Class="mb-2">Recent imports</MudText>
@if (_batches.Count == 0)
{
<MudText Typo="Typo.body2" Color="Color.Secondary">No imports yet.</MudText>
}
else
{
<MudSimpleTable Dense="true" Hover="true">
<thead><tr><th>#</th><th>Source</th><th style="text-align:right">Rows</th><th>Imported</th><th>Status</th><th></th></tr></thead>
<tbody>
@foreach (var batch in _batches)
{
<tr>
<td>@batch.Id</td>
<td>@(batch.SourceName ?? "—")</td>
<td style="text-align:right">@batch.RowCount</td>
<td>@batch.CreatedAt.LocalDateTime.ToString("yyyy-MM-dd HH:mm")</td>
<td>
@if (batch.RevertedAt is not null)
{
<MudChip T="string" Size="Size.Small" Color="Color.Default">reverted</MudChip>
}
else
{
<MudChip T="string" Size="Size.Small" Color="Color.Success">active</MudChip>
}
</td>
<td style="text-align:right">
<MudButton Size="Size.Small" Variant="Variant.Outlined" Color="Color.Error"
StartIcon="@Icons.Material.Filled.Undo"
Disabled="@(batch.RevertedAt is not null || _reverting == batch.Id)"
OnClick="@(() => RevertAsync(batch))">Revert</MudButton>
</td>
</tr>
}
</tbody>
</MudSimpleTable>
}
</MudPaper>
</MudItem>
</MudGrid>
@code {
@@ -78,9 +131,49 @@
private bool _referenceLoaded;
private string _profileName = "Strom";
private StagedImport? _preview;
private List<ImportBatch> _batches = [];
private int? _reverting;
protected override async Task OnInitializedAsync() =>
protected override async Task OnInitializedAsync()
{
_referenceLoaded = await ReferenceImporter.IsLoadedAsync();
await LoadBatchesAsync();
}
private async Task LoadBatchesAsync()
{
await using var db = await DbFactory.CreateDbContextAsync();
_batches = await db.ImportBatches.AsNoTracking()
.OrderByDescending(b => b.Id)
.Take(25)
.ToListAsync();
}
private async Task RevertAsync(ImportBatch batch)
{
if (!await Confirm.ConfirmAsync(Dialogs, "Revert import?",
$"Delete all {batch.RowCount} rows from import #{batch.Id} ({batch.SourceName ?? "unnamed"}) and recompute the affected meters?",
"Revert"))
{
return;
}
_reverting = batch.Id;
try
{
await ImportService.RevertAsync(batch.Id);
Snackbar.Add($"Import #{batch.Id} reverted.", Severity.Success);
await LoadBatchesAsync();
}
catch (Exception ex)
{
Snackbar.Add($"Revert failed: {ex.Message}", Severity.Error);
}
finally
{
_reverting = null;
}
}
private async Task LoadReferenceAsync()
{
@@ -91,6 +184,7 @@
await ReferenceImporter.LoadAsync(dir);
_referenceLoaded = true;
Snackbar.Add("Reference data loaded.", Severity.Success);
await LoadBatchesAsync();
}
catch (Exception ex)
{
src/App/Components/Pages/ImportWizard.razor
+393
View File
@@ -0,0 +1,393 @@
@page "/import/wizard"
@inject MeterVault.Infrastructure.Import.CsvImporter CsvImporter
@inject MeterVault.Infrastructure.Import.ImportService ImportService
@inject Microsoft.EntityFrameworkCore.IDbContextFactory<MeterVault.Infrastructure.Persistence.MeterVaultDbContext> DbFactory
@inject ISnackbar Snackbar
@inject NavigationManager Nav
@using System.Text.Json
@using MeterVault.Core.Domain
@using MeterVault.Infrastructure.Import
@using MeterVault.Infrastructure.Persistence
@using Microsoft.EntityFrameworkCore
<PageTitle>MeterVault — Import wizard</PageTitle>
<div class="d-flex align-center mb-4" style="gap:1rem">
<MudIconButton Icon="@Icons.Material.Filled.ArrowBack" Href="/import" aria-label="Back to Import" />
<MudText Typo="Typo.h4">Import wizard</MudText>
</div>
<MudText Typo="Typo.body2" Color="Color.Secondary" Class="mb-4">
Upload any CSV, map its columns to your meters and categories, preview what would be staged, then
commit it as a revertible import. Values may use the German dialect (decimal comma, unit suffixes,
<code>Monat JJJJ</code> or <code>TT.MM.JJJJ</code> dates) — the same parser the reference sheets use.
</MudText>
<MudPaper Class="pa-4 mb-4" Elevation="2">
<div class="d-flex align-center" style="gap:1rem; flex-wrap:wrap">
<MudButton HtmlTag="label" Variant="Variant.Filled" Color="Color.Primary"
StartIcon="@Icons.Material.Filled.UploadFile" for="wizardUpload">
Choose CSV
</MudButton>
<InputFile id="wizardUpload" OnChange="OnFileAsync" accept=".csv" style="display:none" />
@if (_fileName is not null)
{
<MudText><b>@_fileName</b> — @_rows.Count rows, @_colCount columns</MudText>
}
</div>
</MudPaper>
@if (_colCount > 0)
{
<MudPaper Class="pa-4 mb-4" Elevation="2">
<MudText Typo="Typo.h6" Class="mb-2">1. Parsing options</MudText>
<MudGrid>
<MudItem xs="12" sm="6" md="3">
<MudSelect T="int" @bind-Value="_dateColumn" Label="Date column" Dense="true">
@foreach (var i in Enumerable.Range(0, _colCount))
{
<MudSelectItem T="int" Value="i">@ColLabel(i)</MudSelectItem>
}
</MudSelect>
</MudItem>
<MudItem xs="12" sm="6" md="3">
<MudSelect T="DateKind" @bind-Value="_dateKind" Label="Date format" Dense="true">
<MudSelectItem T="DateKind" Value="DateKind.Auto">Auto-detect</MudSelectItem>
<MudSelectItem T="DateKind" Value="DateKind.MonthName">Month name (Januar 2024)</MudSelectItem>
<MudSelectItem T="DateKind" Value="DateKind.DayDotMonthYear">Day (31.12.2024)</MudSelectItem>
</MudSelect>
</MudItem>
<MudItem xs="6" sm="6" md="2">
<MudNumericField T="int" @bind-Value="_headerRow" Label="Header row" Min="0" Margin="Margin.Dense" />
</MudItem>
<MudItem xs="6" sm="6" md="2">
<MudNumericField T="int" @bind-Value="_firstDataRow" Label="First data row" Min="0" Margin="Margin.Dense" />
</MudItem>
<MudItem xs="12" md="2" Class="d-flex flex-column">
<MudSwitch T="bool" @bind-Value="_skipAllZero" Label="Skip zero rows" Color="Color.Primary" />
<MudSwitch T="bool" @bind-Value="_detectSwaps" Label="Detect swaps" Color="Color.Primary" />
</MudItem>
</MudGrid>
</MudPaper>
<MudPaper Class="pa-4 mb-4" Elevation="2">
<MudText Typo="Typo.h6" Class="mb-2">2. Column preview</MudText>
<div style="overflow-x:auto">
<MudSimpleTable Dense="true" Bordered="true" Style="min-width:100%">
<thead>
<tr>
@foreach (var i in Enumerable.Range(0, _colCount))
{
<th style="@(i == _dateColumn ? "background:var(--mud-palette-primary-hover)" : "")">
Col @i@(i == _dateColumn ? " 📅" : "")
</th>
}
</tr>
</thead>
<tbody>
@foreach (var r in Enumerable.Range(0, Math.Min(_rows.Count, _firstDataRow + 8)))
{
<tr style="@(r < _firstDataRow ? "opacity:0.5" : "")">
@foreach (var i in Enumerable.Range(0, _colCount))
{
<td style="font-size:0.78rem; white-space:nowrap">@Cell(r, i)</td>
}
</tr>
}
</tbody>
</MudSimpleTable>
</div>
<MudText Typo="Typo.caption" Color="Color.Secondary">
Faded rows are before the first data row. The 📅 column supplies the date.
</MudText>
</MudPaper>
<MudPaper Class="pa-4 mb-4" Elevation="2">
<MudText Typo="Typo.h6" Class="mb-2">3. Map columns</MudText>
<div style="overflow-x:auto">
<MudSimpleTable Dense="true" Style="min-width:100%">
<thead>
<tr><th>Column</th><th>Sample</th><th style="min-width:160px">Role</th><th style="min-width:220px">Target</th><th style="min-width:120px">Unit</th></tr>
</thead>
<tbody>
@foreach (var i in Enumerable.Range(0, _colCount))
{
<tr>
<td>
<b>Col @i</b>
@if (!string.IsNullOrWhiteSpace(Header(i)))
{
<br /><span style="font-size:0.72rem; opacity:0.7">@Header(i)</span>
}
</td>
<td style="font-size:0.8rem; max-width:140px; overflow:hidden; text-overflow:ellipsis">@Sample(i)</td>
<td>
<MudSelect T="MappingRole" @bind-Value="_columns[i].Role" Dense="true" Margin="Margin.Dense">
@foreach (var role in Enum.GetValues<MappingRole>())
{
<MudSelectItem T="MappingRole" Value="role">@role</MudSelectItem>
}
</MudSelect>
</td>
<td>
@if (_columns[i].Role is MappingRole.Reading or MappingRole.Delivery or MappingRole.TankLevel)
{
<MudSelect T="int?" @bind-Value="_columns[i].MeterId" Dense="true" Margin="Margin.Dense"
Placeholder="Select meter" Clearable="true">
@foreach (var m in _meters)
{
<MudSelectItem T="int?" Value="m.Id">@m.Name (@m.Unit)</MudSelectItem>
}
</MudSelect>
}
else if (_columns[i].Role == MappingRole.ManualCost)
{
<MudSelect T="int?" @bind-Value="_columns[i].CategoryId" Dense="true" Margin="Margin.Dense"
Placeholder="Select category" Clearable="true">
@foreach (var c in _categories)
{
<MudSelectItem T="int?" Value="c.Id">@c.Name</MudSelectItem>
}
</MudSelect>
}
</td>
<td>
@if (_columns[i].Role is MappingRole.Reading or MappingRole.Delivery or MappingRole.TankLevel)
{
<MudTextField @bind-Value="_columns[i].Unit" Placeholder="e.g. kWh" Margin="Margin.Dense" />
}
</td>
</tr>
}
</tbody>
</MudSimpleTable>
</div>
</MudPaper>
@if (_validationErrors.Count > 0)
{
<MudAlert Severity="Severity.Warning" Class="mb-4">
@foreach (var err in _validationErrors)
{
<div>@err</div>
}
</MudAlert>
}
<MudPaper Class="pa-4 mb-4" Elevation="2">
<div class="d-flex align-center mb-2" style="gap:1rem; flex-wrap:wrap">
<MudText Typo="Typo.h6">4. Preview &amp; commit</MudText>
<MudButton Variant="Variant.Outlined" Color="Color.Primary" StartIcon="@Icons.Material.Filled.Visibility"
OnClick="Preview">Dry-run preview</MudButton>
<MudButton Variant="Variant.Filled" Color="Color.Success" StartIcon="@Icons.Material.Filled.Save"
OnClick="CommitAsync" Disabled="_staged is null || _staged.TotalRows == 0 || _committing">
Commit import
</MudButton>
@if (_committing)
{
<MudProgressCircular Size="Size.Small" Indeterminate="true" Color="Color.Primary" />
}
</div>
@if (_staged is not null)
{
<div class="d-flex mt-2" style="gap:2rem; flex-wrap:wrap">
<MudText>Readings: <b>@_staged.Readings.Count</b></MudText>
<MudText>Events: <b>@_staged.Events.Count</b></MudText>
<MudText>Manual costs: <b>@_staged.ManualCosts.Count</b></MudText>
<MudText>Skipped rows: <b>@_staged.SkippedRows</b></MudText>
</div>
@if (_staged.TotalRows == 0)
{
<MudAlert Severity="Severity.Info" Dense="true" Class="mt-2">
Nothing staged. Check the first-data-row, date column and column mappings above.
</MudAlert>
}
@if (_staged.Warnings.Count > 0)
{
<MudExpansionPanels Class="mt-3">
<MudExpansionPanel Text="@($"{_staged.Warnings.Count} warnings")">
@foreach (var warning in _staged.Warnings.Take(100))
{
<MudText Typo="Typo.body2">@warning</MudText>
}
</MudExpansionPanel>
</MudExpansionPanels>
}
}
</MudPaper>
}
@code {
private sealed class ColumnState
{
public MappingRole Role { get; set; } = MappingRole.Ignore;
public int? MeterId { get; set; }
public int? CategoryId { get; set; }
public string? Unit { get; set; }
}
private string? _fileName;
private string? _csvText;
private List<string[]> _rows = [];
private int _colCount;
private ColumnState[] _columns = [];
private int _dateColumn;
private DateKind _dateKind = DateKind.Auto;
private int _headerRow;
private int _firstDataRow = 1;
private bool _skipAllZero = true;
private bool _detectSwaps;
private List<Meter> _meters = [];
private List<CostCategory> _categories = [];
private StagedImport? _staged;
private List<string> _validationErrors = [];
private bool _committing;
private async Task OnFileAsync(InputFileChangeEventArgs args)
{
var file = args.File;
if (file is null)
{
return;
}
_fileName = file.Name;
using (var reader = new StreamReader(file.OpenReadStream(maxAllowedSize: 10 * 1024 * 1024)))
{
_csvText = await reader.ReadToEndAsync();
}
_rows = CsvImporter.ReadRawRows(new StringReader(_csvText)).ToList();
_colCount = _rows.Count == 0 ? 0 : _rows.Max(r => r.Length);
_columns = Enumerable.Range(0, _colCount).Select(_ => new ColumnState()).ToArray();
_dateColumn = 0;
_staged = null;
_validationErrors = [];
await using var db = await DbFactory.CreateDbContextAsync();
_meters = await db.Meters.AsNoTracking().OrderBy(m => m.Name).ToListAsync();
_categories = await db.CostCategories.AsNoTracking().OrderBy(c => c.Sort).ThenBy(c => c.Name).ToListAsync();
}
private void Preview()
{
_validationErrors = Validate();
if (_validationErrors.Count > 0)
{
_staged = null;
return;
}
using var reader = new StringReader(_csvText ?? string.Empty);
_staged = CsvImporter.Stage(BuildProfile(), reader);
}
private async Task CommitAsync()
{
if (_staged is null || _staged.TotalRows == 0)
{
return;
}
_committing = true;
try
{
var mappingJson = JsonSerializer.Serialize(new
{
dateColumn = _dateColumn,
dateKind = _dateKind.ToString(),
firstDataRow = _firstDataRow,
columns = _columns
.Select((c, i) => new { index = i, role = c.Role.ToString(), c.MeterId, c.CategoryId, c.Unit })
.Where(c => c.role != nameof(MappingRole.Ignore)),
});
var batchId = await ImportService.CommitAsync(_staged, _fileName, mappingJson);
Snackbar.Add($"Imported batch #{batchId}: {_staged.TotalRows} rows staged. Consumption recomputed.", Severity.Success);
Nav.NavigateTo("/import");
}
catch (Exception ex)
{
Snackbar.Add($"Commit failed: {ex.Message}", Severity.Error);
}
finally
{
_committing = false;
}
}
private List<string> Validate()
{
var errors = new List<string>();
if (_columns.Count(c => c.Role != MappingRole.Ignore) == 0)
{
errors.Add("Map at least one column to a role other than Ignore.");
}
for (var i = 0; i < _columns.Length; i++)
{
var c = _columns[i];
if (c.Role is MappingRole.Reading or MappingRole.Delivery or MappingRole.TankLevel && c.MeterId is null)
{
errors.Add($"Col {i} ({c.Role}) needs a target meter.");
}
if (c.Role == MappingRole.ManualCost && c.CategoryId is null)
{
errors.Add($"Col {i} (ManualCost) needs a target category.");
}
}
return errors;
}
private MappingProfile BuildProfile() => new()
{
Name = _fileName ?? "Custom CSV",
HeaderRowIndex = _headerRow,
FirstDataRowIndex = _firstDataRow,
DateColumn = _dateColumn,
DateKind = _dateKind,
SkipAllZeroRows = _skipAllZero,
DetectCumulativeSwaps = _detectSwaps,
Columns = Enumerable.Range(0, _colCount)
.Where(i => _columns[i].Role != MappingRole.Ignore)
.Select(i => new ColumnMapping
{
Index = i,
Role = _columns[i].Role,
MeterId = _columns[i].MeterId,
CategoryId = _columns[i].CategoryId,
Unit = string.IsNullOrWhiteSpace(_columns[i].Unit) ? null : _columns[i].Unit,
})
.ToList(),
};
private string Cell(int row, int col) =>
row >= 0 && row < _rows.Count && col < _rows[row].Length ? _rows[row][col] : string.Empty;
private string Header(int col) => Cell(_headerRow, col);
private string Sample(int col)
{
for (var r = _firstDataRow; r < _rows.Count && r < _firstDataRow + 20; r++)
{
var value = Cell(r, col);
if (!string.IsNullOrWhiteSpace(value))
{
return value;
}
}
return string.Empty;
}
private string ColLabel(int col)
{
var header = Header(col);
return string.IsNullOrWhiteSpace(header) ? $"Col {col}" : $"Col {col}: {header}";
}
}
src/App/Confirm.cs
+8
View File
@@ -11,4 +11,12 @@ public static class Confirm
.ConfigureAwait(false);
return result == true;
}
/// <summary>A generic yes/cancel confirmation with a caller-supplied confirm-button label.</summary>
public static async Task<bool> ConfirmAsync(IDialogService dialog, string title, string message, string confirmText)
{
var result = await dialog.ShowMessageBoxAsync(title, message, yesText: confirmText, cancelText: "Cancel")
.ConfigureAwait(false);
return result == true;
}
}
src/Infrastructure/Import/CsvImporter.cs
+6
View File
@@ -222,6 +222,12 @@ public sealed class CsvImporter
return true;
}
/// <summary>
/// Reads the raw CSV grid (no profile applied) so the import wizard can show a column preview
/// and let the user map columns before staging. Rows are ragged (their own field counts).
/// </summary>
public static IReadOnlyList<string[]> ReadRawRows(TextReader reader) => ReadRows(reader);
internal static List<string[]> ReadRows(TextReader reader)
{
var config = new CsvConfiguration(CultureInfo.InvariantCulture)