Fix defects found auditing the ingestion, import and connector changes
ci / build-test (push) Successful in 1m16s
ci / build-test (push) Successful in 1m16s
An audit of this session's commits found several real problems, three of which
lose or expose data. Ordered by severity.
Live recompute was not atomic. RecomputeMeterAsync clears a meter's series with
ExecuteDelete, which commits by itself when no transaction is ambient, and only
then adds the rebuilt rows. Between the two the meter had *no* consumption:
a dashboard read reported zero, and a crash or cancelled request made the loss
permanent, for data the SDD treats as the long-term source of truth (§5.5).
Import and the events API already wrapped their recomputes; live ingestion,
which I added this session, did not. Now shares one transaction, joining an
ambient one rather than nesting.
The MQTT backfill migration counted brokers without regard to is_enabled. One
live broker plus a disabled leftover counted two, declined to backfill, and left
those sources unbound — which under endpoint-scoped routing means silently and
permanently dead. The "two or more is ambiguous" reasoning did not hold there:
the worker only ever connected to enabled endpoints. Corrected by a follow-up
migration rather than an edit, since the original may already have run; it
touches only rows still NULL, so hand-made bindings are safe.
A mapping edited after a dry run committed the *old* staged rows under the
*new* mapping. Readings went to the previous meter while the batch recorded the
current mapping — wrong data, provenance contradicting it, no exception. The
earlier fix re-validated but did not detect staleness. Commit now compares the
mapping against the one the preview was staged under and refuses.
"Test connection" sent a stored token to whatever Base URL was in the dialog.
Encrypting secrets at rest means the UI can decrypt what the operator can no
longer read, so this turned the button into an exfiltration primitive: point it
at any host, the token arrives as a Bearer header. A stored token now only goes
to the origin it was saved for; testing elsewhere requires typing it again.
A source that cannot ingest looked identical to a healthy one. Endpoint-scoped
routing made unbound and mis-bound sources silently dead, while the Sources tab
showed no connector at all and the delete dialog still promised sources would be
"unlinked". Added a Connector column that names the fault, stopped offering
disabled connectors (both workers filter on IsEnabled), and made the delete
warning say ingestion stops.
Virtual meters rendered four zero tiles: they evaluate on read and only
materialize when a cost category references them (§14.1), so summing
consumption is a confident lie about a working meter. They now report nothing
and the page explains why.
Re-importing an overlapping file failed at the database with EF's "An error
occurred while saving the entity changes", naming neither meter nor date — the
diagnosis problem a3af483 set out to fix, via the path its guard could not see.
Checked up front now, bounded by each meter's staged range.
The LXC updater left the service stopped on any failure. set -e plus an
explicit stop means Restart=always does not apply, so an OOM-killed publish or
a brief Gitea outage took MeterVault down until someone noticed. An EXIT trap
restarts the previous build and says so.
Claude-Session: https://claude.ai/code/session_01V6joyergfvVLFEizH1hJLd
This commit is contained in:
@@ -168,6 +168,8 @@ else
|
||||
// the field stays blank and only a typed value replaces what is stored.
|
||||
TokenEnc = ha.TokenEnc,
|
||||
UseDirectToken = !string.IsNullOrWhiteSpace(ha.TokenEnc),
|
||||
// The host the stored token was saved against; a stored token is never sent anywhere else.
|
||||
SavedBaseUrl = ha.BaseUrl,
|
||||
};
|
||||
}
|
||||
else
|
||||
@@ -197,13 +199,29 @@ else
|
||||
// saved, so a bad token is caught before it is stored.
|
||||
if (_working.UseDirectToken)
|
||||
{
|
||||
var token = !string.IsNullOrWhiteSpace(_working.Token)
|
||||
? _working.Token
|
||||
: (Secrets.TryUnprotect(_working.TokenEnc, out var stored) ? stored : null);
|
||||
|
||||
_testResult = string.IsNullOrWhiteSpace(token)
|
||||
? new HaTestResult(false, "Enter a token first.")
|
||||
: await HaTester.TestAsync(_working.BaseUrl, token, _working.TestEntityId);
|
||||
if (!string.IsNullOrWhiteSpace(_working.Token))
|
||||
{
|
||||
_testResult = await HaTester.TestAsync(_working.BaseUrl, _working.Token, _working.TestEntityId);
|
||||
}
|
||||
else if (!SameOrigin(_working.BaseUrl, _working.SavedBaseUrl))
|
||||
{
|
||||
// Storing the token encrypted means the UI can decrypt something the operator
|
||||
// can no longer read. Sending it to a Base URL edited in this dialog would turn
|
||||
// "Test connection" into an exfiltration primitive — point it at any host and the
|
||||
// token arrives as a Bearer header. A stored secret only ever goes to the origin
|
||||
// it was saved for; testing elsewhere means typing the token again.
|
||||
_testResult = new HaTestResult(false,
|
||||
"Base URL differs from the saved one. Re-enter the token to test against a different host — "
|
||||
+ "a stored token is only sent to the host it was saved for.");
|
||||
}
|
||||
else if (Secrets.TryUnprotect(_working.TokenEnc, out var stored) && stored is { Length: > 0 })
|
||||
{
|
||||
_testResult = await HaTester.TestAsync(_working.BaseUrl, stored, _working.TestEntityId);
|
||||
}
|
||||
else
|
||||
{
|
||||
_testResult = new HaTestResult(false, "Enter a token first.");
|
||||
}
|
||||
}
|
||||
else if (string.IsNullOrWhiteSpace(_working.TokenEnv))
|
||||
{
|
||||
@@ -295,7 +313,11 @@ else
|
||||
{
|
||||
await using var db = await DbFactory.CreateDbContextAsync();
|
||||
var sourceCount = await db.MeterSources.CountAsync(s => s.EndpointId == endpoint.Id);
|
||||
var note = sourceCount > 0 ? $" {sourceCount} source(s) reference it and will be unlinked." : "";
|
||||
// Routing is endpoint-scoped, so "unlinked" now means those sources stop ingesting entirely
|
||||
// rather than falling back to any broker. Say so plainly.
|
||||
var note = sourceCount > 0
|
||||
? $" {sourceCount} source(s) use it and will stop ingesting until reassigned to another connector."
|
||||
: "";
|
||||
if (!await Confirm.DeleteAsync(DialogService, "Delete connector", $"Delete '{endpoint.Name}'?{note}"))
|
||||
{
|
||||
return;
|
||||
@@ -308,6 +330,18 @@ else
|
||||
|
||||
private static string? Trim(string? value) => string.IsNullOrWhiteSpace(value) ? null : value.Trim();
|
||||
|
||||
/// <summary>
|
||||
/// Whether two URLs address the same host. Compares scheme, host and port rather than the raw
|
||||
/// string, so a trailing slash or a path tweak does not force the token to be re-typed. Fails
|
||||
/// closed: anything unparsable counts as a different origin.
|
||||
/// </summary>
|
||||
private static bool SameOrigin(string? a, string? b) =>
|
||||
Uri.TryCreate(a, UriKind.Absolute, out var left)
|
||||
&& Uri.TryCreate(b, UriKind.Absolute, out var right)
|
||||
&& string.Equals(left.Scheme, right.Scheme, StringComparison.OrdinalIgnoreCase)
|
||||
&& string.Equals(left.Host, right.Host, StringComparison.OrdinalIgnoreCase)
|
||||
&& left.Port == right.Port;
|
||||
|
||||
/// <summary>
|
||||
/// Encrypts a newly typed secret, or keeps the stored ciphertext when the field was left blank.
|
||||
/// The plaintext is never sent to the browser, so blank means "unchanged", not "cleared".
|
||||
@@ -340,6 +374,9 @@ else
|
||||
/// <summary>Stored ciphertext, round-tripped so leaving <see cref="Token"/> blank keeps it.</summary>
|
||||
public string? TokenEnc { get; set; }
|
||||
|
||||
/// <summary>Base URL as saved, so an edited one can be told from the token's own host.</summary>
|
||||
public string? SavedBaseUrl { get; set; }
|
||||
|
||||
public bool HasStoredToken => !string.IsNullOrWhiteSpace(TokenEnc);
|
||||
|
||||
// MQTT broker
|
||||
|
||||
@@ -243,6 +243,7 @@
|
||||
private List<Meter> _meters = [];
|
||||
private List<CostCategory> _categories = [];
|
||||
private StagedImport? _staged;
|
||||
private string? _stagedMapping;
|
||||
private List<string> _validationErrors = [];
|
||||
private bool _committing;
|
||||
|
||||
@@ -265,6 +266,7 @@
|
||||
_columns = Enumerable.Range(0, _colCount).Select(_ => new ColumnState()).ToArray();
|
||||
_dateColumn = 0;
|
||||
_staged = null;
|
||||
_stagedMapping = null;
|
||||
_validationErrors = [];
|
||||
|
||||
await using var db = await DbFactory.CreateDbContextAsync();
|
||||
@@ -283,8 +285,23 @@
|
||||
|
||||
using var reader = new StringReader(_csvText ?? string.Empty);
|
||||
_staged = CsvImporter.Stage(BuildProfile(), reader);
|
||||
_stagedMapping = BuildMappingJson();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The mapping as persisted on the batch for provenance, and — compared against the mapping the
|
||||
/// preview was staged under — the check that the two still agree.
|
||||
/// </summary>
|
||||
private string BuildMappingJson() => JsonSerializer.Serialize(new
|
||||
{
|
||||
dateColumn = _dateColumn,
|
||||
dateKind = _dateKind.ToString(),
|
||||
firstDataRow = _firstDataRow,
|
||||
columns = _columns
|
||||
.Select((c, i) => new { index = i, role = c.Role.ToString(), c.MeterId, c.CategoryId, c.Unit })
|
||||
.Where(c => c.role != nameof(MappingRole.Ignore)),
|
||||
});
|
||||
|
||||
private async Task CommitAsync()
|
||||
{
|
||||
if (_staged is null || _staged.TotalRows == 0)
|
||||
@@ -300,19 +317,22 @@
|
||||
return;
|
||||
}
|
||||
|
||||
// The staged rows were built from the mapping as it stood at preview time. Editing a target
|
||||
// afterwards leaves them pointing at the old meter while the batch would record the new
|
||||
// mapping — wrong data, with a provenance record that contradicts it and no error to notice.
|
||||
// Nothing here can re-derive the rows, so refuse rather than write either version.
|
||||
var mappingJson = BuildMappingJson();
|
||||
if (!string.Equals(mappingJson, _stagedMapping, StringComparison.Ordinal))
|
||||
{
|
||||
_staged = null;
|
||||
_stagedMapping = null;
|
||||
_validationErrors = ["The mapping changed after the preview. Run the dry run again, then commit."];
|
||||
return;
|
||||
}
|
||||
|
||||
_committing = true;
|
||||
try
|
||||
{
|
||||
var mappingJson = JsonSerializer.Serialize(new
|
||||
{
|
||||
dateColumn = _dateColumn,
|
||||
dateKind = _dateKind.ToString(),
|
||||
firstDataRow = _firstDataRow,
|
||||
columns = _columns
|
||||
.Select((c, i) => new { index = i, role = c.Role.ToString(), c.MeterId, c.CategoryId, c.Unit })
|
||||
.Where(c => c.role != nameof(MappingRole.Ignore)),
|
||||
});
|
||||
|
||||
var batchId = await ImportService.CommitAsync(_staged, _fileName, mappingJson);
|
||||
Snackbar.Add($"Imported batch #{batchId}: {_staged.TotalRows} rows staged. Consumption recomputed.", Severity.Success);
|
||||
Nav.NavigateTo("/import");
|
||||
|
||||
@@ -100,6 +100,14 @@ else
|
||||
}
|
||||
}
|
||||
|
||||
@if (_periods is null && _detail.Mode == MeterMode.Virtual)
|
||||
{
|
||||
<MudAlert Severity="Severity.Info" Dense="true" Class="mb-2">
|
||||
Virtual meter — its value is an expression over other meters, evaluated when read, so it has
|
||||
no stored series of its own. See <MudLink Href="/trends">Trends</MudLink> for its figures.
|
||||
</MudAlert>
|
||||
}
|
||||
|
||||
<MudExpansionPanels Elevation="0" Class="mb-2">
|
||||
<MudExpansionPanel Text="Meter register details">
|
||||
<div class="d-flex flex-wrap" style="gap:2rem">
|
||||
@@ -243,13 +251,27 @@ else
|
||||
else
|
||||
{
|
||||
<MudSimpleTable Dense="true" Hover="true">
|
||||
<thead><tr><th>Type</th><th>Target</th><th>Enabled</th><th>Last seen</th><th style="text-align:right">Last value</th><th>Status</th><th style="text-align:right">Actions</th></tr></thead>
|
||||
<thead><tr><th>Type</th><th>Target</th><th>Connector</th><th>Enabled</th><th>Last seen</th><th style="text-align:right">Last value</th><th>Status</th><th style="text-align:right">Actions</th></tr></thead>
|
||||
<tbody>
|
||||
@foreach (var s in _sources)
|
||||
{
|
||||
<tr>
|
||||
<td>@s.SourceType</td>
|
||||
<td>@SourceTarget(s)</td>
|
||||
<td>
|
||||
@{ var problem = ConnectorProblem(s); }
|
||||
@if (problem is null)
|
||||
{
|
||||
@(_endpoints.FirstOrDefault(e => e.Id == s.EndpointId)?.Name ?? "—")
|
||||
}
|
||||
else
|
||||
{
|
||||
<MudTooltip Text="@problem">
|
||||
<MudChip T="string" Size="Size.Small" Color="Color.Error" Variant="Variant.Text"
|
||||
Icon="@Icons.Material.Filled.LinkOff">@problem</MudChip>
|
||||
</MudTooltip>
|
||||
}
|
||||
</td>
|
||||
<td>@(s.IsEnabled ? "yes" : "no")</td>
|
||||
<td>@(s.LastSeenAt?.ToString("yyyy-MM-dd HH:mm") ?? "—")</td>
|
||||
<td style="text-align:right">@(s.LastValue is { } v ? Format.Number(v, 2) : "—")</td>
|
||||
@@ -448,6 +470,12 @@ else
|
||||
Snackbar.Add($"'{selected.Name}' is a {selected.Type} connector; a {_sourceEdit.SourceType} source needs {needed}.", Severity.Error);
|
||||
return;
|
||||
}
|
||||
|
||||
if (!selected.IsEnabled)
|
||||
{
|
||||
Snackbar.Add($"'{selected.Name}' is disabled, so this source would never ingest. Enable it first.", Severity.Error);
|
||||
return;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
@@ -527,8 +555,33 @@ else
|
||||
_ => null,
|
||||
};
|
||||
|
||||
// Only enabled connectors can ingest: both MQTT and HA workers filter on IsEnabled, so offering
|
||||
// a disabled one would produce a source that saves cleanly and then never runs.
|
||||
private List<IngestionEndpoint> ConnectorsFor(EndpointType type) =>
|
||||
_endpoints.Where(e => e.Type == type).ToList();
|
||||
_endpoints.Where(e => e.Type == type && e.IsEnabled).ToList();
|
||||
|
||||
/// <summary>
|
||||
/// Why this source cannot ingest, or null if it can. Routing is endpoint-scoped, so an unbound
|
||||
/// or mis-bound source is silently dead — and deleting a connector unlinks its sources, which
|
||||
/// used to be harmless. Without this column such a source is indistinguishable from a healthy
|
||||
/// one at "Enabled: yes".
|
||||
/// </summary>
|
||||
private string? ConnectorProblem(MeterSource source)
|
||||
{
|
||||
if (RequiredEndpointType(source.SourceType) is not { } needed)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var endpoint = _endpoints.FirstOrDefault(e => e.Id == source.EndpointId);
|
||||
return endpoint switch
|
||||
{
|
||||
null => "no connector — never ingests",
|
||||
{ IsEnabled: false } => $"'{endpoint.Name}' is disabled",
|
||||
_ when endpoint.Type != needed => $"'{endpoint.Name}' is {endpoint.Type}, needs {needed}",
|
||||
_ => null,
|
||||
};
|
||||
}
|
||||
|
||||
// Changing the source type can invalidate the chosen connector (an HA connector cannot serve an
|
||||
// MQTT source), so drop a selection that no longer fits rather than saving a mismatched pair.
|
||||
|
||||
Reference in New Issue
Block a user