From e23df37a3f0b747b20d7f09102b464f21ba0b9f7 Mon Sep 17 00:00:00 2001 From: Florian Schmidt Date: Sat, 18 Jul 2026 15:41:52 +0200 Subject: [PATCH] Connectors: allow secrets to be entered in the UI, encrypted at rest MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reference-only secrets (SDD §6.4) meant adding a connector required editing a file on the server and restarting the service. In practice that leads to the token being pasted into the env-var *name* field, which fails with "environment variable '' is not set" and gives no hint what went wrong. Add a second storage form, chosen per connector: type the secret in and it is encrypted via ASP.NET Core data protection before it is stored. The env-var reference stays as an equal alternative — this widens the choice rather than replacing it. Exactly one form survives a save, so a stale secret cannot linger and silently win; EndpointSecret.Resolve is the single resolution path. The guarantee that matters is preserved: no plaintext in the database, so pg_dump and JSON exports carry nothing usable. The trust boundary is stated plainly in §6.4 — the key ring is on disk, so this protects against leaked database content, not an attacker who already has the host, which is the same boundary an env var has. Details worth noting: - Key ring defaults to /var/lib/metervault/keys, outside the app directory, because the LXC updater republishes /opt/metervault on every update. Docker gets a named volume. Overridable via MeterVault__DataProtectionKeyPath. - Undecryptable ciphertext (key ring lost) falls back rather than throwing: an ingestion worker on a timer should degrade, not crash. - The stored secret is never sent to the browser; a blank field means "unchanged", not "cleared". - MQTT usernames are stored as-is — §6.4 covers tokens and passwords, and encrypting a username would only blank the field on every edit. - ExportService drops *_enc values: bound to the originating key ring, so useless where an export would be restored. Expect to re-enter after a restore. - HaConnectionTester now takes a resolved token, so the admin UI can test a token that has been typed but not yet saved. SDD §6.4 and §9 updated to describe both forms rather than contradict the code. Claude-Session: https://claude.ai/code/session_01V6joyergfvVLFEizH1hJLd --- CLAUDE.md | 2 +- Directory.Packages.props | 1 + deploy/docker-compose.yml | 6 + docs/SDD.md | 11 +- .../Components/Pages/Admin/Connectors.razor | 126 +++++++++++++++++- src/App/Program.cs | 36 +++++ src/Infrastructure/Backup/ExportService.cs | 22 ++- src/Infrastructure/DependencyInjection.cs | 3 + .../Ingestion/EndpointConfig.cs | 27 ++-- .../Ingestion/EndpointSecret.cs | 29 ++++ .../Ingestion/HaConnectionTester.cs | 21 ++- .../Ingestion/HaEndpointConfig.cs | 21 ++- .../Ingestion/HomeAssistantWebSocketWorker.cs | 7 +- .../Ingestion/HomeAssistantWorker.cs | 4 +- .../Ingestion/MqttIngestionWorker.cs | 9 +- .../MeterVault.Infrastructure.csproj | 1 + .../Security/SecretProtector.cs | 64 +++++++++ .../Ingestion/EndpointSecretTests.cs | 118 ++++++++++++++++ .../Ingestion/HaConnectionTesterTests.cs | 18 +-- .../HomeAssistantWebSocketWorkerTests.cs | 8 +- 20 files changed, 478 insertions(+), 56 deletions(-) create mode 100644 src/Infrastructure/Ingestion/EndpointSecret.cs create mode 100644 src/Infrastructure/Security/SecretProtector.cs create mode 100644 tests/Integration.Tests/Ingestion/EndpointSecretTests.cs diff --git a/CLAUDE.md b/CLAUDE.md index 62cbb09..9250451 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -81,7 +81,7 @@ sources (Tasmota/HA/MQTT/manual/CSV) **Time & DST (SDD §10):** store UTC everywhere; bucket and display in the instance timezone (default `Europe/Berlin`). "Daily cost" boundaries are local-midnight — use `time_bucket(..., 'Europe/Berlin')`. -**Secrets (SDD §6.4):** broker/HA tokens are **never** stored in DB plaintext. `ingestion_endpoint.config` holds a *reference* (env var name / Docker secret path) resolved at runtime. +**Secrets (SDD §6.4):** broker/HA tokens are **never** stored in DB plaintext. Two forms, chosen per connector in the admin UI: a *reference* (`token_env`/`password_env` naming an env var or Docker secret path) resolved at runtime, or *encrypted at rest* (`token_enc`/`password_enc`) via `SecretProtector` over the ASP.NET Core data-protection key ring. Exactly one survives a save; `EndpointSecret.Resolve` is the single resolution path (encrypted wins). The key ring lives outside the app directory (`MeterVault__DataProtectionKeyPath`, default `/var/lib/metervault/keys`) because the LXC updater republishes `/opt/metervault`. `ExportService` drops `*_enc` values — they are bound to the originating key ring. ## Reference-data behaviours the code must reproduce (from `sampledata/`) diff --git a/Directory.Packages.props b/Directory.Packages.props index de217f0..541e99f 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -15,6 +15,7 @@ + diff --git a/deploy/docker-compose.yml b/deploy/docker-compose.yml index b277346..aea4112 100644 --- a/deploy/docker-compose.yml +++ b/deploy/docker-compose.yml @@ -36,9 +36,14 @@ services: # Set true for a populated demo: loads the bundled Energiebilanz dataset on first start # (idempotent). Leave false for a clean instance. MeterVault__SeedReferenceData: ${METERVAULT_SEED:-false} + # Key ring for connector secrets typed into the admin UI. On the named volume below so it + # survives image updates — lose it and every stored token must be re-entered. + MeterVault__DataProtectionKeyPath: /var/lib/metervault/keys # REST API is closed by default. Set a key to enable it (or AllowAnonymousApi on a trusted LAN): # MeterVault__ApiKeys__0: your-secret-key # MeterVault__AllowAnonymousApi: "true" + volumes: + - metervault_keys:/var/lib/metervault/keys ports: - "${METERVAULT_PORT:-8760}:8760" healthcheck: @@ -51,3 +56,4 @@ services: volumes: metervault_db: + metervault_keys: diff --git a/docs/SDD.md b/docs/SDD.md index 721a535..c8634bc 100644 --- a/docs/SDD.md +++ b/docs/SDD.md @@ -413,7 +413,14 @@ Normalized/rolled-up volumes are tiny regardless: daily consumption = 1000 × 36 - Ship the four reference CSVs as built-in example imports and as test fixtures. ### 6.4 Secrets -Tokens/passwords are **never** stored in plaintext in the DB. `ingestion_endpoint.config` holds a *reference* (env var name / Docker secret path); the app resolves at runtime. Document this clearly. +Tokens/passwords are **never** stored in plaintext in the DB. Two storage forms satisfy this, chosen per connector in the admin UI: + +- **By reference** — `ingestion_endpoint.config` names an env var / Docker secret path (`token_env`, `password_env`); the app resolves it at runtime. +- **Encrypted at rest** — the operator types the secret into the connector dialog and it is stored encrypted (`token_enc`, `password_enc`) under the ASP.NET Core data-protection key ring. + +Exactly one form survives a save; switching clears the other. The encrypted form exists because reference-only forced a file edit plus a service restart to add a connector, which in practice led to tokens being pasted into the env-var *name* field. It keeps the guarantee that matters — a `pg_dump` or JSON export carries nothing usable — but note the trust boundary: the key ring is on disk, so it protects against leaked database content, not against an attacker who already has the host. That is the same boundary as an env var, which is equally readable from `/proc`. + +The key ring must be persisted outside the app directory (`MeterVault__DataProtectionKeyPath`, default `/var/lib/metervault/keys`), or a redeploy that replaces the content root will orphan every stored secret. MQTT *usernames* are not secrets and are stored as-is. JSON exports drop `*_enc` values: they are bound to the originating key ring and so are useless where an export would be restored — expect to re-enter secrets after a restore. --- @@ -490,7 +497,7 @@ OpenAPI/Swagger published. API-key auth for automation endpoints; UI uses the se - **Time & DST:** store UTC; bucket and display in the instance timezone (default `Europe/Berlin`). "Daily cost" boundaries are local-midnight — use `time_bucket(..., 'Europe/Berlin')`. - **Auth:** optional built-in local accounts; **reverse-proxy trust** mode honouring `X-Forwarded-User`/`Remote-User` behind Authelia/Traefik; API keys for machine access. Default: single admin user + one ingest API key. - **Observability:** `/healthz`, structured logs (Serilog), optional Prometheus `/metrics`. -- **Config:** environment variables + a settings UI; secrets via env/Docker secrets (never in DB plaintext). +- **Config:** environment variables + a settings UI; secrets via env/Docker secrets or encrypted at rest (never in DB plaintext) — see §6.4. - **i18n:** `en` (default for OSS) + `de`; locale-aware number/currency/date. Ship a German locale that matches the source data conventions. - **Backup:** document `pg_dump`/Timescale backup; provide a full **JSON export/import** for portability. - **Performance:** dashboards read aggregates only; raw reads paginated and time-bounded. diff --git a/src/App/Components/Pages/Admin/Connectors.razor b/src/App/Components/Pages/Admin/Connectors.razor index f4856b5..3b32a76 100644 --- a/src/App/Components/Pages/Admin/Connectors.razor +++ b/src/App/Components/Pages/Admin/Connectors.razor @@ -1,6 +1,7 @@ @page "/admin/connectors" @inject Microsoft.EntityFrameworkCore.IDbContextFactory DbFactory @inject MeterVault.Infrastructure.Ingestion.HaConnectionTester HaTester +@inject MeterVault.Infrastructure.Security.SecretProtector Secrets @inject ISnackbar Snackbar @inject IDialogService DialogService @using Microsoft.EntityFrameworkCore @@ -70,7 +71,22 @@ else @if (_working.Type == EndpointType.HomeAssistant) { - + + @if (_working.UseDirectToken) + { + + + Encrypted before it is stored; database dumps and JSON exports carry nothing usable. + + } + else + { + + + The variable's name, not the token. Set it on the server and restart the app. + + } On: subscribe to state changes and ingest in real time. Off: REST poll on each source's interval. @@ -93,8 +109,21 @@ else - - + + @if (_working.UseDirectCredentials) + { + + + + Encrypted before it is stored; database dumps and JSON exports carry nothing usable. + + } + else + { + + + } } @@ -135,6 +164,10 @@ else { Id = endpoint.Id, Type = endpoint.Type, Name = endpoint.Name, IsEnabled = endpoint.IsEnabled, BaseUrl = ha.BaseUrl, TokenEnv = ha.TokenEnv, UseWebSocket = ha.UseWebSocket, + // Carry the ciphertext through untouched and never send the secret to the browser: + // the field stays blank and only a typed value replaces what is stored. + TokenEnc = ha.TokenEnc, + UseDirectToken = !string.IsNullOrWhiteSpace(ha.TokenEnc), }; } else @@ -145,6 +178,9 @@ else Id = endpoint.Id, Type = endpoint.Type, Name = endpoint.Name, IsEnabled = endpoint.IsEnabled, Host = mqtt.Host, Port = mqtt.Port, Tls = mqtt.Tls, UsernameEnv = mqtt.UsernameEnv, PasswordEnv = mqtt.PasswordEnv, + Username = mqtt.Username, PasswordEnc = mqtt.PasswordEnc, + UseDirectCredentials = + !string.IsNullOrWhiteSpace(mqtt.Username) || !string.IsNullOrWhiteSpace(mqtt.PasswordEnc), ExtraTopics = string.Join(", ", mqtt.ExtraTopics), }; } @@ -157,7 +193,33 @@ else _testResult = null; try { - _testResult = await HaTester.TestAsync(_working.BaseUrl, _working.TokenEnv, _working.TestEntityId); + // Test what the connector would actually use — including a token typed but not yet + // saved, so a bad token is caught before it is stored. + if (_working.UseDirectToken) + { + var token = !string.IsNullOrWhiteSpace(_working.Token) + ? _working.Token + : (Secrets.TryUnprotect(_working.TokenEnc, out var stored) ? stored : null); + + _testResult = string.IsNullOrWhiteSpace(token) + ? new HaTestResult(false, "Enter a token first.") + : await HaTester.TestAsync(_working.BaseUrl, token, _working.TestEntityId); + } + else if (string.IsNullOrWhiteSpace(_working.TokenEnv)) + { + _testResult = new HaTestResult(false, + "Name the environment variable holding the token, or switch on \"Enter the token here\"."); + } + else if (Environment.GetEnvironmentVariable(_working.TokenEnv) is not { Length: > 0 } envToken) + { + _testResult = new HaTestResult(false, + $"Environment variable '{_working.TokenEnv}' is not set on the server. Set it and restart the app, " + + "or switch on \"Enter the token here\" to store the token directly."); + } + else + { + _testResult = await HaTester.TestAsync(_working.BaseUrl, envToken, _working.TestEntityId); + } } finally { @@ -173,15 +235,36 @@ else return; } + if (_working.Type == EndpointType.HomeAssistant + && _working.UseDirectToken + && string.IsNullOrWhiteSpace(_working.Token) + && !_working.HasStoredToken) + { + Snackbar.Add("Enter the token, or switch off \"Enter the token here\" and name an env var.", Severity.Warning); + return; + } + var config = _working.Type == EndpointType.HomeAssistant - ? new HaEndpointConfig { BaseUrl = Trim(_working.BaseUrl), TokenEnv = Trim(_working.TokenEnv), UseWebSocket = _working.UseWebSocket }.ToJson() + ? new HaEndpointConfig + { + BaseUrl = Trim(_working.BaseUrl), + UseWebSocket = _working.UseWebSocket, + // Exactly one storage form survives a save: switching modes clears the other, so a + // stale token cannot linger and silently win at resolution time. + TokenEnv = _working.UseDirectToken ? null : Trim(_working.TokenEnv), + TokenEnc = _working.UseDirectToken ? ProtectOrKeep(_working.Token, _working.TokenEnc) : null, + }.ToJson() : new EndpointConfig { Host = string.IsNullOrWhiteSpace(_working.Host) ? "localhost" : _working.Host.Trim(), Port = _working.Port, Tls = _working.Tls, - UsernameEnv = Trim(_working.UsernameEnv), - PasswordEnv = Trim(_working.PasswordEnv), + UsernameEnv = _working.UseDirectCredentials ? null : Trim(_working.UsernameEnv), + PasswordEnv = _working.UseDirectCredentials ? null : Trim(_working.PasswordEnv), + Username = _working.UseDirectCredentials ? Trim(_working.Username) : null, + PasswordEnc = _working.UseDirectCredentials + ? ProtectOrKeep(_working.Password, _working.PasswordEnc) + : null, ExtraTopics = SplitTopics(_working.ExtraTopics), }.ToJson(); @@ -225,6 +308,13 @@ else private static string? Trim(string? value) => string.IsNullOrWhiteSpace(value) ? null : value.Trim(); + /// + /// Encrypts a newly typed secret, or keeps the stored ciphertext when the field was left blank. + /// The plaintext is never sent to the browser, so blank means "unchanged", not "cleared". + /// + private string? ProtectOrKeep(string? typed, string? existingCiphertext) => + string.IsNullOrWhiteSpace(typed) ? existingCiphertext : Secrets.Protect(typed); + private static IReadOnlyList SplitTopics(string? csv) => string.IsNullOrWhiteSpace(csv) ? [] : [.. csv.Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)]; @@ -241,6 +331,17 @@ else public bool UseWebSocket { get; set; } public string? TestEntityId { get; set; } + /// True to store the token here (encrypted); false to name an env var. + public bool UseDirectToken { get; set; } + + /// Typed token. Always blank on load — a stored secret is never sent to the browser. + public string? Token { get; set; } + + /// Stored ciphertext, round-tripped so leaving blank keeps it. + public string? TokenEnc { get; set; } + + public bool HasStoredToken => !string.IsNullOrWhiteSpace(TokenEnc); + // MQTT broker public string? Host { get; set; } = "localhost"; public int Port { get; set; } = 1883; @@ -248,5 +349,16 @@ else public string? UsernameEnv { get; set; } public string? PasswordEnv { get; set; } public string? ExtraTopics { get; set; } + + public bool UseDirectCredentials { get; set; } + + /// Username entered directly — not a secret, so shown when editing. + public string? Username { get; set; } + + public string? Password { get; set; } + + public string? PasswordEnc { get; set; } + + public bool HasStoredPassword => !string.IsNullOrWhiteSpace(PasswordEnc); } } diff --git a/src/App/Program.cs b/src/App/Program.cs index 70d02f5..61e3782 100644 --- a/src/App/Program.cs +++ b/src/App/Program.cs @@ -3,6 +3,7 @@ using MeterVault.App.Components; using MeterVault.Infrastructure; using MeterVault.Infrastructure.Options; using MeterVault.Infrastructure.Persistence; +using Microsoft.AspNetCore.DataProtection; using Microsoft.EntityFrameworkCore; using MudBlazor.Services; using Serilog; @@ -28,6 +29,41 @@ try ?? "Host=localhost;Port=5432;Database=metervault;Username=metervault;Password=metervault"; builder.Services.AddMeterVaultInfrastructure(connectionString); + // Key ring for connector secrets typed into the admin UI. It must outlive the app directory: + // the LXC updater republishes /opt/metervault on every update, so keys stored beside the + // binaries would be destroyed and every saved token would need re-entering. Override with + // MeterVault__DataProtectionKeyPath (Docker: point it at a mounted volume). + var keyPath = builder.Configuration["MeterVault:DataProtectionKeyPath"]; + if (string.IsNullOrWhiteSpace(keyPath)) + { + keyPath = OperatingSystem.IsWindows() + ? Path.Combine( + Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), "MeterVault", "keys") + : "/var/lib/metervault/keys"; + } + + try + { + Directory.CreateDirectory(keyPath); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + // Falling back beats refusing to boot, but say so plainly: on the fallback path an update + // that replaces the content root loses the keys, and stored secrets stop decrypting. + var fallback = Path.Combine(builder.Environment.ContentRootPath, "keys"); + Log.Warning(ex, + "Cannot create data-protection key ring at {KeyPath}; falling back to {Fallback}. " + + "Secrets entered in the admin UI will not survive a redeploy that replaces the content " + + "root — set MeterVault__DataProtectionKeyPath to a writable persistent directory", + keyPath, fallback); + keyPath = fallback; + Directory.CreateDirectory(keyPath); + } + + builder.Services.AddDataProtection() + .SetApplicationName("MeterVault") + .PersistKeysToFileSystem(new DirectoryInfo(keyPath)); + var options = builder.Configuration.GetSection(MeterVaultOptions.SectionName).Get() ?? new MeterVaultOptions(); if (options.EnableLiveIngestion) diff --git a/src/Infrastructure/Backup/ExportService.cs b/src/Infrastructure/Backup/ExportService.cs index 7d6a0bf..27eb814 100644 --- a/src/Infrastructure/Backup/ExportService.cs +++ b/src/Infrastructure/Backup/ExportService.cs @@ -19,6 +19,25 @@ public sealed class ExportService(MeterVaultDbContext db) private readonly MeterVaultDbContext _db = db; + /// + /// Strips encrypted connector secrets from an export. They are ciphertext, not plaintext, so + /// this is not a §6.4 requirement — but the export exists for portability (§9), and ciphertext + /// is bound to the originating instance's key ring, so it is useless anywhere it could be + /// restored and merely widens the blast radius if the key ring also leaks. Env-var references + /// survive: they name a variable and reveal nothing. Restoring means re-entering the secrets. + /// + private static List RedactSecrets(List endpoints) + { + foreach (var endpoint in endpoints) + { + endpoint.Config = endpoint.Type == EndpointType.HomeAssistant + ? (Ingestion.HaEndpointConfig.Parse(endpoint.Config) with { TokenEnc = null }).ToJson() + : (Ingestion.EndpointConfig.Parse(endpoint.Config) with { PasswordEnc = null }).ToJson(); + } + + return endpoints; + } + public async Task ExportJsonAsync(CancellationToken cancellationToken = default) { // Load without navigation includes so serialization is a clean, cycle-free tree. @@ -27,7 +46,8 @@ public sealed class ExportService(MeterVaultDbContext db) EnergyTypes = await _db.EnergyTypes.AsNoTracking().ToListAsync(cancellationToken).ConfigureAwait(false), CostCategories = await _db.CostCategories.AsNoTracking().ToListAsync(cancellationToken).ConfigureAwait(false), Meters = await _db.Meters.AsNoTracking().ToListAsync(cancellationToken).ConfigureAwait(false), - IngestionEndpoints = await _db.IngestionEndpoints.AsNoTracking().ToListAsync(cancellationToken).ConfigureAwait(false), + IngestionEndpoints = RedactSecrets( + await _db.IngestionEndpoints.AsNoTracking().ToListAsync(cancellationToken).ConfigureAwait(false)), MeterSources = await _db.MeterSources.AsNoTracking().ToListAsync(cancellationToken).ConfigureAwait(false), Tanks = await _db.Tanks.AsNoTracking().ToListAsync(cancellationToken).ConfigureAwait(false), Tariffs = await _db.Tariffs.AsNoTracking().ToListAsync(cancellationToken).ConfigureAwait(false), diff --git a/src/Infrastructure/DependencyInjection.cs b/src/Infrastructure/DependencyInjection.cs index bf7c11a..02ee1a3 100644 --- a/src/Infrastructure/DependencyInjection.cs +++ b/src/Infrastructure/DependencyInjection.cs @@ -37,6 +37,9 @@ public static class DependencyInjection // "Test connection" works without the background workers running. services.AddHttpClient(); services.AddScoped(); + // Singleton: wraps one IDataProtector, and the ingestion workers (themselves singletons) + // resolve connector secrets on every reconnect. + services.AddSingleton(); services.AddScoped(); services.AddScoped(); services.AddScoped(); diff --git a/src/Infrastructure/Ingestion/EndpointConfig.cs b/src/Infrastructure/Ingestion/EndpointConfig.cs index 3f0a3ef..4def8c1 100644 --- a/src/Infrastructure/Ingestion/EndpointConfig.cs +++ b/src/Infrastructure/Ingestion/EndpointConfig.cs @@ -4,9 +4,10 @@ using System.Text.Json.Serialization; namespace MeterVault.Infrastructure.Ingestion; /// -/// The parsed JSON for an MQTT broker. Secrets -/// are stored by reference only (SDD §6.4): / -/// name environment variables resolved at runtime, never plaintext credentials in the database. +/// The parsed JSON for an MQTT broker. Credentials +/// are never held here as plaintext (SDD §6.4): / +/// name environment variables resolved at runtime, or / +/// hold them encrypted under the app's data-protection key ring. /// public sealed record EndpointConfig { @@ -29,6 +30,15 @@ public sealed record EndpointConfig public string? PasswordEnv { get; init; } + /// + /// Username entered directly in the admin UI. Held as-is: §6.4 covers tokens and passwords, and + /// encrypting a username would only blank the field on every edit for no security gain. + /// + public string? Username { get; init; } + + /// Password encrypted by (entered in the admin UI). + public string? PasswordEnc { get; init; } + public static EndpointConfig Parse(string? json) { if (string.IsNullOrWhiteSpace(json)) @@ -48,10 +58,11 @@ public sealed record EndpointConfig public string ToJson() => JsonSerializer.Serialize(this, Options); - public string? ResolveUsername() => Resolve(UsernameEnv); + public string? ResolveUsername(Security.SecretProtector? protector = null) => + !string.IsNullOrWhiteSpace(Username) + ? Username + : EndpointSecret.Resolve(null, UsernameEnv, protector); - public string? ResolvePassword() => Resolve(PasswordEnv); - - private static string? Resolve(string? envVarName) => - string.IsNullOrWhiteSpace(envVarName) ? null : Environment.GetEnvironmentVariable(envVarName); + public string? ResolvePassword(Security.SecretProtector? protector = null) => + EndpointSecret.Resolve(PasswordEnc, PasswordEnv, protector); } diff --git a/src/Infrastructure/Ingestion/EndpointSecret.cs b/src/Infrastructure/Ingestion/EndpointSecret.cs new file mode 100644 index 0000000..99795ae --- /dev/null +++ b/src/Infrastructure/Ingestion/EndpointSecret.cs @@ -0,0 +1,29 @@ +using MeterVault.Infrastructure.Security; + +namespace MeterVault.Infrastructure.Ingestion; + +/// +/// Resolves one connector secret from the two storage forms an endpoint config may use: a value +/// typed into the admin UI and encrypted at rest, or the name of an environment variable resolved +/// at runtime. Neither form keeps plaintext in the database (SDD §6.4). +/// +internal static class EndpointSecret +{ + /// + /// Encrypted wins over the env-var reference when both are present. They are mutually exclusive + /// in the UI, so both being set means a connector was switched from one mode to the other and + /// the write did not clear the old field; honouring the encrypted value keeps the connector on + /// whichever secret was most recently entered. + /// + public static string? Resolve(string? encrypted, string? envVarName, SecretProtector? protector) + { + if (!string.IsNullOrWhiteSpace(encrypted) + && protector is not null + && protector.TryUnprotect(encrypted, out var plaintext)) + { + return plaintext; + } + + return string.IsNullOrWhiteSpace(envVarName) ? null : Environment.GetEnvironmentVariable(envVarName); + } +} diff --git a/src/Infrastructure/Ingestion/HaConnectionTester.cs b/src/Infrastructure/Ingestion/HaConnectionTester.cs index ffe7f56..64a5b5d 100644 --- a/src/Infrastructure/Ingestion/HaConnectionTester.cs +++ b/src/Infrastructure/Ingestion/HaConnectionTester.cs @@ -7,32 +7,31 @@ namespace MeterVault.Infrastructure.Ingestion; public sealed record HaTestResult(bool Ok, string Message, double? SampleValue = null); /// -/// Verifies a Home Assistant connection from the admin UI: checks the base URL + resolved token -/// against GET /api/, and optionally reads one entity's state. Confirms the app can actually -/// read HA before a source is relied upon (SDD §6.2). Token is resolved by reference (env var). +/// Verifies a Home Assistant connection from the admin UI: checks the base URL + token against +/// GET /api/, and optionally reads one entity's state. Confirms the app can actually read HA +/// before a source is relied upon (SDD §6.2). /// +/// +/// Takes the token already resolved rather than a reference to one. The admin UI must be able to +/// test a token that has been typed but not yet saved (so not yet encrypted), and keeping the two +/// storage forms out of here leaves one resolution path in . +/// public sealed class HaConnectionTester(IHttpClientFactory httpClientFactory, ILogger logger) { private readonly IHttpClientFactory _httpClientFactory = httpClientFactory; private readonly ILogger _logger = logger; public async Task TestAsync( - string? baseUrl, string? tokenEnv, string? entityId, CancellationToken cancellationToken = default) + string? baseUrl, string? token, string? entityId, CancellationToken cancellationToken = default) { if (string.IsNullOrWhiteSpace(baseUrl)) { return new HaTestResult(false, "Base URL is required."); } - if (string.IsNullOrWhiteSpace(tokenEnv)) - { - return new HaTestResult(false, "Token env-var name is required (the token is resolved from it at runtime)."); - } - - var token = Environment.GetEnvironmentVariable(tokenEnv); if (string.IsNullOrWhiteSpace(token)) { - return new HaTestResult(false, $"Environment variable '{tokenEnv}' is not set on the server."); + return new HaTestResult(false, "No token available to test."); } var client = _httpClientFactory.CreateClient(); diff --git a/src/Infrastructure/Ingestion/HaEndpointConfig.cs b/src/Infrastructure/Ingestion/HaEndpointConfig.cs index ad78948..2504b0a 100644 --- a/src/Infrastructure/Ingestion/HaEndpointConfig.cs +++ b/src/Infrastructure/Ingestion/HaEndpointConfig.cs @@ -1,12 +1,14 @@ using System.Text.Json; using System.Text.Json.Serialization; +using MeterVault.Infrastructure.Security; namespace MeterVault.Infrastructure.Ingestion; /// /// The parsed JSON for a Home Assistant -/// connection (SDD §6.2). The long-lived token is stored by reference only: -/// names an environment variable resolved at runtime — never the token itself (SDD §6.4). +/// connection (SDD §6.2). The long-lived token is never held here as plaintext (SDD §6.4): either +/// names an environment variable resolved at runtime, or +/// holds it encrypted under the app's data-protection key ring. /// public sealed record HaEndpointConfig { @@ -22,6 +24,12 @@ public sealed record HaEndpointConfig /// Name of the environment variable holding the long-lived access token. public string? TokenEnv { get; init; } + /// + /// The long-lived access token, encrypted by . Set when + /// the operator typed the token into the admin UI instead of naming an environment variable. + /// + public string? TokenEnc { get; init; } + /// /// When true, a persistent WebSocket subscription pushes state changes in real time /// (); when false (default) the REST poll worker @@ -48,7 +56,10 @@ public sealed record HaEndpointConfig public string ToJson() => JsonSerializer.Serialize(this, Options); - /// Resolves the token from the referenced environment variable (null if unset). - public string? ResolveToken() => - string.IsNullOrWhiteSpace(TokenEnv) ? null : Environment.GetEnvironmentVariable(TokenEnv); + /// + /// Resolves the token: the encrypted value when one was entered directly, otherwise the + /// referenced environment variable. Null when neither yields anything. + /// + public string? ResolveToken(SecretProtector? protector = null) => + EndpointSecret.Resolve(TokenEnc, TokenEnv, protector); } diff --git a/src/Infrastructure/Ingestion/HomeAssistantWebSocketWorker.cs b/src/Infrastructure/Ingestion/HomeAssistantWebSocketWorker.cs index 3c37d27..d8c8873 100644 --- a/src/Infrastructure/Ingestion/HomeAssistantWebSocketWorker.cs +++ b/src/Infrastructure/Ingestion/HomeAssistantWebSocketWorker.cs @@ -17,10 +17,12 @@ namespace MeterVault.Infrastructure.Ingestion; /// subscribes to state_changed events, and ingests changes for the endpoint's configured /// entities as they happen. Reconnects with capped backoff. Endpoints without UseWebSocket stay /// on the REST poll worker () — each endpoint is served by exactly one. -/// Tokens are resolved from environment variables named in the endpoint config, never stored plaintext. +/// Tokens are resolved from the endpoint config — an env-var reference or an encrypted value — +/// never stored as plaintext. /// public sealed class HomeAssistantWebSocketWorker( IServiceScopeFactory scopeFactory, + Security.SecretProtector secrets, ILogger logger) : BackgroundService { private static readonly TimeSpan SuperviseInterval = TimeSpan.FromSeconds(15); @@ -29,6 +31,7 @@ public sealed class HomeAssistantWebSocketWorker( private static readonly TimeSpan EntityMapTtl = TimeSpan.FromSeconds(30); private readonly IServiceScopeFactory _scopeFactory = scopeFactory; + private readonly Security.SecretProtector _secrets = secrets; private readonly ILogger _logger = logger; private readonly ConcurrentDictionary _connections = new(); @@ -107,7 +110,7 @@ public sealed class HomeAssistantWebSocketWorker( return; // switched to poll mode. } - token = config.ResolveToken(); + token = config.ResolveToken(_secrets); } if (string.IsNullOrWhiteSpace(config.BaseUrl) || string.IsNullOrWhiteSpace(token)) diff --git a/src/Infrastructure/Ingestion/HomeAssistantWorker.cs b/src/Infrastructure/Ingestion/HomeAssistantWorker.cs index 5a8f7d8..cc32280 100644 --- a/src/Infrastructure/Ingestion/HomeAssistantWorker.cs +++ b/src/Infrastructure/Ingestion/HomeAssistantWorker.cs @@ -17,12 +17,14 @@ namespace MeterVault.Infrastructure.Ingestion; public sealed class HomeAssistantWorker( IServiceScopeFactory scopeFactory, IHttpClientFactory httpClientFactory, + Security.SecretProtector secrets, ILogger logger) : BackgroundService { private static readonly TimeSpan TickInterval = TimeSpan.FromSeconds(10); private readonly IServiceScopeFactory _scopeFactory = scopeFactory; private readonly IHttpClientFactory _httpClientFactory = httpClientFactory; + private readonly Security.SecretProtector _secrets = secrets; private readonly ILogger _logger = logger; private readonly ConcurrentDictionary _nextPoll = new(); @@ -104,7 +106,7 @@ public sealed class HomeAssistantWorker( MeterSource source, SourceConfig config, CancellationToken cancellationToken) { var endpointConfig = HaEndpointConfig.Parse(endpoint.Config); - var token = endpointConfig.ResolveToken(); + var token = endpointConfig.ResolveToken(_secrets); if (string.IsNullOrWhiteSpace(endpointConfig.BaseUrl) || string.IsNullOrWhiteSpace(token) || string.IsNullOrWhiteSpace(config.EntityId)) diff --git a/src/Infrastructure/Ingestion/MqttIngestionWorker.cs b/src/Infrastructure/Ingestion/MqttIngestionWorker.cs index 8ac3243..7c1de0b 100644 --- a/src/Infrastructure/Ingestion/MqttIngestionWorker.cs +++ b/src/Infrastructure/Ingestion/MqttIngestionWorker.cs @@ -16,11 +16,14 @@ namespace MeterVault.Infrastructure.Ingestion; /// are logged and retried on the next tick rather than crashing the app. /// public sealed class MqttIngestionWorker( - IServiceScopeFactory scopeFactory, ILogger logger) : BackgroundService + IServiceScopeFactory scopeFactory, + Security.SecretProtector secrets, + ILogger logger) : BackgroundService { private static readonly TimeSpan ReconnectInterval = TimeSpan.FromSeconds(15); private readonly IServiceScopeFactory _scopeFactory = scopeFactory; + private readonly Security.SecretProtector _secrets = secrets; private readonly ILogger _logger = logger; private readonly MqttClientFactory _factory = new(); private readonly ConcurrentDictionary _clients = new(); @@ -132,10 +135,10 @@ public sealed class MqttIngestionWorker( .WithTcpServer(config.Host, config.Port) .WithCleanSession(); - var username = config.ResolveUsername(); + var username = config.ResolveUsername(_secrets); if (!string.IsNullOrWhiteSpace(username)) { - builder = builder.WithCredentials(username, config.ResolvePassword() ?? string.Empty); + builder = builder.WithCredentials(username, config.ResolvePassword(_secrets) ?? string.Empty); } if (config.Tls) diff --git a/src/Infrastructure/MeterVault.Infrastructure.csproj b/src/Infrastructure/MeterVault.Infrastructure.csproj index 0702083..117b4ae 100644 --- a/src/Infrastructure/MeterVault.Infrastructure.csproj +++ b/src/Infrastructure/MeterVault.Infrastructure.csproj @@ -8,6 +8,7 @@ + diff --git a/src/Infrastructure/Security/SecretProtector.cs b/src/Infrastructure/Security/SecretProtector.cs new file mode 100644 index 0000000..e02cd5b --- /dev/null +++ b/src/Infrastructure/Security/SecretProtector.cs @@ -0,0 +1,64 @@ +using System.Security.Cryptography; +using Microsoft.AspNetCore.DataProtection; + +namespace MeterVault.Infrastructure.Security; + +/// +/// Encrypts connector secrets that an operator types into the admin UI, so +/// ingestion_endpoint.config holds ciphertext rather than the token itself. +/// +/// +/// SDD §6.4 requires that tokens are never in the database as plaintext. Naming an environment +/// variable satisfies that but forces a file edit plus a service restart to add a connector, which +/// is hostile enough that people paste the token into the name field instead. Encrypting at rest +/// keeps the guarantee that matters — a pg_dump or JSON export carries nothing usable — while +/// letting the token be entered where it is configured. +/// +/// The key ring lives on disk outside the database, so this protects against leaked database +/// content, not against an attacker who already has the host: they can read the keys and decrypt. +/// That is the same trust boundary as an env var, which is equally readable from /proc. +/// Losing the key ring makes existing secrets undecryptable, and they must be re-entered. +/// +public sealed class SecretProtector +{ + // Changing this string orphans every secret encrypted under the old value. + private const string Purpose = "MeterVault.IngestionEndpoint.Secrets.v1"; + + private readonly IDataProtector _protector; + + public SecretProtector(IDataProtectionProvider provider) + { + ArgumentNullException.ThrowIfNull(provider); + _protector = provider.CreateProtector(Purpose); + } + + public string Protect(string plaintext) + { + ArgumentException.ThrowIfNullOrWhiteSpace(plaintext); + return _protector.Protect(plaintext); + } + + /// + /// Decrypts a stored secret. Returns false rather than throwing when the ciphertext cannot be + /// read — a rotated-away or restored-without-the-key-ring deployment should degrade to "this + /// connector has no usable secret" and surface that, not crash an ingestion worker on a timer. + /// + public bool TryUnprotect(string? ciphertext, out string? plaintext) + { + plaintext = null; + if (string.IsNullOrWhiteSpace(ciphertext)) + { + return false; + } + + try + { + plaintext = _protector.Unprotect(ciphertext); + return true; + } + catch (CryptographicException) + { + return false; + } + } +} diff --git a/tests/Integration.Tests/Ingestion/EndpointSecretTests.cs b/tests/Integration.Tests/Ingestion/EndpointSecretTests.cs new file mode 100644 index 0000000..e29de77 --- /dev/null +++ b/tests/Integration.Tests/Ingestion/EndpointSecretTests.cs @@ -0,0 +1,118 @@ +using MeterVault.Infrastructure.Ingestion; +using MeterVault.Infrastructure.Security; +using Microsoft.AspNetCore.DataProtection; + +namespace MeterVault.Integration.Tests.Ingestion; + +/// +/// Connector secrets may be stored two ways (SDD §6.4): encrypted at rest after being typed into +/// the admin UI, or as the name of an environment variable resolved at runtime. These pin which +/// form wins and, more importantly, that neither form ever leaves plaintext in the config JSON. +/// +public sealed class EndpointSecretTests +{ + private const string UnsetVar = "METERVAULT_DEFINITELY_UNSET_TOKEN_VAR"; + + private static SecretProtector NewProtector() => new(new EphemeralDataProtectionProvider()); + + [Fact] + public void Encrypted_token_round_trips_and_is_not_plaintext_in_the_config() + { + var protector = NewProtector(); + var config = new HaEndpointConfig + { + BaseUrl = "http://ha.local:8123", + TokenEnc = protector.Protect("super-secret-token"), + }; + + Assert.Equal("super-secret-token", config.ResolveToken(protector)); + + // This JSON is what lands in ingestion_endpoint.config, in pg_dump, and in a JSON export. + Assert.DoesNotContain("super-secret-token", config.ToJson(), StringComparison.Ordinal); + } + + [Fact] + public void Env_var_is_used_when_no_encrypted_token_is_present() + { + var variable = $"MV_TEST_TOKEN_{Guid.NewGuid():N}"; + Environment.SetEnvironmentVariable(variable, "from-the-environment"); + try + { + var config = new HaEndpointConfig { TokenEnv = variable }; + Assert.Equal("from-the-environment", config.ResolveToken(NewProtector())); + } + finally + { + Environment.SetEnvironmentVariable(variable, null); + } + } + + [Fact] + public void Encrypted_token_wins_when_both_forms_are_set() + { + var protector = NewProtector(); + var variable = $"MV_TEST_TOKEN_{Guid.NewGuid():N}"; + Environment.SetEnvironmentVariable(variable, "from-the-environment"); + try + { + var config = new HaEndpointConfig + { + TokenEnv = variable, + TokenEnc = protector.Protect("typed-in-the-ui"), + }; + + Assert.Equal("typed-in-the-ui", config.ResolveToken(protector)); + } + finally + { + Environment.SetEnvironmentVariable(variable, null); + } + } + + [Fact] + public void Undecryptable_ciphertext_falls_back_instead_of_throwing() + { + // A key ring restored without its keys: the worker must degrade, not crash on a timer. + var config = new HaEndpointConfig { TokenEnc = "not-valid-ciphertext", TokenEnv = UnsetVar }; + + var exception = Record.Exception(() => config.ResolveToken(NewProtector())); + + Assert.Null(exception); + Assert.Null(config.ResolveToken(NewProtector())); + } + + [Fact] + public void Resolving_without_a_protector_still_reads_the_env_var() + { + // ResolveToken() is called with no protector in unit contexts; the env-var path must work. + var variable = $"MV_TEST_TOKEN_{Guid.NewGuid():N}"; + Environment.SetEnvironmentVariable(variable, "plain-env"); + try + { + Assert.Equal("plain-env", new HaEndpointConfig { TokenEnv = variable }.ResolveToken()); + } + finally + { + Environment.SetEnvironmentVariable(variable, null); + } + } + + [Fact] + public void Mqtt_password_is_encrypted_while_username_stays_readable() + { + var protector = NewProtector(); + var config = new EndpointConfig + { + Host = "broker.local", + Username = "metervault", + PasswordEnc = protector.Protect("broker-password"), + }; + + Assert.Equal("metervault", config.ResolveUsername(protector)); + Assert.Equal("broker-password", config.ResolvePassword(protector)); + + var json = config.ToJson(); + Assert.DoesNotContain("broker-password", json, StringComparison.Ordinal); + Assert.Contains("metervault", json, StringComparison.Ordinal); + } +} diff --git a/tests/Integration.Tests/Ingestion/HaConnectionTesterTests.cs b/tests/Integration.Tests/Ingestion/HaConnectionTesterTests.cs index 877d6ed..bb7c312 100644 --- a/tests/Integration.Tests/Ingestion/HaConnectionTesterTests.cs +++ b/tests/Integration.Tests/Ingestion/HaConnectionTesterTests.cs @@ -13,29 +13,19 @@ public sealed class HaConnectionTesterTests [Fact] public async Task Missing_base_url_fails_without_network() { - var result = await NewTester().TestAsync(baseUrl: "", tokenEnv: "SOME_TOKEN", entityId: null); + var result = await NewTester().TestAsync(baseUrl: "", token: "a-token", entityId: null); Assert.False(result.Ok); Assert.Contains("Base URL", result.Message, StringComparison.OrdinalIgnoreCase); } [Fact] - public async Task Missing_token_env_fails_without_network() + public async Task Missing_token_fails_without_network() { - var result = await NewTester().TestAsync(baseUrl: "http://ha.local:8123", tokenEnv: "", entityId: null); + var result = await NewTester().TestAsync(baseUrl: "http://ha.local:8123", token: "", entityId: null); Assert.False(result.Ok); - Assert.Contains("env-var", result.Message, StringComparison.OrdinalIgnoreCase); - } - - [Fact] - public async Task Unset_token_env_var_fails_without_network() - { - var result = await NewTester().TestAsync( - baseUrl: "http://ha.local:8123", tokenEnv: "METERVAULT_DEFINITELY_UNSET_TOKEN_VAR", entityId: null); - - Assert.False(result.Ok); - Assert.Contains("is not set", result.Message, StringComparison.OrdinalIgnoreCase); + Assert.Contains("token", result.Message, StringComparison.OrdinalIgnoreCase); } private static HaConnectionTester NewTester() => diff --git a/tests/Integration.Tests/Ingestion/HomeAssistantWebSocketWorkerTests.cs b/tests/Integration.Tests/Ingestion/HomeAssistantWebSocketWorkerTests.cs index 4635aa2..f7e0132 100644 --- a/tests/Integration.Tests/Ingestion/HomeAssistantWebSocketWorkerTests.cs +++ b/tests/Integration.Tests/Ingestion/HomeAssistantWebSocketWorkerTests.cs @@ -66,7 +66,9 @@ public sealed class HomeAssistantWebSocketWorkerTests(TimescaleFixture fx) await using var provider = BuildProvider(fx.ConnectionString); var worker = new HomeAssistantWebSocketWorker( - provider.GetRequiredService(), NullLogger.Instance); + provider.GetRequiredService(), + provider.GetRequiredService(), + NullLogger.Instance); await worker.StartAsync(CancellationToken.None); try @@ -107,6 +109,10 @@ public sealed class HomeAssistantWebSocketWorkerTests(TimescaleFixture fx) .UseSnakeCaseNamingConvention()); services.AddScoped(sp => sp.GetRequiredService>().CreateDbContext()); services.AddScoped(); + // Ephemeral keys: this test's token comes from an env var, so nothing needs to outlive the run. + services.AddSingleton( + new Microsoft.AspNetCore.DataProtection.EphemeralDataProtectionProvider()); + services.AddSingleton(); return services.BuildServiceProvider(); }