Connectors: allow secrets to be entered in the UI, encrypted at rest

Reference-only secrets (SDD §6.4) meant adding a connector required editing a
file on the server and restarting the service. In practice that leads to the
token being pasted into the env-var *name* field, which fails with "environment
variable '<token>' is not set" and gives no hint what went wrong.

Add a second storage form, chosen per connector: type the secret in and it is
encrypted via ASP.NET Core data protection before it is stored. The env-var
reference stays as an equal alternative — this widens the choice rather than
replacing it. Exactly one form survives a save, so a stale secret cannot linger
and silently win; EndpointSecret.Resolve is the single resolution path.

The guarantee that matters is preserved: no plaintext in the database, so
pg_dump and JSON exports carry nothing usable. The trust boundary is stated
plainly in §6.4 — the key ring is on disk, so this protects against leaked
database content, not an attacker who already has the host, which is the same
boundary an env var has.

Details worth noting:
- Key ring defaults to /var/lib/metervault/keys, outside the app directory,
  because the LXC updater republishes /opt/metervault on every update. Docker
  gets a named volume. Overridable via MeterVault__DataProtectionKeyPath.
- Undecryptable ciphertext (key ring lost) falls back rather than throwing: an
  ingestion worker on a timer should degrade, not crash.
- The stored secret is never sent to the browser; a blank field means
  "unchanged", not "cleared".
- MQTT usernames are stored as-is — §6.4 covers tokens and passwords, and
  encrypting a username would only blank the field on every edit.
- ExportService drops *_enc values: bound to the originating key ring, so
  useless where an export would be restored. Expect to re-enter after a restore.
- HaConnectionTester now takes a resolved token, so the admin UI can test a
  token that has been typed but not yet saved.

SDD §6.4 and §9 updated to describe both forms rather than contradict the code.

Claude-Session: https://claude.ai/code/session_01V6joyergfvVLFEizH1hJLd

src/Infrastructure/Backup/ExportService.cs

@@ -19,6 +19,25 @@ public sealed class ExportService(MeterVaultDbContext db)
 
     private readonly MeterVaultDbContext _db = db;
 
+    /// <summary>
+    /// Strips encrypted connector secrets from an export. They are ciphertext, not plaintext, so
+    /// this is not a §6.4 requirement — but the export exists for portability (§9), and ciphertext
+    /// is bound to the originating instance's key ring, so it is useless anywhere it could be
+    /// restored and merely widens the blast radius if the key ring also leaks. Env-var references
+    /// survive: they name a variable and reveal nothing. Restoring means re-entering the secrets.
+    /// </summary>
+    private static List<IngestionEndpoint> RedactSecrets(List<IngestionEndpoint> endpoints)
+    {
+        foreach (var endpoint in endpoints)
+        {
+            endpoint.Config = endpoint.Type == EndpointType.HomeAssistant
+                ? (Ingestion.HaEndpointConfig.Parse(endpoint.Config) with { TokenEnc = null }).ToJson()
+                : (Ingestion.EndpointConfig.Parse(endpoint.Config) with { PasswordEnc = null }).ToJson();
+        }
+
+        return endpoints;
+    }
+
     public async Task<string> ExportJsonAsync(CancellationToken cancellationToken = default)
     {
         // Load without navigation includes so serialization is a clean, cycle-free tree.
@@ -27,7 +46,8 @@ public sealed class ExportService(MeterVaultDbContext db)
             EnergyTypes = await _db.EnergyTypes.AsNoTracking().ToListAsync(cancellationToken).ConfigureAwait(false),
             CostCategories = await _db.CostCategories.AsNoTracking().ToListAsync(cancellationToken).ConfigureAwait(false),
             Meters = await _db.Meters.AsNoTracking().ToListAsync(cancellationToken).ConfigureAwait(false),
-            IngestionEndpoints = await _db.IngestionEndpoints.AsNoTracking().ToListAsync(cancellationToken).ConfigureAwait(false),
+            IngestionEndpoints = RedactSecrets(
+                await _db.IngestionEndpoints.AsNoTracking().ToListAsync(cancellationToken).ConfigureAwait(false)),
             MeterSources = await _db.MeterSources.AsNoTracking().ToListAsync(cancellationToken).ConfigureAwait(false),
             Tanks = await _db.Tanks.AsNoTracking().ToListAsync(cancellationToken).ConfigureAwait(false),
             Tariffs = await _db.Tariffs.AsNoTracking().ToListAsync(cancellationToken).ConfigureAwait(false),

src/Infrastructure/DependencyInjection.cs

@@ -37,6 +37,9 @@ public static class DependencyInjection
         // "Test connection" works without the background workers running.
         services.AddHttpClient();
         services.AddScoped<HaConnectionTester>();
+        // Singleton: wraps one IDataProtector, and the ingestion workers (themselves singletons)
+        // resolve connector secrets on every reconnect.
+        services.AddSingleton<Security.SecretProtector>();
         services.AddScoped<Costing.CostService>();
         services.AddScoped<Dashboard.DashboardService>();
         services.AddScoped<Dashboard.SolarService>();

src/Infrastructure/Ingestion/EndpointConfig.cs

@@ -4,9 +4,10 @@ using System.Text.Json.Serialization;
 namespace MeterVault.Infrastructure.Ingestion;
 
 /// <summary>
-/// The parsed <see cref="Core.Domain.IngestionEndpoint.Config"/> JSON for an MQTT broker. Secrets
-/// are stored by reference only (SDD §6.4): <see cref="UsernameEnv"/>/<see cref="PasswordEnv"/>
-/// name environment variables resolved at runtime, never plaintext credentials in the database.
+/// The parsed <see cref="Core.Domain.IngestionEndpoint.Config"/> JSON for an MQTT broker. Credentials
+/// are never held here as plaintext (SDD §6.4): <see cref="UsernameEnv"/>/<see cref="PasswordEnv"/>
+/// name environment variables resolved at runtime, or <see cref="UsernameEnc"/>/<see cref="PasswordEnc"/>
+/// hold them encrypted under the app's data-protection key ring.
 /// </summary>
 public sealed record EndpointConfig
 {
@@ -29,6 +30,15 @@ public sealed record EndpointConfig
 
     public string? PasswordEnv { get; init; }
 
+    /// <summary>
+    /// Username entered directly in the admin UI. Held as-is: §6.4 covers tokens and passwords, and
+    /// encrypting a username would only blank the field on every edit for no security gain.
+    /// </summary>
+    public string? Username { get; init; }
+
+    /// <summary>Password encrypted by <see cref="Security.SecretProtector"/> (entered in the admin UI).</summary>
+    public string? PasswordEnc { get; init; }
+
     public static EndpointConfig Parse(string? json)
     {
         if (string.IsNullOrWhiteSpace(json))
@@ -48,10 +58,11 @@ public sealed record EndpointConfig
 
     public string ToJson() => JsonSerializer.Serialize(this, Options);
 
-    public string? ResolveUsername() => Resolve(UsernameEnv);
+    public string? ResolveUsername(Security.SecretProtector? protector = null) =>
+        !string.IsNullOrWhiteSpace(Username)
+            ? Username
+            : EndpointSecret.Resolve(null, UsernameEnv, protector);
 
-    public string? ResolvePassword() => Resolve(PasswordEnv);
-
-    private static string? Resolve(string? envVarName) =>
-        string.IsNullOrWhiteSpace(envVarName) ? null : Environment.GetEnvironmentVariable(envVarName);
+    public string? ResolvePassword(Security.SecretProtector? protector = null) =>
+        EndpointSecret.Resolve(PasswordEnc, PasswordEnv, protector);
 }

src/Infrastructure/Ingestion/EndpointSecret.cs

@@ -0,0 +1,29 @@
+using MeterVault.Infrastructure.Security;
+
+namespace MeterVault.Infrastructure.Ingestion;
+
+/// <summary>
+/// Resolves one connector secret from the two storage forms an endpoint config may use: a value
+/// typed into the admin UI and encrypted at rest, or the name of an environment variable resolved
+/// at runtime. Neither form keeps plaintext in the database (SDD §6.4).
+/// </summary>
+internal static class EndpointSecret
+{
+    /// <summary>
+    /// Encrypted wins over the env-var reference when both are present. They are mutually exclusive
+    /// in the UI, so both being set means a connector was switched from one mode to the other and
+    /// the write did not clear the old field; honouring the encrypted value keeps the connector on
+    /// whichever secret was most recently entered.
+    /// </summary>
+    public static string? Resolve(string? encrypted, string? envVarName, SecretProtector? protector)
+    {
+        if (!string.IsNullOrWhiteSpace(encrypted)
+            && protector is not null
+            && protector.TryUnprotect(encrypted, out var plaintext))
+        {
+            return plaintext;
+        }
+
+        return string.IsNullOrWhiteSpace(envVarName) ? null : Environment.GetEnvironmentVariable(envVarName);
+    }
+}

src/Infrastructure/Ingestion/HaConnectionTester.cs

@@ -7,32 +7,31 @@ namespace MeterVault.Infrastructure.Ingestion;
 public sealed record HaTestResult(bool Ok, string Message, double? SampleValue = null);
 
 /// <summary>
-/// Verifies a Home Assistant connection from the admin UI: checks the base URL + resolved token
-/// against <c>GET /api/</c>, and optionally reads one entity's state. Confirms the app can actually
-/// read HA before a source is relied upon (SDD §6.2). Token is resolved by reference (env var).
+/// Verifies a Home Assistant connection from the admin UI: checks the base URL + token against
+/// <c>GET /api/</c>, and optionally reads one entity's state. Confirms the app can actually read HA
+/// before a source is relied upon (SDD §6.2).
 /// </summary>
+/// <remarks>
+/// Takes the token already resolved rather than a reference to one. The admin UI must be able to
+/// test a token that has been typed but not yet saved (so not yet encrypted), and keeping the two
+/// storage forms out of here leaves one resolution path in <see cref="EndpointSecret"/>.
+/// </remarks>
 public sealed class HaConnectionTester(IHttpClientFactory httpClientFactory, ILogger<HaConnectionTester> logger)
 {
     private readonly IHttpClientFactory _httpClientFactory = httpClientFactory;
     private readonly ILogger<HaConnectionTester> _logger = logger;
 
     public async Task<HaTestResult> TestAsync(
-        string? baseUrl, string? tokenEnv, string? entityId, CancellationToken cancellationToken = default)
+        string? baseUrl, string? token, string? entityId, CancellationToken cancellationToken = default)
     {
         if (string.IsNullOrWhiteSpace(baseUrl))
         {
             return new HaTestResult(false, "Base URL is required.");
         }
 
-        if (string.IsNullOrWhiteSpace(tokenEnv))
-        {
-            return new HaTestResult(false, "Token env-var name is required (the token is resolved from it at runtime).");
-        }
-
-        var token = Environment.GetEnvironmentVariable(tokenEnv);
         if (string.IsNullOrWhiteSpace(token))
         {
-            return new HaTestResult(false, $"Environment variable '{tokenEnv}' is not set on the server.");
+            return new HaTestResult(false, "No token available to test.");
         }
 
         var client = _httpClientFactory.CreateClient();

src/Infrastructure/Ingestion/HaEndpointConfig.cs

@@ -1,12 +1,14 @@
 using System.Text.Json;
 using System.Text.Json.Serialization;
+using MeterVault.Infrastructure.Security;
 
 namespace MeterVault.Infrastructure.Ingestion;
 
 /// <summary>
 /// The parsed <see cref="Core.Domain.IngestionEndpoint.Config"/> JSON for a Home Assistant
-/// connection (SDD §6.2). The long-lived token is stored by reference only: <see cref="TokenEnv"/>
-/// names an environment variable resolved at runtime — never the token itself (SDD §6.4).
+/// connection (SDD §6.2). The long-lived token is never held here as plaintext (SDD §6.4): either
+/// <see cref="TokenEnv"/> names an environment variable resolved at runtime, or
+/// <see cref="TokenEnc"/> holds it encrypted under the app's data-protection key ring.
 /// </summary>
 public sealed record HaEndpointConfig
 {
@@ -22,6 +24,12 @@ public sealed record HaEndpointConfig
     /// <summary>Name of the environment variable holding the long-lived access token.</summary>
     public string? TokenEnv { get; init; }
 
+    /// <summary>
+    /// The long-lived access token, encrypted by <see cref="Security.SecretProtector"/>. Set when
+    /// the operator typed the token into the admin UI instead of naming an environment variable.
+    /// </summary>
+    public string? TokenEnc { get; init; }
+
     /// <summary>
     /// When true, a persistent WebSocket subscription pushes state changes in real time
     /// (<see cref="HomeAssistantWebSocketWorker"/>); when false (default) the REST poll worker
@@ -48,7 +56,10 @@ public sealed record HaEndpointConfig
 
     public string ToJson() => JsonSerializer.Serialize(this, Options);
 
-    /// <summary>Resolves the token from the referenced environment variable (null if unset).</summary>
-    public string? ResolveToken() =>
-        string.IsNullOrWhiteSpace(TokenEnv) ? null : Environment.GetEnvironmentVariable(TokenEnv);
+    /// <summary>
+    /// Resolves the token: the encrypted value when one was entered directly, otherwise the
+    /// referenced environment variable. Null when neither yields anything.
+    /// </summary>
+    public string? ResolveToken(SecretProtector? protector = null) =>
+        EndpointSecret.Resolve(TokenEnc, TokenEnv, protector);
 }

src/Infrastructure/Ingestion/HomeAssistantWebSocketWorker.cs

@@ -17,10 +17,12 @@ namespace MeterVault.Infrastructure.Ingestion;
 /// subscribes to <c>state_changed</c> events, and ingests changes for the endpoint's configured
 /// entities as they happen. Reconnects with capped backoff. Endpoints without <c>UseWebSocket</c> stay
 /// on the REST poll worker (<see cref="HomeAssistantWorker"/>) — each endpoint is served by exactly one.
-/// Tokens are resolved from environment variables named in the endpoint config, never stored plaintext.
+/// Tokens are resolved from the endpoint config — an env-var reference or an encrypted value —
+/// never stored as plaintext.
 /// </summary>
 public sealed class HomeAssistantWebSocketWorker(
     IServiceScopeFactory scopeFactory,
+    Security.SecretProtector secrets,
     ILogger<HomeAssistantWebSocketWorker> logger) : BackgroundService
 {
     private static readonly TimeSpan SuperviseInterval = TimeSpan.FromSeconds(15);
@@ -29,6 +31,7 @@ public sealed class HomeAssistantWebSocketWorker(
     private static readonly TimeSpan EntityMapTtl = TimeSpan.FromSeconds(30);
 
     private readonly IServiceScopeFactory _scopeFactory = scopeFactory;
+    private readonly Security.SecretProtector _secrets = secrets;
     private readonly ILogger<HomeAssistantWebSocketWorker> _logger = logger;
     private readonly ConcurrentDictionary<int, Task> _connections = new();
 
@@ -107,7 +110,7 @@ public sealed class HomeAssistantWebSocketWorker(
                     return; // switched to poll mode.
                 }
 
-                token = config.ResolveToken();
+                token = config.ResolveToken(_secrets);
             }
 
             if (string.IsNullOrWhiteSpace(config.BaseUrl) || string.IsNullOrWhiteSpace(token))

src/Infrastructure/Ingestion/HomeAssistantWorker.cs

@@ -17,12 +17,14 @@ namespace MeterVault.Infrastructure.Ingestion;
 public sealed class HomeAssistantWorker(
     IServiceScopeFactory scopeFactory,
     IHttpClientFactory httpClientFactory,
+    Security.SecretProtector secrets,
     ILogger<HomeAssistantWorker> logger) : BackgroundService
 {
     private static readonly TimeSpan TickInterval = TimeSpan.FromSeconds(10);
 
     private readonly IServiceScopeFactory _scopeFactory = scopeFactory;
     private readonly IHttpClientFactory _httpClientFactory = httpClientFactory;
+    private readonly Security.SecretProtector _secrets = secrets;
     private readonly ILogger<HomeAssistantWorker> _logger = logger;
     private readonly ConcurrentDictionary<int, DateTimeOffset> _nextPoll = new();
 
@@ -104,7 +106,7 @@ public sealed class HomeAssistantWorker(
         MeterSource source, SourceConfig config, CancellationToken cancellationToken)
     {
         var endpointConfig = HaEndpointConfig.Parse(endpoint.Config);
-        var token = endpointConfig.ResolveToken();
+        var token = endpointConfig.ResolveToken(_secrets);
 
         if (string.IsNullOrWhiteSpace(endpointConfig.BaseUrl) || string.IsNullOrWhiteSpace(token)
             || string.IsNullOrWhiteSpace(config.EntityId))

src/Infrastructure/Ingestion/MqttIngestionWorker.cs

@@ -16,11 +16,14 @@ namespace MeterVault.Infrastructure.Ingestion;
 /// are logged and retried on the next tick rather than crashing the app.
 /// </summary>
 public sealed class MqttIngestionWorker(
-    IServiceScopeFactory scopeFactory, ILogger<MqttIngestionWorker> logger) : BackgroundService
+    IServiceScopeFactory scopeFactory,
+    Security.SecretProtector secrets,
+    ILogger<MqttIngestionWorker> logger) : BackgroundService
 {
     private static readonly TimeSpan ReconnectInterval = TimeSpan.FromSeconds(15);
 
     private readonly IServiceScopeFactory _scopeFactory = scopeFactory;
+    private readonly Security.SecretProtector _secrets = secrets;
     private readonly ILogger<MqttIngestionWorker> _logger = logger;
     private readonly MqttClientFactory _factory = new();
     private readonly ConcurrentDictionary<int, IMqttClient> _clients = new();
@@ -132,10 +135,10 @@ public sealed class MqttIngestionWorker(
             .WithTcpServer(config.Host, config.Port)
             .WithCleanSession();
 
-        var username = config.ResolveUsername();
+        var username = config.ResolveUsername(_secrets);
         if (!string.IsNullOrWhiteSpace(username))
         {
-            builder = builder.WithCredentials(username, config.ResolvePassword() ?? string.Empty);
+            builder = builder.WithCredentials(username, config.ResolvePassword(_secrets) ?? string.Empty);
         }
 
         if (config.Tls)

src/Infrastructure/MeterVault.Infrastructure.csproj

@@ -8,6 +8,7 @@
     <PackageReference Include="MQTTnet" />
     <PackageReference Include="Microsoft.Extensions.Hosting.Abstractions" />
     <PackageReference Include="Microsoft.Extensions.Http" />
+    <PackageReference Include="Microsoft.AspNetCore.DataProtection.Abstractions" />
   </ItemGroup>
 
   <ItemGroup>

src/Infrastructure/Security/SecretProtector.cs

@@ -0,0 +1,64 @@
+using System.Security.Cryptography;
+using Microsoft.AspNetCore.DataProtection;
+
+namespace MeterVault.Infrastructure.Security;
+
+/// <summary>
+/// Encrypts connector secrets that an operator types into the admin UI, so
+/// <c>ingestion_endpoint.config</c> holds ciphertext rather than the token itself.
+/// </summary>
+/// <remarks>
+/// SDD §6.4 requires that tokens are never in the database as plaintext. Naming an environment
+/// variable satisfies that but forces a file edit plus a service restart to add a connector, which
+/// is hostile enough that people paste the token into the name field instead. Encrypting at rest
+/// keeps the guarantee that matters — a <c>pg_dump</c> or JSON export carries nothing usable — while
+/// letting the token be entered where it is configured.
+///
+/// The key ring lives on disk outside the database, so this protects against leaked database
+/// content, not against an attacker who already has the host: they can read the keys and decrypt.
+/// That is the same trust boundary as an env var, which is equally readable from <c>/proc</c>.
+/// Losing the key ring makes existing secrets undecryptable, and they must be re-entered.
+/// </remarks>
+public sealed class SecretProtector
+{
+    // Changing this string orphans every secret encrypted under the old value.
+    private const string Purpose = "MeterVault.IngestionEndpoint.Secrets.v1";
+
+    private readonly IDataProtector _protector;
+
+    public SecretProtector(IDataProtectionProvider provider)
+    {
+        ArgumentNullException.ThrowIfNull(provider);
+        _protector = provider.CreateProtector(Purpose);
+    }
+
+    public string Protect(string plaintext)
+    {
+        ArgumentException.ThrowIfNullOrWhiteSpace(plaintext);
+        return _protector.Protect(plaintext);
+    }
+
+    /// <summary>
+    /// Decrypts a stored secret. Returns false rather than throwing when the ciphertext cannot be
+    /// read — a rotated-away or restored-without-the-key-ring deployment should degrade to "this
+    /// connector has no usable secret" and surface that, not crash an ingestion worker on a timer.
+    /// </summary>
+    public bool TryUnprotect(string? ciphertext, out string? plaintext)
+    {
+        plaintext = null;
+        if (string.IsNullOrWhiteSpace(ciphertext))
+        {
+            return false;
+        }
+
+        try
+        {
+            plaintext = _protector.Unprotect(ciphertext);
+            return true;
+        }
+        catch (CryptographicException)
+        {
+            return false;
+        }
+    }
+}