From 3e0e5aec39645de022e5227a53cb987f93a63a92 Mon Sep 17 00:00:00 2001 From: LucHeart Date: Sat, 21 Nov 2026 23:52:38 +0100 Subject: [PATCH 1/3] feat: log each migration and materialize control stats in the background --- .../Tests/TimescaleDbRequirementTests.cs | 36 ++++ API/Program.cs | 3 + .../20261120201415_MaterializeControlStats.cs | 19 +- Common/OpenShockMiddlewareHelper.cs | 16 +- .../ControlStatsBackfillService.cs | 174 ++++++++++++++++++ 5 files changed, 230 insertions(+), 18 deletions(-) create mode 100644 Common/Services/ControlStats/ControlStatsBackfillService.cs diff --git a/API.IntegrationTests/Tests/TimescaleDbRequirementTests.cs b/API.IntegrationTests/Tests/TimescaleDbRequirementTests.cs index 09b0c26d..a808b4aa 100644 --- a/API.IntegrationTests/Tests/TimescaleDbRequirementTests.cs +++ b/API.IntegrationTests/Tests/TimescaleDbRequirementTests.cs @@ -1,8 +1,10 @@ using Microsoft.AspNetCore.Builder; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging.Abstractions; using Npgsql; using OpenShock.Common; using OpenShock.Common.Options; +using OpenShock.Common.Services.ControlStats; using Testcontainers.PostgreSql; namespace OpenShock.API.IntegrationTests.Tests; @@ -63,11 +65,45 @@ public async Task Community_ApplyingMigrationsOnNewDatabase_Works() await container.StartAsync(); await ApplyMigrations(WithDatabase(container, "created_on_startup")); + await ControlStatsBackfillService.EnsureMaterializedAsync(WithDatabase(container, "created_on_startup"), NullLogger.Instance); await Assert.That(await HasExtension(container, "created_on_startup")).IsTrue(); await Assert.That(await HasControlStatsPolicy(container, "created_on_startup")).IsTrue(); } + [Test] + public async Task Backfill_MaterializesHistoryThenAddsPolicy() + { + await using var container = CreateContainer(Community); + await container.StartAsync(); + var connectionString = WithDatabase(container, "backfill"); + + await ApplyMigrations(connectionString); + await Assert.That(await HasControlStatsPolicy(container, "backfill")).IsFalse(); + + // Logs spanning several backfill windows, the owner foreign key is skipped instead of creating a user + await Psql(container, "backfill", + """ + SET session_replication_role = replica; + INSERT INTO shocker_control_logs (id, owner_user_id, shocker_id, controlled_by_user_id, intensity, max_intensity, duration, type, custom_name, live_control, created_at) + SELECT gen_random_uuid(), gen_random_uuid(), gen_random_uuid(), NULL, 50, 50, 1000, 'shock', NULL, false, now() - age + FROM unnest(ARRAY[INTERVAL '200 days', INTERVAL '40 days', INTERVAL '2 days']) AS age; + """); + + await ControlStatsBackfillService.EnsureMaterializedAsync(connectionString, NullLogger.Instance); + // A second run finds the policy and does nothing + await ControlStatsBackfillService.EnsureMaterializedAsync(connectionString, NullLogger.Instance); + + await Assert.That(await HasControlStatsPolicy(container, "backfill")).IsTrue(); + + // Only what is materialized, real-time aggregation would show the logs either way + await Psql(container, "backfill", "ALTER MATERIALIZED VIEW control_stats SET (timescaledb.materialized_only = true)"); + await using var connection = new NpgsqlConnection(connectionString); + await connection.OpenAsync(); + await using var command = new NpgsqlCommand("SELECT coalesce(sum(count), 0) FROM control_stats", connection); + await Assert.That(Convert.ToInt64(await command.ExecuteScalarAsync())).IsEqualTo(3); + } + [Test] public async Task Community_Check_LeavesNothingBehind() { diff --git a/API/Program.cs b/API/Program.cs index 1afa8814..312a5225 100644 --- a/API/Program.cs +++ b/API/Program.cs @@ -15,6 +15,7 @@ using OpenShock.Common.Extensions; using OpenShock.Common.Hubs; using OpenShock.Common.Services; +using OpenShock.Common.Services.ControlStats; using OpenShock.Common.Services.Device; using OpenShock.Common.Services.Ota; using Serilog; @@ -139,6 +140,8 @@ static void DefaultOptions(RemoteAuthenticationOptions options, string provider) builder.AddCloudflareTurnstileService(); builder.Services.AddHostedService(); +builder.Services.AddHostedService(provider => new ControlStatsBackfillService(databaseOptions.Conn, + provider.GetRequiredService>())); var app = builder.Build(); diff --git a/Common/Migrations/20261120201415_MaterializeControlStats.cs b/Common/Migrations/20261120201415_MaterializeControlStats.cs index e5f04240..c8da8c7b 100644 --- a/Common/Migrations/20261120201415_MaterializeControlStats.cs +++ b/Common/Migrations/20261120201415_MaterializeControlStats.cs @@ -5,28 +5,15 @@ namespace OpenShock.Common.Migrations; /// -/// Separate from as it can't run in a transaction. +/// Used to materialize the control_stats history and add its refresh policy here, which held up startup for as +/// long as aggregating every control log took. That moved to ControlStatsBackfillService, which does it +/// in the background and skips databases where this migration already did it. /// public partial class _20261120201415_MaterializeControlStats : Migration { /// protected override void Up(MigrationBuilder migrationBuilder) { - migrationBuilder.Sql( - "CALL refresh_continuous_aggregate('control_stats', NULL, now() - INTERVAL '1 hour');", - suppressTransaction: true); - - // Added after the refresh above, which fails if the policy's job is refreshing at the same time - migrationBuilder.Sql( - """ - SELECT add_continuous_aggregate_policy('control_stats', - start_offset => INTERVAL '3 days', - end_offset => INTERVAL '1 hour', - schedule_interval => INTERVAL '5 minutes', - if_not_exists => true, - initial_start => now() + INTERVAL '5 minutes'); - """, - suppressTransaction: true); } /// diff --git a/Common/OpenShockMiddlewareHelper.cs b/Common/OpenShockMiddlewareHelper.cs index e713370d..9986e113 100644 --- a/Common/OpenShockMiddlewareHelper.cs +++ b/Common/OpenShockMiddlewareHelper.cs @@ -1,9 +1,11 @@ -using System.Net; +using System.Diagnostics; +using System.Net; using System.Text; using Microsoft.AspNetCore.Diagnostics.HealthChecks; using Microsoft.AspNetCore.HttpOverrides; using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; using Microsoft.EntityFrameworkCore.Storage; using Microsoft.Extensions.Diagnostics.HealthChecks; using Npgsql; @@ -238,7 +240,17 @@ public static async Task ApplyPendingOpenShockMigrations(th if (pendingMigrations.Length > 0) { logger.LogInformation("Found pending migrations, applying [{@Migrations}]", string.Join(", ", pendingMigrations)); - await migrationContext.Database.MigrateAsync(); + + // One at a time, so a slow migration shows up in the logs + var migrator = migrationContext.GetService(); + foreach (var migration in pendingMigrations) + { + logger.LogInformation("Applying migration {Migration}", migration); + var stopwatch = Stopwatch.StartNew(); + await migrator.MigrateAsync(migration); + logger.LogInformation("Applied migration {Migration} in {Elapsed}", migration, stopwatch.Elapsed); + } + logger.LogInformation("Applied database migrations... proceeding with startup"); // A migration may have introduced new Postgres types (e.g. a new enum, or a new value on an diff --git a/Common/Services/ControlStats/ControlStatsBackfillService.cs b/Common/Services/ControlStats/ControlStatsBackfillService.cs new file mode 100644 index 00000000..c4324c69 --- /dev/null +++ b/Common/Services/ControlStats/ControlStatsBackfillService.cs @@ -0,0 +1,174 @@ +using System.Diagnostics; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Npgsql; + +namespace OpenShock.Common.Services.ControlStats; + +/// +/// Materializes the history of control_stats after startup instead of in a migration, which held up startup +/// for as long as it took to aggregate every control log. Until it is done stats stay correct, real-time aggregation +/// computes whatever isn't materialized yet from the logs. The refresh policy is only added once the history is +/// materialized: it moves the watermark to the present, so anything older left unmaterialized would drop out. +/// +public sealed class ControlStatsBackfillService : BackgroundService +{ + // Arbitrary, unique to this job, so only one instance backfills at a time + private const long LockKey = 0x4F53_4353_4246; // "OSCSBF" + + private static readonly TimeSpan Window = TimeSpan.FromDays(30); + private static readonly TimeSpan RetryDelay = TimeSpan.FromMinutes(1); + + private readonly string _connectionString; + private readonly ILogger _logger; + + public ControlStatsBackfillService(string connectionString, ILogger logger) + { + _connectionString = connectionString; + _logger = logger; + } + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + while (!stoppingToken.IsCancellationRequested) + { + try + { + await EnsureMaterializedAsync(_connectionString, _logger, stoppingToken); + return; + } + catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) + { + return; + } + catch (Exception e) + { + _logger.LogError(e, "Materializing control_stats failed, retrying in {Delay}", RetryDelay); + } + + await Task.Delay(RetryDelay, stoppingToken); + } + } + + /// + /// Materializes the history and adds the refresh policy, unless the policy already exists. Safe to run from + /// several instances and to interrupt, a later run continues from what is already materialized. + /// + public static async Task EnsureMaterializedAsync(string connectionString, ILogger logger, CancellationToken cancellationToken = default) + { + await using var connection = new NpgsqlConnection(connectionString); + await connection.OpenAsync(cancellationToken); + + if (await HasPolicyAsync(connection, cancellationToken)) return; + + if (!await ScalarAsync(connection, $"SELECT pg_try_advisory_lock({LockKey})", cancellationToken)) + { + logger.LogInformation("control_stats is being materialized by another instance"); + return; + } + + try + { + // Checked again, another instance may have finished between the check and taking the lock + if (await HasPolicyAsync(connection, cancellationToken)) return; + + var stopwatch = Stopwatch.StartNew(); + await MaterializeHistoryAsync(connection, logger, cancellationToken); + + await ExecuteAsync(connection, + """ + SELECT add_continuous_aggregate_policy('control_stats', + start_offset => INTERVAL '3 days', + end_offset => INTERVAL '1 hour', + schedule_interval => INTERVAL '5 minutes', + if_not_exists => true, + initial_start => now() + INTERVAL '5 minutes') + """, cancellationToken); + + logger.LogInformation("Materialized control_stats in {Elapsed}", stopwatch.Elapsed); + } + finally + { + await ExecuteAsync(connection, $"SELECT pg_advisory_unlock({LockKey})", CancellationToken.None); + } + } + + /// + /// Refreshes control_stats oldest first, one window at a time. Each refresh commits on its own, so the + /// progress survives a restart. + /// + public static async Task MaterializeHistoryAsync(NpgsqlConnection connection, ILogger logger, CancellationToken cancellationToken = default) + { + // Bucket aligned, a refresh only covers buckets entirely inside its window + DateTime? first; + DateTime to; + await using (var command = new NpgsqlCommand( + """ + SELECT time_bucket(INTERVAL '15 minutes', min(created_at)), + time_bucket(INTERVAL '15 minutes', now() - INTERVAL '1 hour') + FROM shocker_control_logs + """, connection)) + await using (var reader = await command.ExecuteReaderAsync(cancellationToken)) + { + await reader.ReadAsync(cancellationToken); + first = await reader.IsDBNullAsync(0, cancellationToken) ? null : reader.GetDateTime(0); + to = reader.GetDateTime(1); + } + + if (first is not { } from || from >= to) + { + logger.LogInformation("No control logs to materialize"); + return; + } + + logger.LogInformation("Materializing control_stats from {From:u} to {To:u}", from, to); + + for (var start = from; start < to; start += Window) + { + var end = start + Window < to ? start + Window : to; + await RefreshAsync(connection, start, end, cancellationToken); + logger.LogInformation("Materialized control_stats up to {End:u}", end); + } + } + + private static async Task RefreshAsync(NpgsqlConnection connection, DateTime start, DateTime end, CancellationToken cancellationToken) + { + for (var attempt = 1; ; attempt++) + { + try + { + await using var command = new NpgsqlCommand("CALL refresh_continuous_aggregate('control_stats', $1, $2)", connection); + command.CommandTimeout = 0; + command.Parameters.AddWithValue(start); + command.Parameters.AddWithValue(end); + await command.ExecuteNonQueryAsync(cancellationToken); + return; + } + // Another refresh of the same range is running, TimescaleDB fails instead of waiting + catch (PostgresException e) when (e.SqlState == PostgresErrorCodes.LockNotAvailable && attempt < 10) + { + await Task.Delay(TimeSpan.FromSeconds(attempt), cancellationToken); + } + } + } + + private static Task HasPolicyAsync(NpgsqlConnection connection, CancellationToken cancellationToken) => + ScalarAsync(connection, + """ + SELECT EXISTS (SELECT 1 FROM timescaledb_information.jobs + WHERE proc_name = 'policy_refresh_continuous_aggregate' AND hypertable_name = 'control_stats') + """, cancellationToken); + + private static async Task ScalarAsync(NpgsqlConnection connection, string sql, CancellationToken cancellationToken) + { + await using var command = new NpgsqlCommand(sql, connection); + return (T)(await command.ExecuteScalarAsync(cancellationToken))!; + } + + private static async Task ExecuteAsync(NpgsqlConnection connection, string sql, CancellationToken cancellationToken) + { + await using var command = new NpgsqlCommand(sql, connection); + command.CommandTimeout = 0; + await command.ExecuteNonQueryAsync(cancellationToken); + } +} From f182d8a605f58a495b7bcc3c6b208a8fedec0397 Mon Sep 17 00:00:00 2001 From: LucHeart Date: Sun, 22 Nov 2026 00:01:37 +0100 Subject: [PATCH 2/3] fix: keep checking while another instance backfills --- .../Tests/TimescaleDbRequirementTests.cs | 4 ++-- .../ControlStats/ControlStatsBackfillService.cs | 14 ++++++++------ 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/API.IntegrationTests/Tests/TimescaleDbRequirementTests.cs b/API.IntegrationTests/Tests/TimescaleDbRequirementTests.cs index a808b4aa..cba717bc 100644 --- a/API.IntegrationTests/Tests/TimescaleDbRequirementTests.cs +++ b/API.IntegrationTests/Tests/TimescaleDbRequirementTests.cs @@ -90,9 +90,9 @@ SELECT gen_random_uuid(), gen_random_uuid(), gen_random_uuid(), NULL, 50, 50, 10 FROM unnest(ARRAY[INTERVAL '200 days', INTERVAL '40 days', INTERVAL '2 days']) AS age; """); - await ControlStatsBackfillService.EnsureMaterializedAsync(connectionString, NullLogger.Instance); + await Assert.That(await ControlStatsBackfillService.EnsureMaterializedAsync(connectionString, NullLogger.Instance)).IsTrue(); // A second run finds the policy and does nothing - await ControlStatsBackfillService.EnsureMaterializedAsync(connectionString, NullLogger.Instance); + await Assert.That(await ControlStatsBackfillService.EnsureMaterializedAsync(connectionString, NullLogger.Instance)).IsTrue(); await Assert.That(await HasControlStatsPolicy(container, "backfill")).IsTrue(); diff --git a/Common/Services/ControlStats/ControlStatsBackfillService.cs b/Common/Services/ControlStats/ControlStatsBackfillService.cs index c4324c69..75d12f77 100644 --- a/Common/Services/ControlStats/ControlStatsBackfillService.cs +++ b/Common/Services/ControlStats/ControlStatsBackfillService.cs @@ -34,8 +34,8 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) { try { - await EnsureMaterializedAsync(_connectionString, _logger, stoppingToken); - return; + // Another instance holding the lock may stop before it's done, so keep checking until the policy exists + if (await EnsureMaterializedAsync(_connectionString, _logger, stoppingToken)) return; } catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) { @@ -54,23 +54,24 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) /// Materializes the history and adds the refresh policy, unless the policy already exists. Safe to run from /// several instances and to interrupt, a later run continues from what is already materialized. /// - public static async Task EnsureMaterializedAsync(string connectionString, ILogger logger, CancellationToken cancellationToken = default) + /// Whether the policy exists now, false while another instance is still materializing. + public static async Task EnsureMaterializedAsync(string connectionString, ILogger logger, CancellationToken cancellationToken = default) { await using var connection = new NpgsqlConnection(connectionString); await connection.OpenAsync(cancellationToken); - if (await HasPolicyAsync(connection, cancellationToken)) return; + if (await HasPolicyAsync(connection, cancellationToken)) return true; if (!await ScalarAsync(connection, $"SELECT pg_try_advisory_lock({LockKey})", cancellationToken)) { logger.LogInformation("control_stats is being materialized by another instance"); - return; + return false; } try { // Checked again, another instance may have finished between the check and taking the lock - if (await HasPolicyAsync(connection, cancellationToken)) return; + if (await HasPolicyAsync(connection, cancellationToken)) return true; var stopwatch = Stopwatch.StartNew(); await MaterializeHistoryAsync(connection, logger, cancellationToken); @@ -86,6 +87,7 @@ SELECT add_continuous_aggregate_policy('control_stats', """, cancellationToken); logger.LogInformation("Materialized control_stats in {Elapsed}", stopwatch.Elapsed); + return true; } finally { From 34cce719b559cfee1ce50cb14752de0a11c62199 Mon Sep 17 00:00:00 2001 From: LucHeart Date: Sun, 22 Nov 2026 00:56:03 +0100 Subject: [PATCH 3/3] feat: migrate command for running migrations before a rollout --- .../Tests/TimescaleDbRequirementTests.cs | 36 ---- API/Program.cs | 10 +- .../20261120201415_MaterializeControlStats.cs | 19 +- Common/OpenShockMiddlewareHelper.cs | 5 +- .../ControlStatsBackfillService.cs | 176 ------------------ docker/entrypoint.sh | 3 +- 6 files changed, 28 insertions(+), 221 deletions(-) delete mode 100644 Common/Services/ControlStats/ControlStatsBackfillService.cs diff --git a/API.IntegrationTests/Tests/TimescaleDbRequirementTests.cs b/API.IntegrationTests/Tests/TimescaleDbRequirementTests.cs index cba717bc..09b0c26d 100644 --- a/API.IntegrationTests/Tests/TimescaleDbRequirementTests.cs +++ b/API.IntegrationTests/Tests/TimescaleDbRequirementTests.cs @@ -1,10 +1,8 @@ using Microsoft.AspNetCore.Builder; using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Logging.Abstractions; using Npgsql; using OpenShock.Common; using OpenShock.Common.Options; -using OpenShock.Common.Services.ControlStats; using Testcontainers.PostgreSql; namespace OpenShock.API.IntegrationTests.Tests; @@ -65,45 +63,11 @@ public async Task Community_ApplyingMigrationsOnNewDatabase_Works() await container.StartAsync(); await ApplyMigrations(WithDatabase(container, "created_on_startup")); - await ControlStatsBackfillService.EnsureMaterializedAsync(WithDatabase(container, "created_on_startup"), NullLogger.Instance); await Assert.That(await HasExtension(container, "created_on_startup")).IsTrue(); await Assert.That(await HasControlStatsPolicy(container, "created_on_startup")).IsTrue(); } - [Test] - public async Task Backfill_MaterializesHistoryThenAddsPolicy() - { - await using var container = CreateContainer(Community); - await container.StartAsync(); - var connectionString = WithDatabase(container, "backfill"); - - await ApplyMigrations(connectionString); - await Assert.That(await HasControlStatsPolicy(container, "backfill")).IsFalse(); - - // Logs spanning several backfill windows, the owner foreign key is skipped instead of creating a user - await Psql(container, "backfill", - """ - SET session_replication_role = replica; - INSERT INTO shocker_control_logs (id, owner_user_id, shocker_id, controlled_by_user_id, intensity, max_intensity, duration, type, custom_name, live_control, created_at) - SELECT gen_random_uuid(), gen_random_uuid(), gen_random_uuid(), NULL, 50, 50, 1000, 'shock', NULL, false, now() - age - FROM unnest(ARRAY[INTERVAL '200 days', INTERVAL '40 days', INTERVAL '2 days']) AS age; - """); - - await Assert.That(await ControlStatsBackfillService.EnsureMaterializedAsync(connectionString, NullLogger.Instance)).IsTrue(); - // A second run finds the policy and does nothing - await Assert.That(await ControlStatsBackfillService.EnsureMaterializedAsync(connectionString, NullLogger.Instance)).IsTrue(); - - await Assert.That(await HasControlStatsPolicy(container, "backfill")).IsTrue(); - - // Only what is materialized, real-time aggregation would show the logs either way - await Psql(container, "backfill", "ALTER MATERIALIZED VIEW control_stats SET (timescaledb.materialized_only = true)"); - await using var connection = new NpgsqlConnection(connectionString); - await connection.OpenAsync(); - await using var command = new NpgsqlCommand("SELECT coalesce(sum(count), 0) FROM control_stats", connection); - await Assert.That(Convert.ToInt64(await command.ExecuteScalarAsync())).IsEqualTo(3); - } - [Test] public async Task Community_Check_LeavesNothingBehind() { diff --git a/API/Program.cs b/API/Program.cs index 312a5225..fed96e07 100644 --- a/API/Program.cs +++ b/API/Program.cs @@ -15,7 +15,6 @@ using OpenShock.Common.Extensions; using OpenShock.Common.Hubs; using OpenShock.Common.Services; -using OpenShock.Common.Services.ControlStats; using OpenShock.Common.Services.Device; using OpenShock.Common.Services.Ota; using Serilog; @@ -140,11 +139,16 @@ static void DefaultOptions(RemoteAuthenticationOptions options, string provider) builder.AddCloudflareTurnstileService(); builder.Services.AddHostedService(); -builder.Services.AddHostedService(provider => new ControlStatsBackfillService(databaseOptions.Conn, - provider.GetRequiredService>())); var app = builder.Build(); +// `migrate` applies pending migrations and exits, for running them as a separate step before a rollout +if (args.Contains("migrate")) +{ + await app.ApplyPendingOpenShockMigrations(databaseOptions); + return; +} + // Optional path prefix the API is served under (e.g. "/api" when reverse-proxied on a // shared host). Empty by default -> served at root, unchanged behavior. var apiPathBase = builder.Configuration.GetValue("OpenShock:Api:PathBase"); diff --git a/Common/Migrations/20261120201415_MaterializeControlStats.cs b/Common/Migrations/20261120201415_MaterializeControlStats.cs index c8da8c7b..e5f04240 100644 --- a/Common/Migrations/20261120201415_MaterializeControlStats.cs +++ b/Common/Migrations/20261120201415_MaterializeControlStats.cs @@ -5,15 +5,28 @@ namespace OpenShock.Common.Migrations; /// -/// Used to materialize the control_stats history and add its refresh policy here, which held up startup for as -/// long as aggregating every control log took. That moved to ControlStatsBackfillService, which does it -/// in the background and skips databases where this migration already did it. +/// Separate from as it can't run in a transaction. /// public partial class _20261120201415_MaterializeControlStats : Migration { /// protected override void Up(MigrationBuilder migrationBuilder) { + migrationBuilder.Sql( + "CALL refresh_continuous_aggregate('control_stats', NULL, now() - INTERVAL '1 hour');", + suppressTransaction: true); + + // Added after the refresh above, which fails if the policy's job is refreshing at the same time + migrationBuilder.Sql( + """ + SELECT add_continuous_aggregate_policy('control_stats', + start_offset => INTERVAL '3 days', + end_offset => INTERVAL '1 hour', + schedule_interval => INTERVAL '5 minutes', + if_not_exists => true, + initial_start => now() + INTERVAL '5 minutes'); + """, + suppressTransaction: true); } /// diff --git a/Common/OpenShockMiddlewareHelper.cs b/Common/OpenShockMiddlewareHelper.cs index 9986e113..19789d7a 100644 --- a/Common/OpenShockMiddlewareHelper.cs +++ b/Common/OpenShockMiddlewareHelper.cs @@ -359,7 +359,8 @@ public static async Task WaitForOpenShockSchemaReady(this I logger.LogInformation("Waiting for database schema to be up to date (migrations owned by another host)..."); - var deadline = DateTime.UtcNow + (timeout ?? TimeSpan.FromMinutes(5)); + // No deadline by default, a migration may legitimately take long, e.g. when run as a separate step + DateTime? deadline = timeout is { } t ? DateTime.UtcNow + t : null; var delay = TimeSpan.FromSeconds(1); var maxDelay = TimeSpan.FromSeconds(15); @@ -384,7 +385,7 @@ public static async Task WaitForOpenShockSchemaReady(this I logger.LogWarning(ex, "Could not determine migration state; will retry"); } - if (DateTime.UtcNow >= deadline) + if (deadline is { } d && DateTime.UtcNow >= d) throw new TimeoutException( "Timed out waiting for the database schema to be brought up to date by the migrator. " + "Ensure the API host (the sole migrator) is running and reachable."); diff --git a/Common/Services/ControlStats/ControlStatsBackfillService.cs b/Common/Services/ControlStats/ControlStatsBackfillService.cs deleted file mode 100644 index 75d12f77..00000000 --- a/Common/Services/ControlStats/ControlStatsBackfillService.cs +++ /dev/null @@ -1,176 +0,0 @@ -using System.Diagnostics; -using Microsoft.Extensions.Hosting; -using Microsoft.Extensions.Logging; -using Npgsql; - -namespace OpenShock.Common.Services.ControlStats; - -/// -/// Materializes the history of control_stats after startup instead of in a migration, which held up startup -/// for as long as it took to aggregate every control log. Until it is done stats stay correct, real-time aggregation -/// computes whatever isn't materialized yet from the logs. The refresh policy is only added once the history is -/// materialized: it moves the watermark to the present, so anything older left unmaterialized would drop out. -/// -public sealed class ControlStatsBackfillService : BackgroundService -{ - // Arbitrary, unique to this job, so only one instance backfills at a time - private const long LockKey = 0x4F53_4353_4246; // "OSCSBF" - - private static readonly TimeSpan Window = TimeSpan.FromDays(30); - private static readonly TimeSpan RetryDelay = TimeSpan.FromMinutes(1); - - private readonly string _connectionString; - private readonly ILogger _logger; - - public ControlStatsBackfillService(string connectionString, ILogger logger) - { - _connectionString = connectionString; - _logger = logger; - } - - protected override async Task ExecuteAsync(CancellationToken stoppingToken) - { - while (!stoppingToken.IsCancellationRequested) - { - try - { - // Another instance holding the lock may stop before it's done, so keep checking until the policy exists - if (await EnsureMaterializedAsync(_connectionString, _logger, stoppingToken)) return; - } - catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) - { - return; - } - catch (Exception e) - { - _logger.LogError(e, "Materializing control_stats failed, retrying in {Delay}", RetryDelay); - } - - await Task.Delay(RetryDelay, stoppingToken); - } - } - - /// - /// Materializes the history and adds the refresh policy, unless the policy already exists. Safe to run from - /// several instances and to interrupt, a later run continues from what is already materialized. - /// - /// Whether the policy exists now, false while another instance is still materializing. - public static async Task EnsureMaterializedAsync(string connectionString, ILogger logger, CancellationToken cancellationToken = default) - { - await using var connection = new NpgsqlConnection(connectionString); - await connection.OpenAsync(cancellationToken); - - if (await HasPolicyAsync(connection, cancellationToken)) return true; - - if (!await ScalarAsync(connection, $"SELECT pg_try_advisory_lock({LockKey})", cancellationToken)) - { - logger.LogInformation("control_stats is being materialized by another instance"); - return false; - } - - try - { - // Checked again, another instance may have finished between the check and taking the lock - if (await HasPolicyAsync(connection, cancellationToken)) return true; - - var stopwatch = Stopwatch.StartNew(); - await MaterializeHistoryAsync(connection, logger, cancellationToken); - - await ExecuteAsync(connection, - """ - SELECT add_continuous_aggregate_policy('control_stats', - start_offset => INTERVAL '3 days', - end_offset => INTERVAL '1 hour', - schedule_interval => INTERVAL '5 minutes', - if_not_exists => true, - initial_start => now() + INTERVAL '5 minutes') - """, cancellationToken); - - logger.LogInformation("Materialized control_stats in {Elapsed}", stopwatch.Elapsed); - return true; - } - finally - { - await ExecuteAsync(connection, $"SELECT pg_advisory_unlock({LockKey})", CancellationToken.None); - } - } - - /// - /// Refreshes control_stats oldest first, one window at a time. Each refresh commits on its own, so the - /// progress survives a restart. - /// - public static async Task MaterializeHistoryAsync(NpgsqlConnection connection, ILogger logger, CancellationToken cancellationToken = default) - { - // Bucket aligned, a refresh only covers buckets entirely inside its window - DateTime? first; - DateTime to; - await using (var command = new NpgsqlCommand( - """ - SELECT time_bucket(INTERVAL '15 minutes', min(created_at)), - time_bucket(INTERVAL '15 minutes', now() - INTERVAL '1 hour') - FROM shocker_control_logs - """, connection)) - await using (var reader = await command.ExecuteReaderAsync(cancellationToken)) - { - await reader.ReadAsync(cancellationToken); - first = await reader.IsDBNullAsync(0, cancellationToken) ? null : reader.GetDateTime(0); - to = reader.GetDateTime(1); - } - - if (first is not { } from || from >= to) - { - logger.LogInformation("No control logs to materialize"); - return; - } - - logger.LogInformation("Materializing control_stats from {From:u} to {To:u}", from, to); - - for (var start = from; start < to; start += Window) - { - var end = start + Window < to ? start + Window : to; - await RefreshAsync(connection, start, end, cancellationToken); - logger.LogInformation("Materialized control_stats up to {End:u}", end); - } - } - - private static async Task RefreshAsync(NpgsqlConnection connection, DateTime start, DateTime end, CancellationToken cancellationToken) - { - for (var attempt = 1; ; attempt++) - { - try - { - await using var command = new NpgsqlCommand("CALL refresh_continuous_aggregate('control_stats', $1, $2)", connection); - command.CommandTimeout = 0; - command.Parameters.AddWithValue(start); - command.Parameters.AddWithValue(end); - await command.ExecuteNonQueryAsync(cancellationToken); - return; - } - // Another refresh of the same range is running, TimescaleDB fails instead of waiting - catch (PostgresException e) when (e.SqlState == PostgresErrorCodes.LockNotAvailable && attempt < 10) - { - await Task.Delay(TimeSpan.FromSeconds(attempt), cancellationToken); - } - } - } - - private static Task HasPolicyAsync(NpgsqlConnection connection, CancellationToken cancellationToken) => - ScalarAsync(connection, - """ - SELECT EXISTS (SELECT 1 FROM timescaledb_information.jobs - WHERE proc_name = 'policy_refresh_continuous_aggregate' AND hypertable_name = 'control_stats') - """, cancellationToken); - - private static async Task ScalarAsync(NpgsqlConnection connection, string sql, CancellationToken cancellationToken) - { - await using var command = new NpgsqlCommand(sql, connection); - return (T)(await command.ExecuteScalarAsync(cancellationToken))!; - } - - private static async Task ExecuteAsync(NpgsqlConnection connection, string sql, CancellationToken cancellationToken) - { - await using var command = new NpgsqlCommand(sql, connection); - command.CommandTimeout = 0; - await command.ExecuteNonQueryAsync(cancellationToken); - } -} diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 01876364..ca233bd7 100644 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -2,6 +2,7 @@ set -e DLL_NAME="$1" +shift CERT_PATH="/defaultcert.pfx" @@ -19,4 +20,4 @@ if [ ! -f "$CERT_PATH" ]; then rm key.pem cert.pem fi -exec dotnet "$DLL_NAME" +exec dotnet "$DLL_NAME" "$@"