From 6e4b9226cf47988b43553206a49e6c4a41c0c6e4 Mon Sep 17 00:00:00 2001 From: LucHeart Date: Tue, 27 Oct 2026 23:02:48 +0100 Subject: [PATCH 1/6] feat: control stats from a TimescaleDB hypertable of control logs shocker_control_logs becomes a hypertable (weekly chunks, columnstore after 7 days, dropped after a year) and control_stats a real time continuous aggregate on top of it, replacing the control_stats rollup table along with its batch markers, removal tracking, Redis parking and shutdown handoff. Live control is now logged, one row per continuous burst with its average and peak intensity, written idempotently by the batch update service. Logs no longer have foreign keys to shockers and controllers, so their stats survive them being deleted. An owner's logs are deleted with their account by a trigger. The per user log cleanup job is replaced by the retention policy. Requires the TimescaleDB Community edition. docker-compose builds it on top of postgres:18, so existing data directories keep working. --- API.IntegrationTests/AssemblyAttributes.cs | 11 + .../Docker/InMemoryDatabase.cs | 2 +- .../Helpers/ControlLogHelper.cs | 72 + .../Tests/ControlStatChangesTests.cs | 99 +- .../Tests/ControlStatsTests.cs | 301 ++- API/Controller/Admin/GetUser.cs | 2 +- .../Integrations/GetControlStatChanges.cs | 34 +- API/Controller/Shockers/GetShockerLogs.cs | 137 +- API/Controller/Users/GetSelfControlStats.cs | 14 +- .../Services/ControlStatsAccumulatorTests.cs | 52 - ...201327_AddControlLogHypertable.Designer.cs | 1832 +++++++++++++++++ .../20261120201327_AddControlLogHypertable.cs | 174 ++ ...201415_MaterializeControlStats.Designer.cs | 1832 +++++++++++++++++ .../20261120201415_MaterializeControlStats.cs | 24 + .../OpenShockContextModelSnapshot.cs | 145 +- Common/OpenShockDb/Models/ControlStat.cs | 31 +- Common/OpenShockDb/Models/ControlStatBatch.cs | 13 - .../OpenShockDb/Models/ControlStatRemoval.cs | 14 - Common/OpenShockDb/Models/Shocker.cs | 2 - .../OpenShockDb/Models/ShockerControlLog.cs | 18 +- Common/OpenShockDb/Models/User.cs | 2 - Common/OpenShockDb/OpenShockContext.cs | 87 +- .../OpenShockDb/Queries/ControlStatQueries.cs | 151 -- .../Queries/ShockerControlLogQueries.cs | 45 +- .../BatchUpdate/BatchUpdateService.cs | 184 +- .../BatchUpdate/IBatchUpdateService.cs | 7 +- Common/Services/ControlSender.cs | 25 +- .../ControlStats/ControlStatsAccumulator.cs | 39 - .../Docker/InMemoryDatabase.cs | 2 +- .../Tests/ControlLogRetentionTests.cs | 136 -- Cron/Jobs/ClearOldShockerControlLogs.cs | 41 - Dev/docker-compose.yml | 19 +- .../LifetimeManager/HubLifetime.cs | 44 +- .../LifetimeManager/LiveBurst.cs | 48 + .../LifetimeManager/ShockerState.cs | 5 +- README.md | 5 +- SeedE2E/Seeders/ControlLogSeeder.cs | 26 +- docker-compose.yml | 20 +- 38 files changed, 4516 insertions(+), 1179 deletions(-) create mode 100644 API.IntegrationTests/Helpers/ControlLogHelper.cs delete mode 100644 Common.Tests/Services/ControlStatsAccumulatorTests.cs create mode 100644 Common/Migrations/20261120201327_AddControlLogHypertable.Designer.cs create mode 100644 Common/Migrations/20261120201327_AddControlLogHypertable.cs create mode 100644 Common/Migrations/20261120201415_MaterializeControlStats.Designer.cs create mode 100644 Common/Migrations/20261120201415_MaterializeControlStats.cs delete mode 100644 Common/OpenShockDb/Models/ControlStatBatch.cs delete mode 100644 Common/OpenShockDb/Models/ControlStatRemoval.cs delete mode 100644 Common/OpenShockDb/Queries/ControlStatQueries.cs delete mode 100644 Common/Services/ControlStats/ControlStatsAccumulator.cs delete mode 100644 Cron.IntegrationTests/Tests/ControlLogRetentionTests.cs delete mode 100644 Cron/Jobs/ClearOldShockerControlLogs.cs create mode 100644 LiveControlGateway/LifetimeManager/LiveBurst.cs diff --git a/API.IntegrationTests/AssemblyAttributes.cs b/API.IntegrationTests/AssemblyAttributes.cs index ecb4833be..b906e50c3 100644 --- a/API.IntegrationTests/AssemblyAttributes.cs +++ b/API.IntegrationTests/AssemblyAttributes.cs @@ -17,3 +17,14 @@ public sealed record CiSafeParallelLimit : IParallelLimit { public int Limit => Math.Max(Environment.ProcessorCount * 2, 8); } + +public static class ThreadPoolSetup +{ + // Running tests block on the shared host's boot, which connects to Redis synchronously and needs a free thread + [Before(TestSession)] + public static void RaiseMinThreads() + { + ThreadPool.GetMinThreads(out var workerThreads, out var completionPortThreads); + ThreadPool.SetMinThreads(Math.Max(workerThreads, new CiSafeParallelLimit().Limit * 2), completionPortThreads); + } +} diff --git a/API.IntegrationTests/Docker/InMemoryDatabase.cs b/API.IntegrationTests/Docker/InMemoryDatabase.cs index e12c8a1e6..1ae0c5d44 100644 --- a/API.IntegrationTests/Docker/InMemoryDatabase.cs +++ b/API.IntegrationTests/Docker/InMemoryDatabase.cs @@ -17,7 +17,7 @@ public PostgreSqlContainer Container { get { - _container ??= new PostgreSqlBuilder(image: "postgres:latest") + _container ??= new PostgreSqlBuilder(image: "timescale/timescaledb:latest-pg18") // Community edition, the schema needs it .WithNetwork(DockerNetwork.Instance) .WithName($"tunit-postgresql-{Guid.CreateVersion7()}") .WithDatabase("openshock") diff --git a/API.IntegrationTests/Helpers/ControlLogHelper.cs b/API.IntegrationTests/Helpers/ControlLogHelper.cs new file mode 100644 index 000000000..cacb22849 --- /dev/null +++ b/API.IntegrationTests/Helpers/ControlLogHelper.cs @@ -0,0 +1,72 @@ +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using OpenShock.Common.OpenShockDb; + +namespace OpenShock.API.IntegrationTests.Helpers; + +public static class ControlLogHelper +{ + // Tests run in parallel against one database + private static readonly SemaphoreSlim RefreshLock = new(1, 1); + + public static async Task CreateShocker(WebApplicationFactory factory, Guid ownerId, string name) + { + var (deviceId, _) = await TestHelper.CreateDeviceInDb(factory, ownerId); + + await using var scope = factory.Services.CreateAsyncScope(); + var db = scope.ServiceProvider.GetRequiredService(); + + var shockerId = Guid.CreateVersion7(); + db.Shockers.Add(new Shocker + { + Id = shockerId, + Name = name, + Model = ShockerModelType.CaiXianlin, + RfId = 1234, + DeviceId = deviceId + }); + await db.SaveChangesAsync(); + return shockerId; + } + + public static ShockerControlLog Log(Guid ownerId, Guid shockerId, Guid? controllerId, DateTime at, + byte intensity = 50, uint duration = 1000, ControlType type = ControlType.Shock, bool live = false, + byte? maxIntensity = null) => new() + { + Id = Guid.CreateVersion7(), + OwnerUserId = ownerId, + ShockerId = shockerId, + ControlledByUserId = controllerId, + Intensity = intensity, + MaxIntensity = maxIntensity ?? intensity, + Duration = duration, + Type = type, + CustomName = null, + LiveControl = live, + CreatedAt = at + }; + + public static async Task Insert(WebApplicationFactory factory, params ShockerControlLog[] logs) + { + await using var scope = factory.Services.CreateAsyncScope(); + return await scope.ServiceProvider.GetRequiredService().InsertControlLogsAsync(logs); + } + + /// + /// Logs older than an hour only show up in control_stats once refreshed. + /// + public static async Task RefreshControlStats(WebApplicationFactory factory) + { + await RefreshLock.WaitAsync(); + try + { + await using var scope = factory.Services.CreateAsyncScope(); + await scope.ServiceProvider.GetRequiredService().Database.ExecuteSqlRawAsync( + "CALL refresh_continuous_aggregate('control_stats', NULL, now() - INTERVAL '1 hour')"); + } + finally + { + RefreshLock.Release(); + } + } +} diff --git a/API.IntegrationTests/Tests/ControlStatChangesTests.cs b/API.IntegrationTests/Tests/ControlStatChangesTests.cs index 49ec6549a..48d260f7b 100644 --- a/API.IntegrationTests/Tests/ControlStatChangesTests.cs +++ b/API.IntegrationTests/Tests/ControlStatChangesTests.cs @@ -7,6 +7,7 @@ using OpenShock.Common.OpenShockDb; using OpenShock.Common.Services.AutomationTokens; using OpenShock.Common.Utils; +using static OpenShock.API.IntegrationTests.Helpers.ControlLogHelper; namespace OpenShock.API.IntegrationTests.Tests; @@ -37,12 +38,16 @@ public async Task InitialSync_ReportsEveryRequestedUser_WithReceivedAndGivenTota var controller = await CreateUser("chgctrl"); var idle = await CreateUser("chgidle"); var notRequested = await CreateUser("chgother"); - var shockerId = await CreateShocker(owner, "Changes"); + var shockerId = await CreateShocker(WebApplicationFactory, owner, "Changes"); var at = new DateTime(2026, 3, 1, 12, 0, 0, DateTimeKind.Utc); - await Upsert( - Delta(owner, shockerId, controller, at, count: 2, intensity: 40), - Delta(owner, shockerId, notRequested, at, count: 7)); + await Insert(WebApplicationFactory, + [ + Log(owner, shockerId, controller, at, intensity: 40), + Log(owner, shockerId, controller, at, intensity: 40), + .. Enumerable.Range(0, 7).Select(_ => Log(owner, shockerId, notRequested, at)) + ]); + await RefreshControlStats(WebApplicationFactory); using var client = await CreateTokenClient([AutomationTokenType.ControlStatsRead]); using var doc = await Post(client, new { userIds = new[] { owner, controller, idle } }); @@ -69,13 +74,16 @@ public async Task Since_OnlyReportsUsersWithRecentActivity() { var owner = await CreateUser("chgsince"); var quiet = await CreateUser("chgquiet"); - var shockerId = await CreateShocker(owner, "Since"); - var quietShockerId = await CreateShocker(quiet, "Quiet"); + var shockerId = await CreateShocker(WebApplicationFactory, owner, "Since"); + var quietShockerId = await CreateShocker(WebApplicationFactory, quiet, "Quiet"); // Old activity only, outside of any lookback var old = new DateTime(2026, 1, 1, 12, 0, 0, DateTimeKind.Utc); - await Upsert(Delta(owner, shockerId, owner, old, count: 3), Delta(quiet, quietShockerId, quiet, old, count: 1)); + await Insert(WebApplicationFactory, + Log(owner, shockerId, owner, old), Log(owner, shockerId, owner, old), Log(owner, shockerId, owner, old), + Log(quiet, quietShockerId, quiet, old)); await Backdate(owner, quiet); + await RefreshControlStats(WebApplicationFactory); using var client = await CreateTokenClient([AutomationTokenType.ControlStatsRead]); var userIds = new[] { owner, quiet }; @@ -87,7 +95,7 @@ public async Task Since_OnlyReportsUsersWithRecentActivity() using (var unchanged = await Post(client, new { since = cursor, userIds })) await Assert.That(unchanged.RootElement.GetProperty("users").GetArrayLength()).IsEqualTo(0); - await Upsert(Delta(owner, shockerId, owner, DateTime.UtcNow, count: 1)); + await Insert(WebApplicationFactory, Log(owner, shockerId, owner, DateTime.UtcNow)); using var changed = await Post(client, new { since = cursor, userIds }); var user = changed.RootElement.GetProperty("users").EnumerateArray().Single(); @@ -101,8 +109,10 @@ public async Task Since_AtMinValue_ReportsUsersWithAnyActivity() { var owner = await CreateUser("chgmin"); var idle = await CreateUser("chgminidle"); - var shockerId = await CreateShocker(owner, "MinValue"); - await Upsert(Delta(owner, shockerId, owner, new DateTime(2026, 1, 1, 12, 0, 0, DateTimeKind.Utc), count: 2)); + var shockerId = await CreateShocker(WebApplicationFactory, owner, "MinValue"); + var at = new DateTime(2026, 1, 1, 12, 0, 0, DateTimeKind.Utc); + await Insert(WebApplicationFactory, Log(owner, shockerId, owner, at), Log(owner, shockerId, owner, at)); + await RefreshControlStats(WebApplicationFactory); using var client = await CreateTokenClient([AutomationTokenType.ControlStatsRead]); @@ -118,10 +128,10 @@ public async Task Since_AtMinValue_ReportsUsersWithAnyActivity() } [Test] - public async Task Since_ReportsLateWritesIntoOldBuckets() + public async Task Since_ReportsLateWrites() { var owner = await CreateUser("chglate"); - var shockerId = await CreateShocker(owner, "Late"); + var shockerId = await CreateShocker(WebApplicationFactory, owner, "Late"); using var client = await CreateTokenClient([AutomationTokenType.ControlStatsRead]); var userIds = new[] { owner }; @@ -131,7 +141,7 @@ public async Task Since_ReportsLateWritesIntoOldBuckets() cursor = initial.RootElement.GetProperty("cursor").GetDateTime(); // Live control from hours ago that only reached the database now, e.g. after an outage - await Upsert(Delta(owner, shockerId, owner, DateTime.UtcNow.AddHours(-3), count: 1)); + await Insert(WebApplicationFactory, Log(owner, shockerId, owner, DateTime.UtcNow.AddHours(-3), live: true)); using var changed = await Post(client, new { since = cursor, userIds }); var user = changed.RootElement.GetProperty("users").EnumerateArray().Single(); @@ -139,57 +149,42 @@ public async Task Since_ReportsLateWritesIntoOldBuckets() } [Test] - public async Task Since_ReportsControllersWhoseStatsWereDeleted() + public async Task DeletedOwners_AreLeftOutOfGivenTotals() { var owner = await CreateUser("chgdelown"); var controller = await CreateUser("chgdelctl"); - var shockerId = await CreateShocker(owner, "Deleted"); - await Upsert(Delta(owner, shockerId, controller, new DateTime(2026, 1, 1, 12, 0, 0, DateTimeKind.Utc), count: 4)); - await Backdate(owner); + var shockerId = await CreateShocker(WebApplicationFactory, owner, "Deleted"); + await Insert(WebApplicationFactory, + Log(owner, shockerId, controller, new DateTime(2026, 1, 1, 12, 0, 0, DateTimeKind.Utc))); + await RefreshControlStats(WebApplicationFactory); - using var client = await CreateTokenClient([AutomationTokenType.ControlStatsRead]); - var userIds = new[] { controller }; - - DateTime cursor; - using (var initial = await Post(client, new { userIds })) - cursor = initial.RootElement.GetProperty("cursor").GetDateTime(); - - // As when the owner's account is deleted, which cascades onto their stats await using (var scope = WebApplicationFactory.Services.CreateAsyncScope()) { var db = scope.ServiceProvider.GetRequiredService(); - await db.ControlStats.Where(x => x.OwnerUserId == owner).ExecuteDeleteAsync(); + await db.Users.Where(x => x.Id == owner).ExecuteDeleteAsync(); } - using var changed = await Post(client, new { since = cursor, userIds }); - var user = changed.RootElement.GetProperty("users").EnumerateArray().Single(); + using var client = await CreateTokenClient([AutomationTokenType.ControlStatsRead]); + using var doc = await Post(client, new { userIds = new[] { controller } }); + + var user = doc.RootElement.GetProperty("users").EnumerateArray().Single(); await Assert.That(user.GetProperty("userId").GetGuid()).IsEqualTo(controller); await Assert.That(user.GetProperty("given").GetArrayLength()).IsEqualTo(0); } - /// Moves the users' rows out of the change lookback, as if written long ago. + /// Moves the users' logs out of the change lookback, as if written long ago. private async Task Backdate(params Guid[] ownerIds) { await using var scope = WebApplicationFactory.Services.CreateAsyncScope(); var db = scope.ServiceProvider.GetRequiredService(); var longAgo = DateTime.UtcNow.AddDays(-1); - await db.ControlStats.Where(x => ownerIds.Contains(x.OwnerUserId)) - .ExecuteUpdateAsync(x => x.SetProperty(y => y.UpdatedAt, longAgo)); + await db.ShockerControlLogs.Where(x => ownerIds.Contains(x.OwnerUserId)) + .ExecuteUpdateAsync(x => x.SetProperty(y => y.InsertedAt, longAgo)); } - private static ControlStatDelta Delta(Guid ownerId, Guid shockerId, Guid? controllerId, DateTime at, long count, byte intensity = 50) => - new(ownerId, shockerId, controllerId, at, ControlType.Shock, Live: false, count, DurationMs: count * 1000, - IntensityMs: count * 1000 * intensity, intensity); - private static JsonElement SingleShock(JsonElement totals) => totals.EnumerateArray() .Single(x => x.GetProperty("type").GetString() == "Shock" && !x.GetProperty("live").GetBoolean()); - private async Task Upsert(params ControlStatDelta[] deltas) - { - await using var scope = WebApplicationFactory.Services.CreateAsyncScope(); - await scope.ServiceProvider.GetRequiredService().UpsertControlStatsAsync(deltas); - } - private static async Task Post(HttpClient client, object body) { var response = await client.PostAsync(Endpoint, TestHelper.JsonContent(body)); @@ -219,24 +214,4 @@ private async Task CreateTokenClient(List types client.DefaultRequestHeaders.Add(AuthConstants.AutomationTokenHeaderName, secret); return client; } - - private async Task CreateShocker(Guid ownerId, string name) - { - var (deviceId, _) = await TestHelper.CreateDeviceInDb(WebApplicationFactory, ownerId); - - await using var scope = WebApplicationFactory.Services.CreateAsyncScope(); - var db = scope.ServiceProvider.GetRequiredService(); - - var shockerId = Guid.CreateVersion7(); - db.Shockers.Add(new Shocker - { - Id = shockerId, - Name = name, - Model = ShockerModelType.CaiXianlin, - RfId = 1234, - DeviceId = deviceId - }); - await db.SaveChangesAsync(); - return shockerId; - } -} +} \ No newline at end of file diff --git a/API.IntegrationTests/Tests/ControlStatsTests.cs b/API.IntegrationTests/Tests/ControlStatsTests.cs index 620925ba4..9f82738e9 100644 --- a/API.IntegrationTests/Tests/ControlStatsTests.cs +++ b/API.IntegrationTests/Tests/ControlStatsTests.cs @@ -4,23 +4,27 @@ using Microsoft.Extensions.DependencyInjection; using OpenShock.API.IntegrationTests.Helpers; using OpenShock.Common.OpenShockDb; +using static OpenShock.API.IntegrationTests.Helpers.ControlLogHelper; namespace OpenShock.API.IntegrationTests.Tests; public sealed class ControlStatsTests { + private const string Password = "SecurePassword123#"; + [ClassDataSource(Shared = SharedType.PerTestSession)] public required WebApplicationFactory WebApplicationFactory { get; init; } [Test] - public async Task SendControl_RecordsStats_ThatSurviveLogDeletion() + public async Task SendControl_IsCountedInStats_RightAway() { var user = await TestHelper.CreateAndLoginUser(WebApplicationFactory, TestHelper.UniqueUsername("stats"), - TestHelper.UniqueEmail("stats"), "SecurePassword123#"); - var shockerId = await CreateShocker(user.Id, "Stats Shocker"); + TestHelper.UniqueEmail("stats"), Password); + var shockerId = await CreateShocker(WebApplicationFactory, user.Id, "Stats Shocker"); using var client = TestHelper.CreateAuthenticatedClient(WebApplicationFactory, user.SessionToken); - foreach (var (type, intensity, duration) in new[] { ("Shock", 50, 1000), ("Shock", 100, 2000), ("Vibrate", 30, 500), ("Stop", 1, 300) }) + foreach (var (type, intensity, duration) in new[] + { ("Shock", 50, 1000), ("Shock", 100, 2000), ("Vibrate", 30, 500), ("Stop", 1, 300) }) { var response = await client.PostAsync("/2/shockers/control", TestHelper.JsonContent(new { @@ -29,13 +33,6 @@ public async Task SendControl_RecordsStats_ThatSurviveLogDeletion() await Assert.That(response.StatusCode).IsEqualTo(HttpStatusCode.OK); } - // The rollup must not depend on the logs, which get trimmed - await using (var scope = WebApplicationFactory.Services.CreateAsyncScope()) - { - var db = scope.ServiceProvider.GetRequiredService(); - await db.ShockerControlLogs.Where(x => x.ShockerId == shockerId).ExecuteDeleteAsync(); - } - using var doc = await GetStats(client); var received = doc.RootElement.GetProperty("received"); @@ -45,7 +42,8 @@ public async Task SendControl_RecordsStats_ThatSurviveLogDeletion() var shock = FindTotals(totals, "Shock", live: false); await Assert.That(shock.GetProperty("count").GetInt64()).IsEqualTo(2); await Assert.That(shock.GetProperty("durationMs").GetInt64()).IsEqualTo(3000); - await Assert.That(shock.GetProperty("averageIntensity").GetDouble()).IsEqualTo((50.0 * 1000 + 100.0 * 2000) / 3000).Within(0.001); + await Assert.That(shock.GetProperty("averageIntensity").GetDouble()) + .IsEqualTo((50.0 * 1000 + 100.0 * 2000) / 3000).Within(0.001); await Assert.That(shock.GetProperty("maxIntensity").GetInt32()).IsEqualTo(100); var vibrate = FindTotals(totals, "Vibrate", live: false); @@ -63,154 +61,181 @@ public async Task SendControl_RecordsStats_ThatSurviveLogDeletion() // Controlling your own shocker counts as given too var given = doc.RootElement.GetProperty("given"); - await Assert.That(FindTotals(given.GetProperty("totals"), "Shock", live: false).GetProperty("count").GetInt64()).IsEqualTo(2); - await Assert.That(given.GetProperty("owners").EnumerateArray().Single().GetProperty("userId").GetGuid()).IsEqualTo(user.Id); + await Assert.That(FindTotals(given.GetProperty("totals"), "Shock", live: false).GetProperty("count").GetInt64()) + .IsEqualTo(2); + await Assert.That(given.GetProperty("owners").EnumerateArray().Single().GetProperty("userId").GetGuid()) + .IsEqualTo(user.Id); } [Test] - public async Task DeletingShockers_KeepsTheirStats_WithoutUniqueConflicts() + public async Task LiveControlBursts_CountOnce_WithWeightedAverageAndPeakIntensity() { - var user = await TestHelper.CreateAndLoginUser(WebApplicationFactory, TestHelper.UniqueUsername("statsdel"), - TestHelper.UniqueEmail("statsdel"), "SecurePassword123#"); - var shockerA = await CreateShocker(user.Id, "A"); - var shockerB = await CreateShocker(user.Id, "B"); - var bucket = new DateTime(2026, 3, 1, 12, 15, 0, DateTimeKind.Utc); - - await using (var scope = WebApplicationFactory.Services.CreateAsyncScope()) - { - var db = scope.ServiceProvider.GetRequiredService(); - - // Identical keys apart from the shocker, so after SET NULL they would collide on a full unique index - await db.UpsertControlStatsAsync([ - Delta(user.Id, shockerA, user.Id, bucket, count: 2), - Delta(user.Id, shockerB, user.Id, bucket, count: 3) - ]); + var user = await TestHelper.CreateAndLoginUser(WebApplicationFactory, TestHelper.UniqueUsername("statslive"), + TestHelper.UniqueEmail("statslive"), Password); + var shockerId = await CreateShocker(WebApplicationFactory, user.Id, "Live"); + var now = DateTime.UtcNow; - await db.Shockers.Where(x => x.Id == shockerA).ExecuteDeleteAsync(); - await db.Shockers.Where(x => x.Id == shockerB).ExecuteDeleteAsync(); - - await Assert.That(await db.ControlStats.CountAsync(x => x.OwnerUserId == user.Id && x.ShockerId == null)).IsEqualTo(2); - } + await Insert(WebApplicationFactory, + Log(user.Id, shockerId, user.Id, now, intensity: 30, duration: 2000, live: true, maxIntensity: 80), + Log(user.Id, shockerId, user.Id, now, intensity: 60, duration: 1000, live: true)); using var client = TestHelper.CreateAuthenticatedClient(WebApplicationFactory, user.SessionToken); using var doc = await GetStats(client); - var deleted = doc.RootElement.GetProperty("received").GetProperty("shockers").EnumerateArray().Single(); - await Assert.That(deleted.GetProperty("shockerId").ValueKind).IsEqualTo(JsonValueKind.Null); - await Assert.That(FindTotals(deleted.GetProperty("stats"), "Shock", live: false).GetProperty("count").GetInt64()).IsEqualTo(5); + var live = FindTotals(doc.RootElement.GetProperty("received").GetProperty("totals"), "Shock", live: true); + await Assert.That(live.GetProperty("count").GetInt64()).IsEqualTo(2); + await Assert.That(live.GetProperty("durationMs").GetInt64()).IsEqualTo(3000); + await Assert.That(live.GetProperty("averageIntensity").GetDouble()) + .IsEqualTo((30.0 * 2000 + 60.0 * 1000) / 3000).Within(0.001); + await Assert.That(live.GetProperty("maxIntensity").GetInt32()).IsEqualTo(80); } [Test] - public async Task Upsert_MergesDuplicateKeysAndAccumulates() + public async Task Insert_RetriedWithSameIds_IsCountedOnce() { - var userId = await TestHelper.CreateUserInDb(WebApplicationFactory, TestHelper.UniqueUsername("statsmerge"), - TestHelper.UniqueEmail("statsmerge"), "SecurePassword123#"); - var shockerId = await CreateShocker(userId, "Merge"); - var bucket = new DateTime(2026, 3, 1, 12, 0, 0, DateTimeKind.Utc); + var userId = await TestHelper.CreateUserInDb(WebApplicationFactory, TestHelper.UniqueUsername("statsretry"), + TestHelper.UniqueEmail("statsretry"), Password); + var shockerId = await CreateShocker(WebApplicationFactory, userId, "Retry"); + ShockerControlLog[] logs = [Log(userId, shockerId, userId, DateTime.UtcNow, live: true)]; + + await Assert.That(await Insert(WebApplicationFactory, logs)).IsEqualTo(1); + // Retry after a commit whose acknowledgement got lost + await Assert.That(await Insert(WebApplicationFactory, logs)).IsEqualTo(0); await using var scope = WebApplicationFactory.Services.CreateAsyncScope(); var db = scope.ServiceProvider.GetRequiredService(); - - // Same bucket, same key twice in one batch, plus a guest (null controller) bucket - await db.UpsertControlStatsAsync([ - Delta(userId, shockerId, userId, bucket.AddMinutes(5), count: 1, intensity: 20), - Delta(userId, shockerId, userId, bucket.AddMinutes(14), count: 1, intensity: 80), - Delta(userId, shockerId, null, bucket, count: 4) - ]); - await db.UpsertControlStatsAsync([Delta(userId, shockerId, userId, bucket, count: 1, intensity: 10)]); - await db.UpsertControlStatsAsync([Delta(userId, shockerId, null, bucket, count: 1)]); - - var rows = await db.ControlStats.AsNoTracking().Where(x => x.ShockerId == shockerId).ToArrayAsync(); - await Assert.That(rows.Length).IsEqualTo(2); - - var own = rows.Single(x => x.ControllerUserId == userId); - await Assert.That(own.OwnerUserId).IsEqualTo(userId); - await Assert.That(own.Bucket).IsEqualTo(bucket); - await Assert.That(own.Count).IsEqualTo(3); - await Assert.That(own.DurationMs).IsEqualTo(3000); - await Assert.That(own.MaxIntensity).IsEqualTo((byte)80); - - await Assert.That(rows.Single(x => x.ControllerUserId == null).Count).IsEqualTo(5); + await Assert.That(await db.ControlStats.Where(x => x.ShockerId == shockerId).SumAsync(x => x.Count)) + .IsEqualTo(1); } [Test] - public async Task Upsert_ForShockerDeletedBeforeFlush_KeepsDeltaWithoutShocker() + public async Task Insert_ForOwnerDeletedBeforeFlush_IsDropped() { var userId = await TestHelper.CreateUserInDb(WebApplicationFactory, TestHelper.UniqueUsername("statsgone"), - TestHelper.UniqueEmail("statsgone"), "SecurePassword123#"); - var shockerId = await CreateShocker(userId, "Gone"); + TestHelper.UniqueEmail("statsgone"), Password); + var shockerId = await CreateShocker(WebApplicationFactory, userId, "Gone"); - await using var scope = WebApplicationFactory.Services.CreateAsyncScope(); - var db = scope.ServiceProvider.GetRequiredService(); - - // Buffered live control activity whose shocker got deleted before the flush - await db.Shockers.Where(x => x.Id == shockerId).ExecuteDeleteAsync(); - await db.UpsertControlStatsAsync([ - Delta(userId, shockerId, userId, new DateTime(2026, 3, 1, 12, 0, 0, DateTimeKind.Utc), count: 3) - ]); + await using (var scope = WebApplicationFactory.Services.CreateAsyncScope()) + { + var db = scope.ServiceProvider.GetRequiredService(); + await db.Users.Where(x => x.Id == userId).ExecuteDeleteAsync(); + } - var row = await db.ControlStats.AsNoTracking().SingleAsync(x => x.OwnerUserId == userId); - await Assert.That(row.ShockerId).IsNull(); - await Assert.That(row.Count).IsEqualTo(3); + // Buffered live control whose owner deleted their account before the flush + await Assert + .That(await Insert(WebApplicationFactory, Log(userId, shockerId, userId, DateTime.UtcNow, live: true))) + .IsEqualTo(0); } [Test] - public async Task ApplyBatch_RetriedWithSameId_IsCountedOnce() + public async Task DeletingShockers_KeepsTheirStats_CombinedIntoOneEntry() { - var userId = await TestHelper.CreateUserInDb(WebApplicationFactory, TestHelper.UniqueUsername("statsbatch"), - TestHelper.UniqueEmail("statsbatch"), "SecurePassword123#"); - var shockerId = await CreateShocker(userId, "Batch"); - var batchId = Guid.CreateVersion7(); - ControlStatDelta[] deltas = [Delta(userId, shockerId, userId, new DateTime(2026, 3, 1, 12, 0, 0, DateTimeKind.Utc), count: 2)]; + var user = await TestHelper.CreateAndLoginUser(WebApplicationFactory, TestHelper.UniqueUsername("statsdel"), + TestHelper.UniqueEmail("statsdel"), Password); + var shockerA = await CreateShocker(WebApplicationFactory, user.Id, "A"); + var shockerB = await CreateShocker(WebApplicationFactory, user.Id, "B"); + var kept = await CreateShocker(WebApplicationFactory, user.Id, "Kept"); + var now = DateTime.UtcNow; + + await Insert(WebApplicationFactory, + Log(user.Id, shockerA, user.Id, now), Log(user.Id, shockerA, user.Id, now), + Log(user.Id, shockerB, user.Id, now), Log(user.Id, shockerB, user.Id, now), + Log(user.Id, shockerB, user.Id, now), + Log(user.Id, kept, user.Id, now)); - await using var scope = WebApplicationFactory.Services.CreateAsyncScope(); - var db = scope.ServiceProvider.GetRequiredService(); + await using (var scope = WebApplicationFactory.Services.CreateAsyncScope()) + { + var db = scope.ServiceProvider.GetRequiredService(); + await db.Shockers.Where(x => x.Id == shockerA || x.Id == shockerB).ExecuteDeleteAsync(); + } - var createdAt = DateTime.UtcNow; - await Assert.That(await db.ApplyControlStatBatchAsync(batchId, createdAt, deltas)).IsEqualTo(ControlStatBatchResult.Applied); - // Retry after a commit whose acknowledgement got lost - await Assert.That(await db.ApplyControlStatBatchAsync(batchId, createdAt, deltas)).IsEqualTo(ControlStatBatchResult.AlreadyApplied); + using var client = TestHelper.CreateAuthenticatedClient(WebApplicationFactory, user.SessionToken); + using var doc = await GetStats(client); + + var shockers = doc.RootElement.GetProperty("received").GetProperty("shockers").EnumerateArray().ToArray(); + await Assert.That(shockers.Length).IsEqualTo(2); - var row = await db.ControlStats.AsNoTracking().SingleAsync(x => x.ShockerId == shockerId); - await Assert.That(row.Count).IsEqualTo(2); + var deleted = shockers.Single(x => x.GetProperty("shockerId").ValueKind == JsonValueKind.Null); + await Assert.That(deleted.GetProperty("shockerName").ValueKind).IsEqualTo(JsonValueKind.Null); + await Assert + .That(FindTotals(deleted.GetProperty("stats"), "Shock", live: false).GetProperty("count").GetInt64()) + .IsEqualTo(5); + + var remaining = shockers.Single(x => x.GetProperty("shockerId").ValueKind != JsonValueKind.Null); + await Assert.That(remaining.GetProperty("shockerName").GetString()).IsEqualTo("Kept"); } [Test] - public async Task ApplyBatch_OlderThanReplayLifetime_IsRefused() + public async Task DeletingAController_KeepsTheirStats() { - var userId = await TestHelper.CreateUserInDb(WebApplicationFactory, TestHelper.UniqueUsername("statsexp"), - TestHelper.UniqueEmail("statsexp"), "SecurePassword123#"); - var shockerId = await CreateShocker(userId, "Expired"); - var createdAt = DateTime.UtcNow - ControlStatQueries.BatchReplayLifetime - TimeSpan.FromMinutes(1); + var owner = await TestHelper.CreateAndLoginUser(WebApplicationFactory, TestHelper.UniqueUsername("statsctlown"), + TestHelper.UniqueEmail("statsctlown"), Password); + var controllerId = await TestHelper.CreateUserInDb(WebApplicationFactory, TestHelper.UniqueUsername("statsctl"), + TestHelper.UniqueEmail("statsctl"), Password); + var shockerId = await CreateShocker(WebApplicationFactory, owner.Id, "Controlled"); - await using var scope = WebApplicationFactory.Services.CreateAsyncScope(); - var db = scope.ServiceProvider.GetRequiredService(); + await Insert(WebApplicationFactory, Log(owner.Id, shockerId, controllerId, DateTime.UtcNow)); - // Its marker may already be pruned, so applying it could count it a second time - var result = await db.ApplyControlStatBatchAsync(Guid.CreateVersion7(), createdAt, - [Delta(userId, shockerId, userId, createdAt, count: 1)]); + await using (var scope = WebApplicationFactory.Services.CreateAsyncScope()) + { + var db = scope.ServiceProvider.GetRequiredService(); + await db.Users.Where(x => x.Id == controllerId).ExecuteDeleteAsync(); + } + + using var client = TestHelper.CreateAuthenticatedClient(WebApplicationFactory, owner.SessionToken); + using var doc = await GetStats(client); - await Assert.That(result).IsEqualTo(ControlStatBatchResult.Expired); - await Assert.That(await db.ControlStats.AnyAsync(x => x.ShockerId == shockerId)).IsFalse(); + var controller = doc.RootElement.GetProperty("received").GetProperty("controllers").EnumerateArray().Single(); + await Assert.That(controller.GetProperty("userId").GetGuid()).IsEqualTo(controllerId); + await Assert.That(controller.GetProperty("user").ValueKind).IsEqualTo(JsonValueKind.Null); + await Assert + .That(FindTotals(controller.GetProperty("stats"), "Shock", live: false).GetProperty("count").GetInt64()) + .IsEqualTo(1); } [Test] - public async Task Stats_GroupsDaysInRequestedTimeZone_AndFiltersRange() + public async Task DeletingTheOwner_DeletesTheirLogs_AndTheirStatsForControllers() { - var user = await TestHelper.CreateAndLoginUser(WebApplicationFactory, TestHelper.UniqueUsername("statstz"), - TestHelper.UniqueEmail("statstz"), "SecurePassword123#"); - var shockerId = await CreateShocker(user.Id, "Tz"); + var ownerId = await TestHelper.CreateUserInDb(WebApplicationFactory, TestHelper.UniqueUsername("statsownd"), + TestHelper.UniqueEmail("statsownd"), Password); + var controller = await TestHelper.CreateAndLoginUser(WebApplicationFactory, + TestHelper.UniqueUsername("statsownctl"), + TestHelper.UniqueEmail("statsownctl"), Password); + var shockerId = await CreateShocker(WebApplicationFactory, ownerId, "Owned"); + + await Insert(WebApplicationFactory, + Log(ownerId, shockerId, controller.Id, new DateTime(2026, 2, 1, 12, 0, 0, DateTimeKind.Utc)), + Log(ownerId, shockerId, controller.Id, DateTime.UtcNow)); + await RefreshControlStats(WebApplicationFactory); await using (var scope = WebApplicationFactory.Services.CreateAsyncScope()) { var db = scope.ServiceProvider.GetRequiredService(); - await db.UpsertControlStatsAsync([ - // 23:00 UTC on Jan 1 is already Jan 2 in Berlin (UTC+1) - Delta(user.Id, shockerId, user.Id, new DateTime(2026, 1, 1, 23, 0, 0, DateTimeKind.Utc), count: 1), - Delta(user.Id, shockerId, user.Id, new DateTime(2026, 1, 5, 12, 0, 0, DateTimeKind.Utc), count: 1) - ]); + await db.Users.Where(x => x.Id == ownerId).ExecuteDeleteAsync(); + await Assert.That(await db.ShockerControlLogs.AnyAsync(x => x.OwnerUserId == ownerId)).IsFalse(); } + using var client = TestHelper.CreateAuthenticatedClient(WebApplicationFactory, controller.SessionToken); + using var doc = await GetStats(client); + + var given = doc.RootElement.GetProperty("given"); + await Assert.That(given.GetProperty("totals").GetArrayLength()).IsEqualTo(0); + await Assert.That(given.GetProperty("owners").GetArrayLength()).IsEqualTo(0); + } + + [Test] + public async Task Stats_GroupsDaysInRequestedTimeZone_AndFiltersRange() + { + var user = await TestHelper.CreateAndLoginUser(WebApplicationFactory, TestHelper.UniqueUsername("statstz"), + TestHelper.UniqueEmail("statstz"), Password); + var shockerId = await CreateShocker(WebApplicationFactory, user.Id, "Tz"); + + await Insert(WebApplicationFactory, + // 23:00 UTC on Jan 1 is already Jan 2 in Berlin (UTC+1) + Log(user.Id, shockerId, user.Id, new DateTime(2026, 1, 1, 23, 0, 0, DateTimeKind.Utc)), + Log(user.Id, shockerId, user.Id, new DateTime(2026, 1, 5, 12, 0, 0, DateTimeKind.Utc))); + await RefreshControlStats(WebApplicationFactory); + using var client = TestHelper.CreateAuthenticatedClient(WebApplicationFactory, user.SessionToken); using (var utc = await GetStats(client)) @@ -227,18 +252,15 @@ await db.UpsertControlStatsAsync([ public async Task Stats_HalfHourOffsetTimeZone_SplitsDaysAtLocalMidnight() { var user = await TestHelper.CreateAndLoginUser(WebApplicationFactory, TestHelper.UniqueUsername("statskol"), - TestHelper.UniqueEmail("statskol"), "SecurePassword123#"); - var shockerId = await CreateShocker(user.Id, "Kolkata"); + TestHelper.UniqueEmail("statskol"), Password); + var shockerId = await CreateShocker(WebApplicationFactory, user.Id, "Kolkata"); - await using (var scope = WebApplicationFactory.Services.CreateAsyncScope()) - { - var db = scope.ServiceProvider.GetRequiredService(); - await db.UpsertControlStatsAsync([ - // Midnight in Kolkata (UTC+5:30) is 18:30 UTC, in the middle of an hour - Delta(user.Id, shockerId, user.Id, new DateTime(2026, 1, 1, 18, 15, 0, DateTimeKind.Utc), count: 1), - Delta(user.Id, shockerId, user.Id, new DateTime(2026, 1, 1, 18, 45, 0, DateTimeKind.Utc), count: 2) - ]); - } + await Insert(WebApplicationFactory, + // Midnight in Kolkata (UTC+5:30) is 18:30 UTC, in the middle of an hour + Log(user.Id, shockerId, user.Id, new DateTime(2026, 1, 1, 18, 15, 0, DateTimeKind.Utc)), + Log(user.Id, shockerId, user.Id, new DateTime(2026, 1, 1, 18, 45, 0, DateTimeKind.Utc)), + Log(user.Id, shockerId, user.Id, new DateTime(2026, 1, 1, 18, 50, 0, DateTimeKind.Utc))); + await RefreshControlStats(WebApplicationFactory); using var client = TestHelper.CreateAuthenticatedClient(WebApplicationFactory, user.SessionToken); @@ -247,7 +269,8 @@ await db.UpsertControlStatsAsync([ using var secondDay = await GetStats(client, "?tz=Asia/Kolkata&from=2026-01-02"); await Assert.That(Days(secondDay)).IsEquivalentTo(new[] { "2026-01-02" }); - await Assert.That(FindTotals(secondDay.RootElement.GetProperty("received").GetProperty("totals"), "Shock", live: false) + await Assert.That(FindTotals(secondDay.RootElement.GetProperty("received").GetProperty("totals"), "Shock", + live: false) .GetProperty("count").GetInt64()).IsEqualTo(2); } @@ -255,7 +278,7 @@ await Assert.That(FindTotals(secondDay.RootElement.GetProperty("received").GetPr public async Task Stats_InvalidTimeZoneOrRange_Returns400() { var user = await TestHelper.CreateAndLoginUser(WebApplicationFactory, TestHelper.UniqueUsername("statsbad"), - TestHelper.UniqueEmail("statsbad"), "SecurePassword123#"); + TestHelper.UniqueEmail("statsbad"), Password); using var client = TestHelper.CreateAuthenticatedClient(WebApplicationFactory, user.SessionToken); var badTz = await client.GetAsync("/1/users/self/control-stats?tz=Not/AZone"); @@ -273,30 +296,6 @@ public async Task Stats_Unauthenticated_Returns401() await Assert.That(response.StatusCode).IsEqualTo(HttpStatusCode.Unauthorized); } - private static ControlStatDelta Delta(Guid ownerId, Guid shockerId, Guid? controllerId, DateTime at, long count, byte intensity = 50) => - new(ownerId, shockerId, controllerId, at, ControlType.Shock, Live: false, count, DurationMs: count * 1000, - IntensityMs: count * 1000 * intensity, intensity); - - private async Task CreateShocker(Guid ownerId, string name) - { - var (deviceId, _) = await TestHelper.CreateDeviceInDb(WebApplicationFactory, ownerId); - - await using var scope = WebApplicationFactory.Services.CreateAsyncScope(); - var db = scope.ServiceProvider.GetRequiredService(); - - var shockerId = Guid.CreateVersion7(); - db.Shockers.Add(new Shocker - { - Id = shockerId, - Name = name, - Model = ShockerModelType.CaiXianlin, - RfId = 1234, - DeviceId = deviceId - }); - await db.SaveChangesAsync(); - return shockerId; - } - private static async Task GetStats(HttpClient client, string query = "") { var response = await client.GetAsync("/1/users/self/control-stats" + query); @@ -309,4 +308,4 @@ private static JsonElement FindTotals(JsonElement totals, string type, bool live private static string[] Days(JsonDocument doc) => doc.RootElement.GetProperty("received").GetProperty("days") .EnumerateArray().Select(x => x.GetProperty("date").GetString()!).ToArray(); -} +} \ No newline at end of file diff --git a/API/Controller/Admin/GetUser.cs b/API/Controller/Admin/GetUser.cs index 32e269beb..67d5a2e29 100644 --- a/API/Controller/Admin/GetUser.cs +++ b/API/Controller/Admin/GetUser.cs @@ -182,7 +182,7 @@ public sealed class AdminUserView CreatedAt = change.CreatedAt, } ).ToArray(), - ShockerControlLogsCount = u.ShockerControlLogs.Count() + ShockerControlLogsCount = _db.ShockerControlLogs.Count(l => l.ControlledByUserId == u.Id) }) .FirstOrDefaultAsync(cancellationToken); diff --git a/API/Controller/Integrations/GetControlStatChanges.cs b/API/Controller/Integrations/GetControlStatChanges.cs index 21bc35465..a05156b0f 100644 --- a/API/Controller/Integrations/GetControlStatChanges.cs +++ b/API/Controller/Integrations/GetControlStatChanges.cs @@ -20,11 +20,8 @@ public sealed partial class IntegrationsController /// /// /// Requires an automation token granting ControlStatsRead. "Changed since" is answered from when each - /// stats row was last written rather than the time it covers, so live control stats written late (e.g. after a - /// database outage) are still picked up. Rows deleted along with their owner lower the totals of everyone who - /// controlled them without leaving a row behind, those are picked up from . The - /// lookback is widened by to cover writes still committing while the previous call - /// read, so a user can be reported again although nothing changed. + /// control log was written, so late writes are still picked up. The lookback is widened by + /// , so a user can be reported again although nothing changed. /// /// The changed users' totals and the cursor to pass next time. /// No automation token granting ControlStatsRead was presented. @@ -53,7 +50,10 @@ public async Task GetControlStatChanges(ControlStatChangesRequest var from = since - DateTime.MinValue > ChangeOverlap ? since - ChangeOverlap : DateTime.SpecifyKind(DateTime.MinValue, DateTimeKind.Utc); - var recent = _db.ControlStats.Where(x => x.UpdatedAt >= from); + var createdFrom = from - DateTime.MinValue > LateWriteWindow + ? from - LateWriteWindow + : DateTime.SpecifyKind(DateTime.MinValue, DateTimeKind.Utc); + var recent = _db.ShockerControlLogs.Where(x => x.InsertedAt >= from && x.CreatedAt >= createdFrom); var owners = await recent .Where(x => userIds.Contains(x.OwnerUserId)) @@ -61,18 +61,12 @@ public async Task GetControlStatChanges(ControlStatChangesRequest .Distinct() .ToArrayAsync(cancellationToken); var controllers = await recent - .Where(x => x.ControllerUserId != null && userIds.Contains(x.ControllerUserId.Value)) - .Select(x => x.ControllerUserId!.Value) + .Where(x => x.ControlledByUserId != null && userIds.Contains(x.ControlledByUserId.Value)) + .Select(x => x.ControlledByUserId!.Value) .Distinct() .ToArrayAsync(cancellationToken); - // Deletions leave no row to be updated, so they are recorded separately - var removed = await _db.ControlStatRemovals - .Where(x => x.RemovedAt >= from && userIds.Contains(x.UserId)) - .Select(x => x.UserId) - .ToArrayAsync(cancellationToken); - - changed = owners.Union(controllers).Union(removed).ToArray(); + changed = owners.Union(controllers).ToArray(); } else { @@ -90,8 +84,10 @@ public async Task GetControlStatChanges(ControlStatChangesRequest g.Sum(x => x.DurationMs), g.Sum(x => x.IntensityMs), g.Max(x => x.MaxIntensity))) .ToArrayAsync(cancellationToken); + // Materialized stats outlive the logs of deleted owners var given = await _db.ControlStats .Where(x => x.ControllerUserId != null && changed.Contains(x.ControllerUserId.Value)) + .Where(x => _db.Users.Any(u => u.Id == x.OwnerUserId)) .GroupBy(x => new { UserId = x.ControllerUserId!.Value, x.Type, x.Live }) .Select(g => new TotalsRow(g.Key.UserId, g.Key.Type, g.Key.Live, g.Sum(x => x.Count), g.Sum(x => x.DurationMs), g.Sum(x => x.IntensityMs), g.Max(x => x.MaxIntensity))) @@ -115,10 +111,12 @@ public async Task GetControlStatChanges(ControlStatChangesRequest } /// - /// Well above how long a stats write takes to commit, and any clock skew between the API and the database, since - /// is stamped by the database when its transaction starts. + /// Above the 5 minute refresh interval of control_stats, logs written more than an hour late only count + /// once it refreshed. /// - private static readonly TimeSpan ChangeOverlap = TimeSpan.FromMinutes(5); + private static readonly TimeSpan ChangeOverlap = TimeSpan.FromMinutes(15); + + private static readonly TimeSpan LateWriteWindow = TimeSpan.FromDays(1); private static ControlStatTotals[] ToTotals(IEnumerable rows) => rows .OrderBy(x => x.Type) diff --git a/API/Controller/Shockers/GetShockerLogs.cs b/API/Controller/Shockers/GetShockerLogs.cs index 9236a3c2a..3733731eb 100644 --- a/API/Controller/Shockers/GetShockerLogs.cs +++ b/API/Controller/Shockers/GetShockerLogs.cs @@ -19,17 +19,24 @@ public sealed partial class ShockerController { private const string AllLogsDefaultSort = "createdOn"; - private static readonly IReadOnlyDictionary> AllLogsSorters = - new Dictionary>(StringComparer.OrdinalIgnoreCase) + private static readonly IReadOnlyDictionary> AllLogsSorters = + new Dictionary>(StringComparer.OrdinalIgnoreCase) { - [AllLogsDefaultSort] = (q, desc) => desc ? q.OrderByDescending(x => x.CreatedAt) : q.OrderBy(x => x.CreatedAt), - ["intensity"] = (q, desc) => desc ? q.OrderByDescending(x => x.Intensity) : q.OrderBy(x => x.Intensity), - ["duration"] = (q, desc) => desc ? q.OrderByDescending(x => x.Duration) : q.OrderBy(x => x.Duration), - ["type"] = (q, desc) => desc ? q.OrderByDescending(x => x.Type) : q.OrderBy(x => x.Type), + [AllLogsDefaultSort] = (q, desc) => desc ? q.OrderByDescending(x => x.Log.CreatedAt) : q.OrderBy(x => x.Log.CreatedAt), + ["intensity"] = (q, desc) => desc ? q.OrderByDescending(x => x.Log.Intensity) : q.OrderBy(x => x.Log.Intensity), + ["duration"] = (q, desc) => desc ? q.OrderByDescending(x => x.Log.Duration) : q.OrderBy(x => x.Log.Duration), + ["type"] = (q, desc) => desc ? q.OrderByDescending(x => x.Log.Type) : q.OrderBy(x => x.Log.Type), ["hubName"] = (q, desc) => desc ? q.OrderByDescending(x => x.Shocker.Device.Name) : q.OrderBy(x => x.Shocker.Device.Name), ["shockerName"] = (q, desc) => desc ? q.OrderByDescending(x => x.Shocker.Name) : q.OrderBy(x => x.Shocker.Name), }; + private sealed class LogRow + { + public required ShockerControlLog Log { get; init; } + public required Shocker Shocker { get; init; } + public required User? Controller { get; init; } + } + /// /// Get the logs for a shocker /// @@ -51,34 +58,47 @@ public async Task GetShockerLogs([FromRoute] Guid shockerId, [Fro var exists = await _db.Shockers.AnyAsync(x => x.Device.OwnerId == CurrentUser.Id && x.Id == shockerId, cancellationToken); if (!exists) return Problem(ShockerError.ShockerNotFound); - var logs = _db.ShockerControlLogs + var page = _db.ShockerControlLogs .Where(x => x.ShockerId == shockerId) .OrderByDescending(x => x.CreatedAt) .Skip((int)offset) - .Take((int)limit) - .Select(x => new LogEntry - { - Id = x.Id, - Duration = x.Duration, - Intensity = x.Intensity, - Type = x.Type, - CreatedOn = x.CreatedAt, - ControlledBy = x.ControlledByUser == null - ? new ControlLogSenderLight - { - Id = Guid.Empty, - Name = "Guest", - Image = GravatarUtils.GuestImageUrl, - CustomName = x.CustomName - } - : new ControlLogSenderLight - { - Id = x.ControlledByUser.Id, - Name = x.ControlledByUser.Name, - Image = x.ControlledByUser.GetImageUrl(), - CustomName = x.CustomName - } - }) + .Take((int)limit); + + var logs = (from x in page + join u in _db.Users on x.ControlledByUserId equals (Guid?)u.Id into controllers + from u in controllers.DefaultIfEmpty() + orderby x.CreatedAt descending + select new LogEntry + { + Id = x.Id, + Duration = x.Duration, + Intensity = x.Intensity, + Type = x.Type, + CreatedOn = x.CreatedAt, + ControlledBy = u != null + ? new ControlLogSenderLight + { + Id = u.Id, + Name = u.Name, + Image = u.GetImageUrl(), + CustomName = x.CustomName + } + : x.ControlledByUserId == null + ? new ControlLogSenderLight + { + Id = Guid.Empty, + Name = "Guest", + Image = GravatarUtils.GuestImageUrl, + CustomName = x.CustomName + } + : new ControlLogSenderLight + { + Id = x.ControlledByUserId.Value, + Name = "Deleted User", + Image = GravatarUtils.GuestImageUrl, + CustomName = x.CustomName + } + }) .AsAsyncEnumerable(); return LegacyDataOk(logs); @@ -101,43 +121,56 @@ public async Task> GetAllShockerLogs( [FromQuery(Name = "shockerIds")] Guid[]? shockerIds, CancellationToken cancellationToken) { - var query = _db.ShockerControlLogs - .Where(x => x.Shocker.Device.OwnerId == CurrentUser.Id); + var logs = _db.ShockerControlLogs.Where(x => x.OwnerUserId == CurrentUser.Id); if (shockerIds is { Length: > 0 }) - query = query.Where(x => ((IEnumerable)shockerIds).Contains(x.ShockerId)); + logs = logs.Where(x => ((IEnumerable)shockerIds).Contains(x.ShockerId)); + + var query = from x in logs + join s in _db.Shockers on x.ShockerId equals s.Id + join u in _db.Users on x.ControlledByUserId equals (Guid?)u.Id into controllers + from u in controllers.DefaultIfEmpty() + select new LogRow { Log = x, Shocker = s, Controller = u }; var ordered = query .ApplySort(pagination, AllLogsSorters, AllLogsDefaultSort) - .ThenBy(x => x.Id); + .ThenBy(x => x.Log.Id); return await ordered.ToPagedResultAsync( x => new LogEntryWithHub { - Id = x.Id, + Id = x.Log.Id, HubId = x.Shocker.Device.Id, HubName = x.Shocker.Device.Name, ShockerId = x.Shocker.Id, ShockerName = x.Shocker.Name, - Duration = x.Duration, - Intensity = x.Intensity, - Type = x.Type, - CreatedOn = x.CreatedAt, - ControlledBy = x.ControlledByUser == null + Duration = x.Log.Duration, + Intensity = x.Log.Intensity, + Type = x.Log.Type, + CreatedOn = x.Log.CreatedAt, + ControlledBy = x.Controller != null ? new ControlLogSenderLight { - Id = Guid.Empty, - Name = "Guest", - Image = GravatarUtils.GuestImageUrl, - CustomName = x.CustomName - } - : new ControlLogSenderLight - { - Id = x.ControlledByUser.Id, - Name = x.ControlledByUser.Name, - Image = x.ControlledByUser.GetImageUrl(), - CustomName = x.CustomName + Id = x.Controller.Id, + Name = x.Controller.Name, + Image = x.Controller.GetImageUrl(), + CustomName = x.Log.CustomName } + : x.Log.ControlledByUserId == null + ? new ControlLogSenderLight + { + Id = Guid.Empty, + Name = "Guest", + Image = GravatarUtils.GuestImageUrl, + CustomName = x.Log.CustomName + } + : new ControlLogSenderLight + { + Id = x.Log.ControlledByUserId.Value, + Name = "Deleted User", + Image = GravatarUtils.GuestImageUrl, + CustomName = x.Log.CustomName + } }, pagination, cancellationToken); diff --git a/API/Controller/Users/GetSelfControlStats.cs b/API/Controller/Users/GetSelfControlStats.cs index e664fd4d0..fb28037aa 100644 --- a/API/Controller/Users/GetSelfControlStats.cs +++ b/API/Controller/Users/GetSelfControlStats.cs @@ -100,10 +100,17 @@ SELECT now() AT TIME ZONE {tz} IS NOT NULL AS "Valid", var shockerIds = received.Select(x => x.Key.ShockerId).OfType().Distinct().ToArray(); var shockerNames = await _db.Shockers - .Where(x => shockerIds.Contains(x.Id)) + .Where(x => x.Device.OwnerId == CurrentUser.Id && shockerIds.Contains(x.Id)) .Select(x => new { x.Id, x.Name }) .ToDictionaryAsync(x => x.Id, x => x.Name, cancellationToken); + // Deleted shockers are combined into one entry + received = received + .Select(x => x.Key.ShockerId is { } id && !shockerNames.ContainsKey(id) + ? x with { Key = x.Key with { ShockerId = null } } + : x) + .ToArray(); + var userIds = received.Select(x => x.Key.ControllerUserId).OfType() .Concat(given.Select(x => x.Key.OwnerUserId)) .Distinct() @@ -119,6 +126,9 @@ SELECT now() AT TIME ZONE {tz} IS NOT NULL AS "Valid", Image = GravatarUtils.GetUserImageUrl(x.Email) }); + // Materialized stats outlive the logs of deleted owners + given = given.Where(x => users.ContainsKey(x.Key.OwnerUserId)).ToArray(); + return Ok(new ControlStatsResponse { Received = new ReceivedControlStats @@ -160,7 +170,7 @@ SELECT now() AT TIME ZONE {tz} IS NOT NULL AS "Valid", .Select(g => new UserControlStats { UserId = g.Key, - User = users.GetValueOrDefault(g.Key), + User = users[g.Key], Stats = Totals(g) }) .ToArray() diff --git a/Common.Tests/Services/ControlStatsAccumulatorTests.cs b/Common.Tests/Services/ControlStatsAccumulatorTests.cs deleted file mode 100644 index b7bcbcd3d..000000000 --- a/Common.Tests/Services/ControlStatsAccumulatorTests.cs +++ /dev/null @@ -1,52 +0,0 @@ -using OpenShock.Common.OpenShockDb; -using OpenShock.Common.Services.ControlStats; - -namespace OpenShock.Common.Tests.Services; - -public sealed class ControlStatsAccumulatorTests -{ - private static readonly Guid Owner = Guid.CreateVersion7(); - private static readonly Guid Shocker = Guid.CreateVersion7(); - private static readonly Guid Controller = Guid.CreateVersion7(); - - private static ControlStatDelta Delta(DateTime at, ControlType type = ControlType.Shock, long count = 1, byte intensity = 50) => - new(Owner, Shocker, Controller, at, type, Live: true, count, DurationMs: 100, IntensityMs: intensity * 100L, intensity); - - [Test] - public async Task Add_MergesWithinTheSameBucket() - { - var accumulator = new ControlStatsAccumulator(); - accumulator.Add(Delta(new DateTime(2026, 1, 1, 10, 31, 0, DateTimeKind.Utc), count: 1, intensity: 20)); - accumulator.Add(Delta(new DateTime(2026, 1, 1, 10, 44, 59, DateTimeKind.Utc), count: 0, intensity: 80)); - - var merged = (await Assert.That(accumulator.DequeueAll()).HasSingleItem())!; - - await Assert.That(merged.Bucket).IsEqualTo(new DateTime(2026, 1, 1, 10, 30, 0, DateTimeKind.Utc)); - await Assert.That(merged.Count).IsEqualTo(1); - await Assert.That(merged.DurationMs).IsEqualTo(200); - await Assert.That(merged.IntensityMs).IsEqualTo(10_000); - await Assert.That(merged.MaxIntensity).IsEqualTo((byte)80); - } - - [Test] - public async Task Add_SplitsAcrossBucketsAndTypes_AndIgnoresStop() - { - var accumulator = new ControlStatsAccumulator(); - accumulator.Add(Delta(new DateTime(2026, 1, 1, 10, 44, 59, DateTimeKind.Utc))); - accumulator.Add(Delta(new DateTime(2026, 1, 1, 10, 45, 0, DateTimeKind.Utc))); - accumulator.Add(Delta(new DateTime(2026, 1, 1, 10, 45, 0, DateTimeKind.Utc), ControlType.Vibrate)); - accumulator.Add(Delta(new DateTime(2026, 1, 1, 10, 45, 0, DateTimeKind.Utc), ControlType.Stop)); - - await Assert.That(accumulator.DequeueAll().Length).IsEqualTo(3); - } - - [Test] - public async Task DequeueAll_Drains() - { - var accumulator = new ControlStatsAccumulator(); - accumulator.Add(Delta(DateTime.UtcNow)); - - await Assert.That(accumulator.DequeueAll().Length).IsEqualTo(1); - await Assert.That(accumulator.DequeueAll()).IsEmpty(); - } -} diff --git a/Common/Migrations/20261120201327_AddControlLogHypertable.Designer.cs b/Common/Migrations/20261120201327_AddControlLogHypertable.Designer.cs new file mode 100644 index 000000000..cdfbcea18 --- /dev/null +++ b/Common/Migrations/20261120201327_AddControlLogHypertable.Designer.cs @@ -0,0 +1,1832 @@ +// +using System; +using System.Collections.Generic; +using System.Net; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; +using OpenShock.Common.OpenShockDb; + +#nullable disable + +namespace OpenShock.Common.Migrations; + +[DbContext(typeof(MigrationOpenShockContext))] +[Migration("20261120201327_AddControlLogHypertable")] +partial class _20261120201327_AddControlLogHypertable +{ + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("Npgsql:CollationDefinition:public.ndcoll", "und-u-ks-level2,und-u-ks-level2,icu,False") + .HasAnnotation("ProductVersion", "11.0.0") + .HasAnnotation("Relational:MaxIdentifierLength", 63); + + NpgsqlModelBuilderExtensions.HasPostgresEnum(modelBuilder, "audit_action", new[] { "login", "logout", "password_changed", "email_change_requested", "email_changed", "username_changed", "api_token_created", "api_token_deleted", "oauth_connected", "oauth_disconnected", "account_deactivated", "account_reactivated", "account_deleted", "automation_token_created", "automation_token_updated", "automation_token_rotated", "automation_token_deleted", "automation_token_used" }); + NpgsqlModelBuilderExtensions.HasPostgresEnum(modelBuilder, "automation_token_type", new[] { "turnstile", "rate_limit", "control_stats_read" }); + NpgsqlModelBuilderExtensions.HasPostgresEnum(modelBuilder, "configuration_value_type", new[] { "string", "bool", "int", "float", "json" }); + NpgsqlModelBuilderExtensions.HasPostgresEnum(modelBuilder, "control_limit_mode", new[] { "clamp", "lerp" }); + NpgsqlModelBuilderExtensions.HasPostgresEnum(modelBuilder, "control_type", new[] { "stop", "shock", "vibrate", "sound" }); + NpgsqlModelBuilderExtensions.HasPostgresEnum(modelBuilder, "email_status", new[] { "pending", "sending", "sent", "failed", "skipped" }); + NpgsqlModelBuilderExtensions.HasPostgresEnum(modelBuilder, "email_type", new[] { "account_activation", "password_reset", "email_verification", "email_change_notice" }); + NpgsqlModelBuilderExtensions.HasPostgresEnum(modelBuilder, "match_type_enum", new[] { "exact", "contains" }); + NpgsqlModelBuilderExtensions.HasPostgresEnum(modelBuilder, "ota_update_status", new[] { "started", "running", "finished", "error", "timeout" }); + NpgsqlModelBuilderExtensions.HasPostgresEnum(modelBuilder, "password_encryption_type", new[] { "bcrypt_enhanced", "pbkdf2" }); + NpgsqlModelBuilderExtensions.HasPostgresEnum(modelBuilder, "permission_type", new[] { "shockers.use", "shockers.edit", "shockers.pause", "devices.edit", "devices.auth", "usershares.edit", "usershares.pause", "publicshares.pause", "publicshares.edit" }); + NpgsqlModelBuilderExtensions.HasPostgresEnum(modelBuilder, "role_type", new[] { "support", "staff", "admin", "system" }); + NpgsqlModelBuilderExtensions.HasPostgresEnum(modelBuilder, "shocker_model_type", new[] { "cai_xianlin", "petrainer", "petrainer_998dr", "wellturn_t330" }); + NpgsqlModelBuilderExtensions.HasPostgresExtension(modelBuilder, "timescaledb"); + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + + modelBuilder.Entity("Microsoft.AspNetCore.DataProtection.EntityFrameworkCore.DataProtectionKey", b => + { + var id = b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(id); + + b.Property("FriendlyName") + .HasColumnType("text"); + + b.Property("Xml") + .HasColumnType("text"); + + b.HasKey("Id"); + + b.ToTable("DataProtectionKeys"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.AdminUsersView", b => + { + b.Property("ActivatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("activated_at"); + + b.Property("ApiTokenCount") + .HasColumnType("integer") + .HasColumnName("api_token_count"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at"); + + b.Property("CreatedByAutomationTokenId") + .HasColumnType("uuid") + .HasColumnName("created_by_automation_token_id"); + + b.Property("DeactivatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("deactivated_at"); + + b.Property("DeactivatedByUserId") + .HasColumnType("uuid") + .HasColumnName("deactivated_by_user_id"); + + b.Property("DeviceCount") + .HasColumnType("integer") + .HasColumnName("device_count"); + + b.Property("Email") + .IsRequired() + .HasColumnType("character varying") + .HasColumnName("email"); + + b.Property("EmailChangeRequestCount") + .HasColumnType("integer") + .HasColumnName("email_change_request_count"); + + b.Property("Id") + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("Name") + .IsRequired() + .HasColumnType("character varying") + .HasColumnName("name"); + + b.Property("NameChangeRequestCount") + .HasColumnType("integer") + .HasColumnName("name_change_request_count"); + + b.Property("PasswordHashType") + .HasColumnType("character varying") + .HasColumnName("password_hash_type"); + + b.Property("PasswordResetCount") + .HasColumnType("integer") + .HasColumnName("password_reset_count"); + + b.Property("Roles") + .IsRequired() + .HasColumnType("role_type[]") + .HasColumnName("roles"); + + b.Property("ShockerControlLogCount") + .HasColumnType("integer") + .HasColumnName("shocker_control_log_count"); + + b.Property("ShockerCount") + .HasColumnType("integer") + .HasColumnName("shocker_count"); + + b.Property("ShockerPublicShareCount") + .HasColumnType("integer") + .HasColumnName("shocker_public_share_count"); + + b.Property("ShockerUserShareCount") + .HasColumnType("integer") + .HasColumnName("shocker_user_share_count"); + + b.ToTable((string)null); + + b.ToView("admin_users_view", (string)null); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.ApiToken", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("CreatedAt") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at") + .HasDefaultValueSql("CURRENT_TIMESTAMP"); + + b.Property("CreatedByIp") + .IsRequired() + .HasColumnType("inet") + .HasColumnName("created_by_ip"); + + b.Property("LastUsed") + .HasColumnType("timestamp with time zone") + .HasColumnName("last_used"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("character varying(64)") + .HasColumnName("name"); + + b.Property>("Permissions") + .IsRequired() + .HasColumnType("permission_type[]") + .HasColumnName("permissions"); + + b.Property("ShockerControlDurationMax") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasDefaultValue(65535) + .HasColumnName("shocker_control_duration_max"); + + b.Property("ShockerControlDurationMin") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasDefaultValue(300) + .HasColumnName("shocker_control_duration_min"); + + b.Property("ShockerControlDurationMode") + .ValueGeneratedOnAdd() + .HasColumnType("control_limit_mode") + .HasDefaultValue(ControlLimitMode.Clamp) + .HasColumnName("shocker_control_duration_mode"); + + b.Property("ShockerControlIntensityMax") + .ValueGeneratedOnAdd() + .HasColumnType("smallint") + .HasDefaultValue((byte)100) + .HasColumnName("shocker_control_intensity_max"); + + b.Property("ShockerControlIntensityMin") + .ValueGeneratedOnAdd() + .HasColumnType("smallint") + .HasDefaultValue((byte)0) + .HasColumnName("shocker_control_intensity_min"); + + b.Property("ShockerControlIntensityMode") + .ValueGeneratedOnAdd() + .HasColumnType("control_limit_mode") + .HasDefaultValue(ControlLimitMode.Clamp) + .HasColumnName("shocker_control_intensity_mode"); + + b.Property("ShockerControlPaused") + .ValueGeneratedOnAdd() + .HasColumnType("boolean") + .HasDefaultValue(false) + .HasColumnName("shocker_control_paused"); + + b.Property("TokenHash") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("character varying(64)") + .HasColumnName("token_hash") + .UseCollation("C"); + + b.Property("UserId") + .HasColumnType("uuid") + .HasColumnName("user_id"); + + b.Property("ValidUntil") + .HasColumnType("timestamp with time zone") + .HasColumnName("valid_until"); + + b.HasKey("Id") + .HasName("api_tokens_pkey"); + + b.HasIndex("TokenHash") + .IsUnique(); + + b.HasIndex("UserId"); + + b.HasIndex("ValidUntil"); + + b.ToTable("api_tokens"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.ApiTokenReport", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("AffectedCount") + .HasColumnType("integer") + .HasColumnName("affected_count"); + + b.Property("CreatedAt") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at") + .HasDefaultValueSql("CURRENT_TIMESTAMP"); + + b.Property("IpAddress") + .IsRequired() + .HasColumnType("inet") + .HasColumnName("ip_address"); + + b.Property("IpCountry") + .HasMaxLength(4) + .HasColumnType("character varying(4)") + .HasColumnName("ip_country"); + + b.Property("SubmittedCount") + .HasColumnType("integer") + .HasColumnName("submitted_count"); + + b.Property("UserId") + .HasColumnType("uuid") + .HasColumnName("user_id"); + + b.HasKey("Id") + .HasName("api_token_reports_pkey"); + + b.HasIndex("UserId"); + + b.ToTable("api_token_reports"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.AutomationToken", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("AutoCleanupAfter") + .HasColumnType("interval") + .HasColumnName("auto_cleanup_after"); + + b.Property("AutoCleanupUsers") + .ValueGeneratedOnAdd() + .HasColumnType("boolean") + .HasDefaultValue(false) + .HasColumnName("auto_cleanup_users"); + + b.Property("CreatedAt") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at") + .HasDefaultValueSql("CURRENT_TIMESTAMP"); + + b.Property("LastRotatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("last_rotated_at"); + + b.Property("LastUsedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("last_used_at"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("character varying(64)") + .HasColumnName("name"); + + b.Property("TokenHash") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("character varying(64)") + .HasColumnName("token_hash") + .UseCollation("C"); + + b.Property>("Types") + .IsRequired() + .HasColumnType("automation_token_type[]") + .HasColumnName("types"); + + b.Property("UseCount") + .ValueGeneratedOnAdd() + .HasColumnType("bigint") + .HasDefaultValue(0L) + .HasColumnName("use_count"); + + b.HasKey("Id") + .HasName("automation_tokens_pkey"); + + b.HasIndex("TokenHash") + .IsUnique(); + + b.ToTable("automation_tokens"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.ConfigurationItem", b => + { + b.Property("Name") + .HasMaxLength(64) + .HasColumnType("character varying(64)") + .HasColumnName("name") + .UseCollation("C"); + + b.Property("CreatedAt") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at") + .HasDefaultValueSql("CURRENT_TIMESTAMP"); + + b.Property("Description") + .IsRequired() + .HasMaxLength(255) + .HasColumnType("character varying(255)") + .HasColumnName("description"); + + b.Property("Type") + .HasColumnType("configuration_value_type") + .HasColumnName("type"); + + b.Property("UpdatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("updated_at"); + + b.Property("Value") + .IsRequired() + .HasMaxLength(4096) + .HasColumnType("character varying(4096)") + .HasColumnName("value"); + + b.HasKey("Name") + .HasName("configuration_pkey"); + + b.ToTable("configuration"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.ControlStat", b => + { + b.Property("Bucket") + .HasColumnType("timestamp with time zone") + .HasColumnName("bucket"); + + b.Property("ControllerUserId") + .HasColumnType("uuid") + .HasColumnName("controller_user_id"); + + b.Property("Count") + .HasColumnType("bigint") + .HasColumnName("count"); + + b.Property("DurationMs") + .HasColumnType("bigint") + .HasColumnName("duration_ms"); + + b.Property("IntensityMs") + .HasColumnType("bigint") + .HasColumnName("intensity_ms"); + + b.Property("Live") + .HasColumnType("boolean") + .HasColumnName("live"); + + b.Property("MaxIntensity") + .HasColumnType("smallint") + .HasColumnName("max_intensity"); + + b.Property("OwnerUserId") + .HasColumnType("uuid") + .HasColumnName("owner_user_id"); + + b.Property("ShockerId") + .HasColumnType("uuid") + .HasColumnName("shocker_id"); + + b.Property("Type") + .HasColumnType("control_type") + .HasColumnName("type"); + + b.ToTable((string)null); + + b.ToView("control_stats", (string)null); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.Device", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("CreatedAt") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at") + .HasDefaultValueSql("CURRENT_TIMESTAMP"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("character varying(64)") + .HasColumnName("name"); + + b.Property("OwnerId") + .HasColumnType("uuid") + .HasColumnName("owner_id"); + + b.Property("Token") + .IsRequired() + .HasMaxLength(256) + .HasColumnType("character varying(256)") + .HasColumnName("token") + .UseCollation("C"); + + b.HasKey("Id") + .HasName("devices_pkey"); + + b.HasIndex("OwnerId"); + + b.HasIndex("Token") + .IsUnique(); + + b.ToTable("devices"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.DeviceOtaUpdate", b => + { + b.Property("DeviceId") + .HasColumnType("uuid") + .HasColumnName("device_id"); + + b.Property("UpdateId") + .HasColumnType("integer") + .HasColumnName("update_id"); + + b.Property("CreatedAt") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at") + .HasDefaultValueSql("CURRENT_TIMESTAMP"); + + b.Property("Message") + .HasMaxLength(128) + .HasColumnType("character varying(128)") + .HasColumnName("message"); + + b.Property("Status") + .HasColumnType("ota_update_status") + .HasColumnName("status"); + + b.Property("Version") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("character varying(64)") + .HasColumnName("version"); + + b.HasKey("DeviceId", "UpdateId") + .HasName("device_ota_updates_pkey"); + + b.HasIndex(new[] { "CreatedAt" }, "device_ota_updates_created_at_idx"); + + b.ToTable("device_ota_updates"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.DiscordWebhook", b => + { + b.Property("Name") + .HasMaxLength(64) + .HasColumnType("character varying(64)") + .HasColumnName("name"); + + b.Property("CreatedAt") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at") + .HasDefaultValueSql("CURRENT_TIMESTAMP"); + + b.Property("Id") + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("WebhookId") + .HasColumnType("bigint") + .HasColumnName("webhook_id"); + + b.Property("WebhookToken") + .IsRequired() + .HasMaxLength(255) + .HasColumnType("character varying(255)") + .HasColumnName("webhook_token"); + + b.HasKey("Name") + .HasName("discord_webhooks_pkey"); + + b.ToTable("discord_webhooks"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.EmailOutboxMessage", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("AttemptCount") + .IsConcurrencyToken() + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasDefaultValue(0) + .HasColumnName("attempt_count"); + + b.Property("CoalesceKey") + .HasMaxLength(128) + .HasColumnType("character varying(128)") + .HasColumnName("coalesce_key"); + + b.Property("CreatedAt") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at") + .HasDefaultValueSql("CURRENT_TIMESTAMP"); + + b.Property("FailedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("failed_at"); + + b.Property("LastError") + .HasMaxLength(1024) + .HasColumnType("character varying(1024)") + .HasColumnName("last_error"); + + b.Property("NextAttemptAt") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasColumnName("next_attempt_at") + .HasDefaultValueSql("CURRENT_TIMESTAMP"); + + b.Property>("Payload") + .IsRequired() + .HasColumnType("jsonb") + .HasColumnName("payload"); + + b.Property("Recipient") + .IsRequired() + .HasMaxLength(320) + .HasColumnType("character varying(320)") + .HasColumnName("recipient"); + + b.Property("RecipientName") + .HasMaxLength(32) + .HasColumnType("character varying(32)") + .HasColumnName("recipient_name"); + + b.Property("SentAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("sent_at"); + + b.Property("Status") + .HasColumnType("email_status") + .HasColumnName("status"); + + b.Property("Type") + .HasColumnType("email_type") + .HasColumnName("type"); + + b.HasKey("Id") + .HasName("email_outbox_pkey"); + + b.HasIndex("CoalesceKey"); + + b.HasIndex("Recipient"); + + b.HasIndex("Status", "NextAttemptAt"); + + b.ToTable("email_outbox"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.EmailProviderBlacklist", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("CreatedAt") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at") + .HasDefaultValueSql("CURRENT_TIMESTAMP"); + + b.Property("Domain") + .IsRequired() + .HasMaxLength(255) + .HasColumnType("character varying(255)") + .HasColumnName("domain") + .UseCollation("ndcoll"); + + b.HasKey("Id") + .HasName("email_provider_blacklist_pkey"); + + var index = b.HasIndex("Domain") + .IsUnique(); + + NpgsqlIndexBuilderExtensions.UseCollation(index, new[] { "ndcoll" }); + + b.ToTable("email_provider_blacklist"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.PublicShare", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("CreatedAt") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at") + .HasDefaultValueSql("CURRENT_TIMESTAMP"); + + b.Property("ExpiresAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("expires_at"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("character varying(64)") + .HasColumnName("name"); + + b.Property("OwnerId") + .HasColumnType("uuid") + .HasColumnName("owner_id"); + + b.HasKey("Id") + .HasName("public_shares_pkey"); + + b.HasIndex("OwnerId"); + + b.ToTable("public_shares"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.PublicShareShocker", b => + { + b.Property("PublicShareId") + .HasColumnType("uuid") + .HasColumnName("public_share_id"); + + b.Property("ShockerId") + .HasColumnType("uuid") + .HasColumnName("shocker_id"); + + b.Property("AllowLiveControl") + .ValueGeneratedOnAdd() + .HasColumnType("boolean") + .HasDefaultValue(false) + .HasColumnName("allow_livecontrol"); + + b.Property("AllowShock") + .ValueGeneratedOnAdd() + .HasColumnType("boolean") + .HasDefaultValue(true) + .HasColumnName("allow_shock"); + + b.Property("AllowSound") + .ValueGeneratedOnAdd() + .HasColumnType("boolean") + .HasDefaultValue(true) + .HasColumnName("allow_sound"); + + b.Property("AllowVibrate") + .ValueGeneratedOnAdd() + .HasColumnType("boolean") + .HasDefaultValue(true) + .HasColumnName("allow_vibrate"); + + b.Property("Cooldown") + .HasColumnType("integer") + .HasColumnName("cooldown"); + + b.Property("IsPaused") + .ValueGeneratedOnAdd() + .HasColumnType("boolean") + .HasDefaultValue(false) + .HasColumnName("is_paused"); + + b.Property("MaxDuration") + .HasColumnType("integer") + .HasColumnName("max_duration"); + + b.Property("MaxIntensity") + .HasColumnType("smallint") + .HasColumnName("max_intensity"); + + b.HasKey("PublicShareId", "ShockerId") + .HasName("public_share_shockers_pkey"); + + b.HasIndex("ShockerId"); + + b.ToTable("public_share_shockers"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.Shocker", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("CreatedAt") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at") + .HasDefaultValueSql("CURRENT_TIMESTAMP"); + + b.Property("DeviceId") + .HasColumnType("uuid") + .HasColumnName("device_id"); + + b.Property("IsPaused") + .ValueGeneratedOnAdd() + .HasColumnType("boolean") + .HasDefaultValue(false) + .HasColumnName("is_paused"); + + b.Property("Model") + .HasColumnType("shocker_model_type") + .HasColumnName("model"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("character varying(64)") + .HasColumnName("name"); + + b.Property("RfId") + .HasColumnType("integer") + .HasColumnName("rf_id"); + + b.HasKey("Id") + .HasName("shockers_pkey"); + + b.HasIndex("DeviceId"); + + b.ToTable("shockers"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.ShockerControlLog", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("CreatedAt") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at") + .HasDefaultValueSql("CURRENT_TIMESTAMP"); + + b.Property("ControlledByUserId") + .HasColumnType("uuid") + .HasColumnName("controlled_by_user_id"); + + b.Property("CustomName") + .HasMaxLength(64) + .HasColumnType("character varying(64)") + .HasColumnName("custom_name"); + + b.Property("Duration") + .HasColumnType("bigint") + .HasColumnName("duration"); + + b.Property("InsertedAt") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasColumnName("inserted_at") + .HasDefaultValueSql("now()"); + + b.Property("Intensity") + .HasColumnType("smallint") + .HasColumnName("intensity"); + + b.Property("LiveControl") + .ValueGeneratedOnAdd() + .HasColumnType("boolean") + .HasDefaultValue(false) + .HasColumnName("live_control"); + + b.Property("MaxIntensity") + .HasColumnType("smallint") + .HasColumnName("max_intensity"); + + b.Property("OwnerUserId") + .HasColumnType("uuid") + .HasColumnName("owner_user_id"); + + b.Property("ShockerId") + .HasColumnType("uuid") + .HasColumnName("shocker_id"); + + b.Property("Type") + .HasColumnType("control_type") + .HasColumnName("type"); + + b.HasKey("Id", "CreatedAt") + .HasName("shocker_control_logs_pkey"); + + b.HasIndex("ControlledByUserId"); + + b.HasIndex("OwnerUserId", "CreatedAt"); + + b.HasIndex("ShockerId", "CreatedAt"); + + b.ToTable("shocker_control_logs"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.ShockerShareCode", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("AllowLiveControl") + .ValueGeneratedOnAdd() + .HasColumnType("boolean") + .HasDefaultValue(true) + .HasColumnName("allow_livecontrol"); + + b.Property("AllowShock") + .ValueGeneratedOnAdd() + .HasColumnType("boolean") + .HasDefaultValue(true) + .HasColumnName("allow_shock"); + + b.Property("AllowSound") + .ValueGeneratedOnAdd() + .HasColumnType("boolean") + .HasDefaultValue(true) + .HasColumnName("allow_sound"); + + b.Property("AllowVibrate") + .ValueGeneratedOnAdd() + .HasColumnType("boolean") + .HasDefaultValue(true) + .HasColumnName("allow_vibrate"); + + b.Property("CreatedAt") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at") + .HasDefaultValueSql("CURRENT_TIMESTAMP"); + + b.Property("IsPaused") + .ValueGeneratedOnAdd() + .HasColumnType("boolean") + .HasDefaultValue(false) + .HasColumnName("is_paused"); + + b.Property("MaxDuration") + .HasColumnType("integer") + .HasColumnName("max_duration"); + + b.Property("MaxIntensity") + .HasColumnType("smallint") + .HasColumnName("max_intensity"); + + b.Property("ShockerId") + .HasColumnType("uuid") + .HasColumnName("shocker_id"); + + b.HasKey("Id") + .HasName("shocker_share_codes_pkey"); + + b.HasIndex("ShockerId"); + + b.ToTable("shocker_share_codes"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.User", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("ActivatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("activated_at"); + + b.Property("CreatedAt") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at") + .HasDefaultValueSql("CURRENT_TIMESTAMP"); + + b.Property("CreatedByAutomationTokenId") + .HasColumnType("uuid") + .HasColumnName("created_by_automation_token_id"); + + b.Property("Email") + .IsRequired() + .HasMaxLength(320) + .HasColumnType("character varying(320)") + .HasColumnName("email"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(32) + .HasColumnType("character varying(32)") + .HasColumnName("name") + .UseCollation("ndcoll"); + + b.Property("PasswordHash") + .HasMaxLength(100) + .HasColumnType("character varying(100)") + .HasColumnName("password_hash") + .UseCollation("C"); + + b.Property>("Roles") + .IsRequired() + .HasColumnType("role_type[]") + .HasColumnName("roles"); + + b.Property("SecurityStamp") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("security_stamp") + .HasDefaultValueSql("gen_random_uuid()"); + + b.HasKey("Id") + .HasName("users_pkey"); + + b.HasIndex("CreatedByAutomationTokenId"); + + b.HasIndex("Email") + .IsUnique(); + + var index = b.HasIndex("Name") + .IsUnique(); + + NpgsqlIndexBuilderExtensions.UseCollation(index, new[] { "ndcoll" }); + + b.ToTable("users"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.UserActivationRequest", b => + { + b.Property("UserId") + .HasColumnType("uuid") + .HasColumnName("user_id"); + + b.Property("CreatedAt") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at") + .HasDefaultValueSql("CURRENT_TIMESTAMP"); + + b.Property("EmailSendAttempts") + .HasColumnType("integer") + .HasColumnName("email_send_attempts"); + + b.Property("TokenHash") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("character varying(128)") + .HasColumnName("token_hash") + .UseCollation("C"); + + b.HasKey("UserId") + .HasName("user_activation_requests_pkey"); + + b.HasIndex("TokenHash") + .IsUnique(); + + b.ToTable("user_activation_requests"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.UserAuditLog", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("Action") + .HasColumnType("audit_action") + .HasColumnName("action"); + + b.Property("ActorId") + .HasColumnType("uuid") + .HasColumnName("actor_id"); + + b.Property("CreatedAt") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at") + .HasDefaultValueSql("CURRENT_TIMESTAMP"); + + b.Property("IpAddress") + .HasMaxLength(40) + .HasColumnType("character varying(40)") + .HasColumnName("ip_address"); + + b.Property("Metadata") + .HasColumnType("jsonb") + .HasColumnName("metadata"); + + b.Property("Reason") + .HasMaxLength(512) + .HasColumnType("character varying(512)") + .HasColumnName("reason"); + + b.Property("UserAgent") + .HasMaxLength(1024) + .HasColumnType("character varying(1024)") + .HasColumnName("user_agent"); + + b.Property("UserId") + .HasColumnType("uuid") + .HasColumnName("user_id"); + + b.HasKey("Id") + .HasName("user_audit_logs_pkey"); + + b.HasIndex("ActorId"); + + b.HasIndex("CreatedAt"); + + b.HasIndex("UserId"); + + b.ToTable("user_audit_logs"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.UserDeactivation", b => + { + b.Property("DeactivatedUserId") + .HasColumnType("uuid") + .HasColumnName("deactivated_user_id"); + + b.Property("CreatedAt") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at") + .HasDefaultValueSql("CURRENT_TIMESTAMP"); + + b.Property("DeactivatedByUserId") + .HasColumnType("uuid") + .HasColumnName("deactivated_by_user_id"); + + b.Property("DeleteLater") + .HasColumnType("boolean") + .HasColumnName("delete_later"); + + b.Property("UserModerationId") + .HasColumnType("uuid") + .HasColumnName("user_moderation_id"); + + b.HasKey("DeactivatedUserId") + .HasName("user_deactivations_pkey"); + + b.HasIndex("DeactivatedByUserId"); + + b.ToTable("user_deactivations"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.UserEmailChange", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("CreatedAt") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at") + .HasDefaultValueSql("CURRENT_TIMESTAMP"); + + b.Property("NewEmail") + .IsRequired() + .HasMaxLength(320) + .HasColumnType("character varying(320)") + .HasColumnName("email_new"); + + b.Property("OldEmail") + .IsRequired() + .HasMaxLength(320) + .HasColumnType("character varying(320)") + .HasColumnName("email_old"); + + b.Property("SecurityStampAtCreate") + .HasColumnType("uuid") + .HasColumnName("security_stamp_at_create"); + + b.Property("TokenHash") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("character varying(128)") + .HasColumnName("token_hash") + .UseCollation("C"); + + b.Property("UsedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("used_at"); + + b.Property("UserId") + .HasColumnType("uuid") + .HasColumnName("user_id"); + + b.HasKey("Id") + .HasName("user_email_changes_pkey"); + + b.HasIndex("CreatedAt"); + + b.HasIndex("UsedAt"); + + b.HasIndex("UserId"); + + b.ToTable("user_email_changes"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.UserNameBlacklist", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("CreatedAt") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at") + .HasDefaultValueSql("CURRENT_TIMESTAMP"); + + b.Property("MatchType") + .HasColumnType("match_type_enum") + .HasColumnName("match_type"); + + b.Property("Value") + .IsRequired() + .HasMaxLength(32) + .HasColumnType("character varying(32)") + .HasColumnName("value") + .UseCollation("ndcoll"); + + b.HasKey("Id") + .HasName("user_name_blacklist_pkey"); + + var index = b.HasIndex("Value") + .IsUnique(); + + NpgsqlIndexBuilderExtensions.UseCollation(index, new[] { "ndcoll" }); + + b.ToTable("user_name_blacklist"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.UserNameChange", b => + { + var id = b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityAlwaysColumn(id); + + b.Property("UserId") + .HasColumnType("uuid") + .HasColumnName("user_id"); + + b.Property("CreatedAt") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at") + .HasDefaultValueSql("CURRENT_TIMESTAMP"); + + b.Property("OldName") + .IsRequired() + .HasMaxLength(32) + .HasColumnType("character varying(32)") + .HasColumnName("old_name"); + + b.HasKey("Id", "UserId") + .HasName("user_name_changes_pkey"); + + b.HasIndex("CreatedAt"); + + b.HasIndex("OldName"); + + b.HasIndex("UserId"); + + b.ToTable("user_name_changes"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.UserOAuthConnection", b => + { + b.Property("ProviderKey") + .HasMaxLength(16) + .HasColumnType("character varying(16)") + .HasColumnName("provider_key") + .UseCollation("C"); + + b.Property("ExternalId") + .HasMaxLength(255) + .HasColumnType("character varying(255)") + .HasColumnName("external_id"); + + b.Property("CreatedAt") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at") + .HasDefaultValueSql("CURRENT_TIMESTAMP"); + + b.Property("DisplayName") + .HasMaxLength(64) + .HasColumnType("character varying(64)") + .HasColumnName("display_name"); + + b.Property("UserId") + .HasColumnType("uuid") + .HasColumnName("user_id"); + + b.HasKey("ProviderKey", "ExternalId") + .HasName("user_oauth_connections_pkey"); + + b.HasIndex("UserId"); + + b.ToTable("user_oauth_connections"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.UserPasswordReset", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("CreatedAt") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at") + .HasDefaultValueSql("CURRENT_TIMESTAMP"); + + b.Property("SecurityStampAtCreate") + .HasColumnType("uuid") + .HasColumnName("security_stamp_at_create"); + + b.Property("TokenHash") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("character varying(100)") + .HasColumnName("token_hash") + .UseCollation("C"); + + b.Property("UsedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("used_at"); + + b.Property("UserId") + .HasColumnType("uuid") + .HasColumnName("user_id"); + + b.HasKey("Id") + .HasName("user_password_resets_pkey"); + + b.HasIndex("UserId"); + + b.ToTable("user_password_resets"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.UserShare", b => + { + b.Property("SharedWithUserId") + .HasColumnType("uuid") + .HasColumnName("shared_with_user_id"); + + b.Property("ShockerId") + .HasColumnType("uuid") + .HasColumnName("shocker_id"); + + b.Property("AllowLiveControl") + .ValueGeneratedOnAdd() + .HasColumnType("boolean") + .HasDefaultValue(true) + .HasColumnName("allow_livecontrol"); + + b.Property("AllowShock") + .ValueGeneratedOnAdd() + .HasColumnType("boolean") + .HasDefaultValue(true) + .HasColumnName("allow_shock"); + + b.Property("AllowSound") + .ValueGeneratedOnAdd() + .HasColumnType("boolean") + .HasDefaultValue(true) + .HasColumnName("allow_sound"); + + b.Property("AllowVibrate") + .ValueGeneratedOnAdd() + .HasColumnType("boolean") + .HasDefaultValue(true) + .HasColumnName("allow_vibrate"); + + b.Property("CreatedAt") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at") + .HasDefaultValueSql("CURRENT_TIMESTAMP"); + + b.Property("IsPaused") + .ValueGeneratedOnAdd() + .HasColumnType("boolean") + .HasDefaultValue(false) + .HasColumnName("is_paused"); + + b.Property("MaxDuration") + .HasColumnType("integer") + .HasColumnName("max_duration"); + + b.Property("MaxIntensity") + .HasColumnType("smallint") + .HasColumnName("max_intensity"); + + b.HasKey("SharedWithUserId", "ShockerId") + .HasName("user_shares_pkey"); + + b.HasIndex("SharedWithUserId"); + + b.HasIndex("ShockerId"); + + b.ToTable("user_shares"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.UserShareInvite", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("CreatedAt") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at") + .HasDefaultValueSql("CURRENT_TIMESTAMP"); + + b.Property("OwnerId") + .HasColumnType("uuid") + .HasColumnName("owner_id"); + + b.Property("RecipientUserId") + .HasColumnType("uuid") + .HasColumnName("user_id"); + + b.HasKey("Id") + .HasName("user_share_invites_pkey"); + + b.HasIndex("OwnerId"); + + b.HasIndex("RecipientUserId"); + + b.ToTable("user_share_invites"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.UserShareInviteShocker", b => + { + b.Property("InviteId") + .HasColumnType("uuid") + .HasColumnName("invite_id"); + + b.Property("ShockerId") + .HasColumnType("uuid") + .HasColumnName("shocker_id"); + + b.Property("AllowLiveControl") + .ValueGeneratedOnAdd() + .HasColumnType("boolean") + .HasDefaultValue(true) + .HasColumnName("allow_livecontrol"); + + b.Property("AllowShock") + .ValueGeneratedOnAdd() + .HasColumnType("boolean") + .HasDefaultValue(true) + .HasColumnName("allow_shock"); + + b.Property("AllowSound") + .ValueGeneratedOnAdd() + .HasColumnType("boolean") + .HasDefaultValue(true) + .HasColumnName("allow_sound"); + + b.Property("AllowVibrate") + .ValueGeneratedOnAdd() + .HasColumnType("boolean") + .HasDefaultValue(true) + .HasColumnName("allow_vibrate"); + + b.Property("IsPaused") + .ValueGeneratedOnAdd() + .HasColumnType("boolean") + .HasDefaultValue(false) + .HasColumnName("is_paused"); + + b.Property("MaxDuration") + .HasColumnType("integer") + .HasColumnName("max_duration"); + + b.Property("MaxIntensity") + .HasColumnType("smallint") + .HasColumnName("max_intensity"); + + b.HasKey("InviteId", "ShockerId") + .HasName("user_share_invite_shockers_pkey"); + + b.HasIndex("ShockerId"); + + b.ToTable("user_share_invite_shockers"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.ApiToken", b => + { + b.HasOne("OpenShock.Common.OpenShockDb.User", "User") + .WithMany("ApiTokens") + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_api_tokens_user_id"); + + b.Navigation("User"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.ApiTokenReport", b => + { + b.HasOne("OpenShock.Common.OpenShockDb.User", "ReportedByUser") + .WithMany("ReportedApiTokens") + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_api_token_reports_reported_by_user_id"); + + b.Navigation("ReportedByUser"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.Device", b => + { + b.HasOne("OpenShock.Common.OpenShockDb.User", "Owner") + .WithMany("Devices") + .HasForeignKey("OwnerId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_devices_owner_id"); + + b.Navigation("Owner"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.DeviceOtaUpdate", b => + { + b.HasOne("OpenShock.Common.OpenShockDb.Device", "Device") + .WithMany("OtaUpdates") + .HasForeignKey("DeviceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_device_ota_updates_device_id"); + + b.Navigation("Device"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.PublicShare", b => + { + b.HasOne("OpenShock.Common.OpenShockDb.User", "Owner") + .WithMany("OwnedPublicShares") + .HasForeignKey("OwnerId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_public_shares_owner_id"); + + b.Navigation("Owner"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.PublicShareShocker", b => + { + b.HasOne("OpenShock.Common.OpenShockDb.PublicShare", "PublicShare") + .WithMany("ShockerMappings") + .HasForeignKey("PublicShareId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_public_share_shockers_public_share_id"); + + b.HasOne("OpenShock.Common.OpenShockDb.Shocker", "Shocker") + .WithMany("PublicShareMappings") + .HasForeignKey("ShockerId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_public_share_shockers_shocker_id"); + + b.Navigation("PublicShare"); + + b.Navigation("Shocker"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.Shocker", b => + { + b.HasOne("OpenShock.Common.OpenShockDb.Device", "Device") + .WithMany("Shockers") + .HasForeignKey("DeviceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_shockers_device_id"); + + b.Navigation("Device"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.ShockerShareCode", b => + { + b.HasOne("OpenShock.Common.OpenShockDb.Shocker", "Shocker") + .WithMany("ShockerShareCodes") + .HasForeignKey("ShockerId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_shocker_share_codes_shocker_id"); + + b.Navigation("Shocker"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.User", b => + { + b.HasOne("OpenShock.Common.OpenShockDb.AutomationToken", "CreatedByAutomationToken") + .WithMany("CreatedUsers") + .HasForeignKey("CreatedByAutomationTokenId") + .OnDelete(DeleteBehavior.Restrict) + .HasConstraintName("fk_users_created_by_automation_token_id"); + + b.Navigation("CreatedByAutomationToken"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.UserActivationRequest", b => + { + b.HasOne("OpenShock.Common.OpenShockDb.User", "User") + .WithOne("UserActivationRequest") + .HasForeignKey("OpenShock.Common.OpenShockDb.UserActivationRequest", "UserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_user_activation_requests_user_id"); + + b.Navigation("User"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.UserAuditLog", b => + { + b.HasOne("OpenShock.Common.OpenShockDb.User", "Actor") + .WithMany("ActorAuditLogs") + .HasForeignKey("ActorId") + .OnDelete(DeleteBehavior.SetNull) + .HasConstraintName("fk_user_audit_logs_actor_id"); + + b.HasOne("OpenShock.Common.OpenShockDb.User", "User") + .WithMany("AuditLogs") + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_user_audit_logs_user_id"); + + b.Navigation("Actor"); + + b.Navigation("User"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.UserDeactivation", b => + { + b.HasOne("OpenShock.Common.OpenShockDb.User", "DeactivatedByUser") + .WithMany() + .HasForeignKey("DeactivatedByUserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_user_deactivations_deactivated_by_user_id"); + + b.HasOne("OpenShock.Common.OpenShockDb.User", "DeactivatedUser") + .WithOne("UserDeactivation") + .HasForeignKey("OpenShock.Common.OpenShockDb.UserDeactivation", "DeactivatedUserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_user_deactivations_deactivated_user_id"); + + b.Navigation("DeactivatedByUser"); + + b.Navigation("DeactivatedUser"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.UserEmailChange", b => + { + b.HasOne("OpenShock.Common.OpenShockDb.User", "User") + .WithMany("EmailChanges") + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_user_email_changes_user_id"); + + b.Navigation("User"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.UserNameChange", b => + { + b.HasOne("OpenShock.Common.OpenShockDb.User", "User") + .WithMany("NameChanges") + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_user_name_changes_user_id"); + + b.Navigation("User"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.UserOAuthConnection", b => + { + b.HasOne("OpenShock.Common.OpenShockDb.User", "User") + .WithMany("OAuthConnections") + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_user_oauth_connections_user_id"); + + b.Navigation("User"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.UserPasswordReset", b => + { + b.HasOne("OpenShock.Common.OpenShockDb.User", "User") + .WithMany("PasswordResets") + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_user_password_resets_user_id"); + + b.Navigation("User"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.UserShare", b => + { + b.HasOne("OpenShock.Common.OpenShockDb.User", "SharedWithUser") + .WithMany("IncomingUserShares") + .HasForeignKey("SharedWithUserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_user_shares_shared_with_user_id"); + + b.HasOne("OpenShock.Common.OpenShockDb.Shocker", "Shocker") + .WithMany("UserShares") + .HasForeignKey("ShockerId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_user_shares_shocker_id"); + + b.Navigation("SharedWithUser"); + + b.Navigation("Shocker"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.UserShareInvite", b => + { + b.HasOne("OpenShock.Common.OpenShockDb.User", "Owner") + .WithMany("OutgoingUserShareInvites") + .HasForeignKey("OwnerId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_user_share_invites_owner_id"); + + b.HasOne("OpenShock.Common.OpenShockDb.User", "RecipientUser") + .WithMany("IncomingUserShareInvites") + .HasForeignKey("RecipientUserId") + .OnDelete(DeleteBehavior.Cascade) + .HasConstraintName("fk_user_share_invites_recipient_user_id"); + + b.Navigation("Owner"); + + b.Navigation("RecipientUser"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.UserShareInviteShocker", b => + { + b.HasOne("OpenShock.Common.OpenShockDb.UserShareInvite", "Invite") + .WithMany("ShockerMappings") + .HasForeignKey("InviteId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_user_share_invite_shockers_invite_id"); + + b.HasOne("OpenShock.Common.OpenShockDb.Shocker", "Shocker") + .WithMany("UserShareInviteShockerMappings") + .HasForeignKey("ShockerId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_user_share_invite_shockers_shocker_id"); + + b.Navigation("Invite"); + + b.Navigation("Shocker"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.AutomationToken", b => + { + b.Navigation("CreatedUsers"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.Device", b => + { + b.Navigation("OtaUpdates"); + + b.Navigation("Shockers"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.PublicShare", b => + { + b.Navigation("ShockerMappings"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.Shocker", b => + { + b.Navigation("PublicShareMappings"); + + b.Navigation("ShockerShareCodes"); + + b.Navigation("UserShareInviteShockerMappings"); + + b.Navigation("UserShares"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.User", b => + { + b.Navigation("ActorAuditLogs"); + + b.Navigation("ApiTokens"); + + b.Navigation("AuditLogs"); + + b.Navigation("Devices"); + + b.Navigation("EmailChanges"); + + b.Navigation("IncomingUserShareInvites"); + + b.Navigation("IncomingUserShares"); + + b.Navigation("NameChanges"); + + b.Navigation("OAuthConnections"); + + b.Navigation("OutgoingUserShareInvites"); + + b.Navigation("OwnedPublicShares"); + + b.Navigation("PasswordResets"); + + b.Navigation("ReportedApiTokens"); + + b.Navigation("UserActivationRequest"); + + b.Navigation("UserDeactivation"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.UserShareInvite", b => + { + b.Navigation("ShockerMappings"); + }); +#pragma warning restore 612, 618 + } +} diff --git a/Common/Migrations/20261120201327_AddControlLogHypertable.cs b/Common/Migrations/20261120201327_AddControlLogHypertable.cs new file mode 100644 index 000000000..d3579b319 --- /dev/null +++ b/Common/Migrations/20261120201327_AddControlLogHypertable.cs @@ -0,0 +1,174 @@ +using System; +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace OpenShock.Common.Migrations; + +/// +public partial class _20261120201327_AddControlLogHypertable : Migration +{ + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropForeignKey( + name: "fk_shocker_control_logs_controlled_by_user_id", + table: "shocker_control_logs"); + + migrationBuilder.DropForeignKey( + name: "fk_shocker_control_logs_shocker_id", + table: "shocker_control_logs"); + + migrationBuilder.DropTable( + name: "control_stat_batches"); + + migrationBuilder.DropTable( + name: "control_stat_removals"); + + migrationBuilder.DropTable( + name: "control_stats"); + + migrationBuilder.Sql("DROP FUNCTION control_stats_record_removals();"); + + migrationBuilder.DropPrimaryKey( + name: "shocker_control_logs_pkey", + table: "shocker_control_logs"); + + migrationBuilder.DropIndex( + name: "IX_shocker_control_logs_shocker_id", + table: "shocker_control_logs"); + + migrationBuilder.AlterDatabase() + .Annotation("Npgsql:CollationDefinition:public.ndcoll", "und-u-ks-level2,und-u-ks-level2,icu,False") + .Annotation("Npgsql:CollationSchemasResolved", true) + .Annotation("Npgsql:Enum:audit_action", "login,logout,password_changed,email_change_requested,email_changed,username_changed,api_token_created,api_token_deleted,oauth_connected,oauth_disconnected,account_deactivated,account_reactivated,account_deleted,automation_token_created,automation_token_updated,automation_token_rotated,automation_token_deleted,automation_token_used") + .Annotation("Npgsql:Enum:automation_token_type", "turnstile,rate_limit,control_stats_read") + .Annotation("Npgsql:Enum:configuration_value_type", "string,bool,int,float,json") + .Annotation("Npgsql:Enum:control_limit_mode", "clamp,lerp") + .Annotation("Npgsql:Enum:control_type", "stop,shock,vibrate,sound") + .Annotation("Npgsql:Enum:email_status", "pending,sending,sent,failed,skipped") + .Annotation("Npgsql:Enum:email_type", "account_activation,password_reset,email_verification,email_change_notice") + .Annotation("Npgsql:Enum:match_type_enum", "exact,contains") + .Annotation("Npgsql:Enum:ota_update_status", "started,running,finished,error,timeout") + .Annotation("Npgsql:Enum:password_encryption_type", "bcrypt_enhanced,pbkdf2") + .Annotation("Npgsql:Enum:permission_type", "shockers.use,shockers.edit,shockers.pause,devices.edit,devices.auth,usershares.edit,usershares.pause,publicshares.pause,publicshares.edit") + .Annotation("Npgsql:Enum:role_type", "support,staff,admin,system") + .Annotation("Npgsql:Enum:shocker_model_type", "cai_xianlin,petrainer,petrainer_998dr,wellturn_t330") + .Annotation("Npgsql:PostgresExtension:timescaledb", ",,") + .OldAnnotation("Npgsql:CollationDefinition:public.ndcoll", "und-u-ks-level2,und-u-ks-level2,icu,False") + .OldAnnotation("Npgsql:CollationSchemasResolved", true) + .OldAnnotation("Npgsql:Enum:audit_action", "login,logout,password_changed,email_change_requested,email_changed,username_changed,api_token_created,api_token_deleted,oauth_connected,oauth_disconnected,account_deactivated,account_reactivated,account_deleted,automation_token_created,automation_token_updated,automation_token_rotated,automation_token_deleted,automation_token_used") + .OldAnnotation("Npgsql:Enum:automation_token_type", "turnstile,rate_limit,control_stats_read") + .OldAnnotation("Npgsql:Enum:configuration_value_type", "string,bool,int,float,json") + .OldAnnotation("Npgsql:Enum:control_limit_mode", "clamp,lerp") + .OldAnnotation("Npgsql:Enum:control_type", "stop,shock,vibrate,sound") + .OldAnnotation("Npgsql:Enum:email_status", "pending,sending,sent,failed,skipped") + .OldAnnotation("Npgsql:Enum:email_type", "account_activation,password_reset,email_verification,email_change_notice") + .OldAnnotation("Npgsql:Enum:match_type_enum", "exact,contains") + .OldAnnotation("Npgsql:Enum:ota_update_status", "started,running,finished,error,timeout") + .OldAnnotation("Npgsql:Enum:password_encryption_type", "bcrypt_enhanced,pbkdf2") + .OldAnnotation("Npgsql:Enum:permission_type", "shockers.use,shockers.edit,shockers.pause,devices.edit,devices.auth,usershares.edit,usershares.pause,publicshares.pause,publicshares.edit") + .OldAnnotation("Npgsql:Enum:role_type", "support,staff,admin,system") + .OldAnnotation("Npgsql:Enum:shocker_model_type", "cai_xianlin,petrainer,petrainer_998dr,wellturn_t330"); + + migrationBuilder.Sql( + """ + ALTER TABLE shocker_control_logs + ADD COLUMN owner_user_id uuid, + ADD COLUMN max_intensity smallint, + ADD COLUMN inserted_at timestamp with time zone; + + UPDATE shocker_control_logs l + SET owner_user_id = d.owner_id, + max_intensity = l.intensity, + inserted_at = l.created_at + FROM shockers s + JOIN devices d ON d.id = s.device_id + WHERE s.id = l.shocker_id; + + ALTER TABLE shocker_control_logs + ALTER COLUMN owner_user_id SET NOT NULL, + ALTER COLUMN max_intensity SET NOT NULL, + ALTER COLUMN inserted_at SET DEFAULT now(), + ALTER COLUMN inserted_at SET NOT NULL; + """); + + migrationBuilder.AddPrimaryKey( + name: "shocker_control_logs_pkey", + table: "shocker_control_logs", + columns: new[] { "id", "created_at" }); + + migrationBuilder.CreateIndex( + name: "IX_shocker_control_logs_owner_user_id_created_at", + table: "shocker_control_logs", + columns: new[] { "owner_user_id", "created_at" }); + + migrationBuilder.CreateIndex( + name: "IX_shocker_control_logs_shocker_id_created_at", + table: "shocker_control_logs", + columns: new[] { "shocker_id", "created_at" }); + + // Segmented by owner, every read and the owner delete below filter by it + migrationBuilder.Sql( + """ + SELECT create_hypertable('shocker_control_logs', by_range('created_at', INTERVAL '7 days'), migrate_data => true); + + ALTER TABLE shocker_control_logs SET ( + timescaledb.enable_columnstore = true, + timescaledb.segmentby = 'owner_user_id', + timescaledb.orderby = 'created_at DESC'); + + CALL add_columnstore_policy('shocker_control_logs', after => INTERVAL '7 days'); + SELECT add_retention_policy('shocker_control_logs', drop_after => INTERVAL '1 year'); + """); + + // Logs have no foreign keys, but the owner's logs go with their account + migrationBuilder.Sql( + """ + CREATE FUNCTION users_delete_shocker_control_logs() RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN + DELETE FROM shocker_control_logs l USING removed r WHERE l.owner_user_id = r.id; + RETURN NULL; + END + $$; + + CREATE TRIGGER users_delete_shocker_control_logs + AFTER DELETE ON users + REFERENCING OLD TABLE AS removed + FOR EACH STATEMENT EXECUTE FUNCTION users_delete_shocker_control_logs(); + """); + + // 15 minute buckets, every time zone's UTC offset is a multiple of that. The refresh window must stay well + // within the logs' retention, refreshing a bucket whose logs were dropped would wipe its stats. + migrationBuilder.Sql( + """ + CREATE MATERIALIZED VIEW control_stats + WITH (timescaledb.continuous, timescaledb.materialized_only = false) AS + SELECT time_bucket(INTERVAL '15 minutes', created_at) AS bucket, + owner_user_id, + shocker_id, + controlled_by_user_id AS controller_user_id, + type, + live_control AS live, + count(*) AS count, + sum(duration)::bigint AS duration_ms, + sum(intensity::bigint * duration)::bigint AS intensity_ms, + max(max_intensity) AS max_intensity + FROM shocker_control_logs + WHERE type <> 'stop' + GROUP BY bucket, owner_user_id, shocker_id, controlled_by_user_id, type, live_control + WITH NO DATA; + + SELECT add_continuous_aggregate_policy('control_stats', + start_offset => INTERVAL '3 days', + end_offset => INTERVAL '1 hour', + schedule_interval => INTERVAL '5 minutes'); + """); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + throw new NotSupportedException("shocker_control_logs can't be turned back into a plain table, restore a backup instead"); + } +} diff --git a/Common/Migrations/20261120201415_MaterializeControlStats.Designer.cs b/Common/Migrations/20261120201415_MaterializeControlStats.Designer.cs new file mode 100644 index 000000000..79fdeb8d0 --- /dev/null +++ b/Common/Migrations/20261120201415_MaterializeControlStats.Designer.cs @@ -0,0 +1,1832 @@ +// +using System; +using System.Collections.Generic; +using System.Net; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; +using OpenShock.Common.OpenShockDb; + +#nullable disable + +namespace OpenShock.Common.Migrations; + +[DbContext(typeof(MigrationOpenShockContext))] +[Migration("20261120201415_MaterializeControlStats")] +partial class _20261120201415_MaterializeControlStats +{ + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("Npgsql:CollationDefinition:public.ndcoll", "und-u-ks-level2,und-u-ks-level2,icu,False") + .HasAnnotation("ProductVersion", "11.0.0") + .HasAnnotation("Relational:MaxIdentifierLength", 63); + + NpgsqlModelBuilderExtensions.HasPostgresEnum(modelBuilder, "audit_action", new[] { "login", "logout", "password_changed", "email_change_requested", "email_changed", "username_changed", "api_token_created", "api_token_deleted", "oauth_connected", "oauth_disconnected", "account_deactivated", "account_reactivated", "account_deleted", "automation_token_created", "automation_token_updated", "automation_token_rotated", "automation_token_deleted", "automation_token_used" }); + NpgsqlModelBuilderExtensions.HasPostgresEnum(modelBuilder, "automation_token_type", new[] { "turnstile", "rate_limit", "control_stats_read" }); + NpgsqlModelBuilderExtensions.HasPostgresEnum(modelBuilder, "configuration_value_type", new[] { "string", "bool", "int", "float", "json" }); + NpgsqlModelBuilderExtensions.HasPostgresEnum(modelBuilder, "control_limit_mode", new[] { "clamp", "lerp" }); + NpgsqlModelBuilderExtensions.HasPostgresEnum(modelBuilder, "control_type", new[] { "stop", "shock", "vibrate", "sound" }); + NpgsqlModelBuilderExtensions.HasPostgresEnum(modelBuilder, "email_status", new[] { "pending", "sending", "sent", "failed", "skipped" }); + NpgsqlModelBuilderExtensions.HasPostgresEnum(modelBuilder, "email_type", new[] { "account_activation", "password_reset", "email_verification", "email_change_notice" }); + NpgsqlModelBuilderExtensions.HasPostgresEnum(modelBuilder, "match_type_enum", new[] { "exact", "contains" }); + NpgsqlModelBuilderExtensions.HasPostgresEnum(modelBuilder, "ota_update_status", new[] { "started", "running", "finished", "error", "timeout" }); + NpgsqlModelBuilderExtensions.HasPostgresEnum(modelBuilder, "password_encryption_type", new[] { "bcrypt_enhanced", "pbkdf2" }); + NpgsqlModelBuilderExtensions.HasPostgresEnum(modelBuilder, "permission_type", new[] { "shockers.use", "shockers.edit", "shockers.pause", "devices.edit", "devices.auth", "usershares.edit", "usershares.pause", "publicshares.pause", "publicshares.edit" }); + NpgsqlModelBuilderExtensions.HasPostgresEnum(modelBuilder, "role_type", new[] { "support", "staff", "admin", "system" }); + NpgsqlModelBuilderExtensions.HasPostgresEnum(modelBuilder, "shocker_model_type", new[] { "cai_xianlin", "petrainer", "petrainer_998dr", "wellturn_t330" }); + NpgsqlModelBuilderExtensions.HasPostgresExtension(modelBuilder, "timescaledb"); + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + + modelBuilder.Entity("Microsoft.AspNetCore.DataProtection.EntityFrameworkCore.DataProtectionKey", b => + { + var id = b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(id); + + b.Property("FriendlyName") + .HasColumnType("text"); + + b.Property("Xml") + .HasColumnType("text"); + + b.HasKey("Id"); + + b.ToTable("DataProtectionKeys"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.AdminUsersView", b => + { + b.Property("ActivatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("activated_at"); + + b.Property("ApiTokenCount") + .HasColumnType("integer") + .HasColumnName("api_token_count"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at"); + + b.Property("CreatedByAutomationTokenId") + .HasColumnType("uuid") + .HasColumnName("created_by_automation_token_id"); + + b.Property("DeactivatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("deactivated_at"); + + b.Property("DeactivatedByUserId") + .HasColumnType("uuid") + .HasColumnName("deactivated_by_user_id"); + + b.Property("DeviceCount") + .HasColumnType("integer") + .HasColumnName("device_count"); + + b.Property("Email") + .IsRequired() + .HasColumnType("character varying") + .HasColumnName("email"); + + b.Property("EmailChangeRequestCount") + .HasColumnType("integer") + .HasColumnName("email_change_request_count"); + + b.Property("Id") + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("Name") + .IsRequired() + .HasColumnType("character varying") + .HasColumnName("name"); + + b.Property("NameChangeRequestCount") + .HasColumnType("integer") + .HasColumnName("name_change_request_count"); + + b.Property("PasswordHashType") + .HasColumnType("character varying") + .HasColumnName("password_hash_type"); + + b.Property("PasswordResetCount") + .HasColumnType("integer") + .HasColumnName("password_reset_count"); + + b.Property("Roles") + .IsRequired() + .HasColumnType("role_type[]") + .HasColumnName("roles"); + + b.Property("ShockerControlLogCount") + .HasColumnType("integer") + .HasColumnName("shocker_control_log_count"); + + b.Property("ShockerCount") + .HasColumnType("integer") + .HasColumnName("shocker_count"); + + b.Property("ShockerPublicShareCount") + .HasColumnType("integer") + .HasColumnName("shocker_public_share_count"); + + b.Property("ShockerUserShareCount") + .HasColumnType("integer") + .HasColumnName("shocker_user_share_count"); + + b.ToTable((string)null); + + b.ToView("admin_users_view", (string)null); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.ApiToken", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("CreatedAt") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at") + .HasDefaultValueSql("CURRENT_TIMESTAMP"); + + b.Property("CreatedByIp") + .IsRequired() + .HasColumnType("inet") + .HasColumnName("created_by_ip"); + + b.Property("LastUsed") + .HasColumnType("timestamp with time zone") + .HasColumnName("last_used"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("character varying(64)") + .HasColumnName("name"); + + b.Property>("Permissions") + .IsRequired() + .HasColumnType("permission_type[]") + .HasColumnName("permissions"); + + b.Property("ShockerControlDurationMax") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasDefaultValue(65535) + .HasColumnName("shocker_control_duration_max"); + + b.Property("ShockerControlDurationMin") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasDefaultValue(300) + .HasColumnName("shocker_control_duration_min"); + + b.Property("ShockerControlDurationMode") + .ValueGeneratedOnAdd() + .HasColumnType("control_limit_mode") + .HasDefaultValue(ControlLimitMode.Clamp) + .HasColumnName("shocker_control_duration_mode"); + + b.Property("ShockerControlIntensityMax") + .ValueGeneratedOnAdd() + .HasColumnType("smallint") + .HasDefaultValue((byte)100) + .HasColumnName("shocker_control_intensity_max"); + + b.Property("ShockerControlIntensityMin") + .ValueGeneratedOnAdd() + .HasColumnType("smallint") + .HasDefaultValue((byte)0) + .HasColumnName("shocker_control_intensity_min"); + + b.Property("ShockerControlIntensityMode") + .ValueGeneratedOnAdd() + .HasColumnType("control_limit_mode") + .HasDefaultValue(ControlLimitMode.Clamp) + .HasColumnName("shocker_control_intensity_mode"); + + b.Property("ShockerControlPaused") + .ValueGeneratedOnAdd() + .HasColumnType("boolean") + .HasDefaultValue(false) + .HasColumnName("shocker_control_paused"); + + b.Property("TokenHash") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("character varying(64)") + .HasColumnName("token_hash") + .UseCollation("C"); + + b.Property("UserId") + .HasColumnType("uuid") + .HasColumnName("user_id"); + + b.Property("ValidUntil") + .HasColumnType("timestamp with time zone") + .HasColumnName("valid_until"); + + b.HasKey("Id") + .HasName("api_tokens_pkey"); + + b.HasIndex("TokenHash") + .IsUnique(); + + b.HasIndex("UserId"); + + b.HasIndex("ValidUntil"); + + b.ToTable("api_tokens"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.ApiTokenReport", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("AffectedCount") + .HasColumnType("integer") + .HasColumnName("affected_count"); + + b.Property("CreatedAt") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at") + .HasDefaultValueSql("CURRENT_TIMESTAMP"); + + b.Property("IpAddress") + .IsRequired() + .HasColumnType("inet") + .HasColumnName("ip_address"); + + b.Property("IpCountry") + .HasMaxLength(4) + .HasColumnType("character varying(4)") + .HasColumnName("ip_country"); + + b.Property("SubmittedCount") + .HasColumnType("integer") + .HasColumnName("submitted_count"); + + b.Property("UserId") + .HasColumnType("uuid") + .HasColumnName("user_id"); + + b.HasKey("Id") + .HasName("api_token_reports_pkey"); + + b.HasIndex("UserId"); + + b.ToTable("api_token_reports"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.AutomationToken", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("AutoCleanupAfter") + .HasColumnType("interval") + .HasColumnName("auto_cleanup_after"); + + b.Property("AutoCleanupUsers") + .ValueGeneratedOnAdd() + .HasColumnType("boolean") + .HasDefaultValue(false) + .HasColumnName("auto_cleanup_users"); + + b.Property("CreatedAt") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at") + .HasDefaultValueSql("CURRENT_TIMESTAMP"); + + b.Property("LastRotatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("last_rotated_at"); + + b.Property("LastUsedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("last_used_at"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("character varying(64)") + .HasColumnName("name"); + + b.Property("TokenHash") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("character varying(64)") + .HasColumnName("token_hash") + .UseCollation("C"); + + b.Property>("Types") + .IsRequired() + .HasColumnType("automation_token_type[]") + .HasColumnName("types"); + + b.Property("UseCount") + .ValueGeneratedOnAdd() + .HasColumnType("bigint") + .HasDefaultValue(0L) + .HasColumnName("use_count"); + + b.HasKey("Id") + .HasName("automation_tokens_pkey"); + + b.HasIndex("TokenHash") + .IsUnique(); + + b.ToTable("automation_tokens"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.ConfigurationItem", b => + { + b.Property("Name") + .HasMaxLength(64) + .HasColumnType("character varying(64)") + .HasColumnName("name") + .UseCollation("C"); + + b.Property("CreatedAt") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at") + .HasDefaultValueSql("CURRENT_TIMESTAMP"); + + b.Property("Description") + .IsRequired() + .HasMaxLength(255) + .HasColumnType("character varying(255)") + .HasColumnName("description"); + + b.Property("Type") + .HasColumnType("configuration_value_type") + .HasColumnName("type"); + + b.Property("UpdatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("updated_at"); + + b.Property("Value") + .IsRequired() + .HasMaxLength(4096) + .HasColumnType("character varying(4096)") + .HasColumnName("value"); + + b.HasKey("Name") + .HasName("configuration_pkey"); + + b.ToTable("configuration"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.ControlStat", b => + { + b.Property("Bucket") + .HasColumnType("timestamp with time zone") + .HasColumnName("bucket"); + + b.Property("ControllerUserId") + .HasColumnType("uuid") + .HasColumnName("controller_user_id"); + + b.Property("Count") + .HasColumnType("bigint") + .HasColumnName("count"); + + b.Property("DurationMs") + .HasColumnType("bigint") + .HasColumnName("duration_ms"); + + b.Property("IntensityMs") + .HasColumnType("bigint") + .HasColumnName("intensity_ms"); + + b.Property("Live") + .HasColumnType("boolean") + .HasColumnName("live"); + + b.Property("MaxIntensity") + .HasColumnType("smallint") + .HasColumnName("max_intensity"); + + b.Property("OwnerUserId") + .HasColumnType("uuid") + .HasColumnName("owner_user_id"); + + b.Property("ShockerId") + .HasColumnType("uuid") + .HasColumnName("shocker_id"); + + b.Property("Type") + .HasColumnType("control_type") + .HasColumnName("type"); + + b.ToTable((string)null); + + b.ToView("control_stats", (string)null); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.Device", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("CreatedAt") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at") + .HasDefaultValueSql("CURRENT_TIMESTAMP"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("character varying(64)") + .HasColumnName("name"); + + b.Property("OwnerId") + .HasColumnType("uuid") + .HasColumnName("owner_id"); + + b.Property("Token") + .IsRequired() + .HasMaxLength(256) + .HasColumnType("character varying(256)") + .HasColumnName("token") + .UseCollation("C"); + + b.HasKey("Id") + .HasName("devices_pkey"); + + b.HasIndex("OwnerId"); + + b.HasIndex("Token") + .IsUnique(); + + b.ToTable("devices"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.DeviceOtaUpdate", b => + { + b.Property("DeviceId") + .HasColumnType("uuid") + .HasColumnName("device_id"); + + b.Property("UpdateId") + .HasColumnType("integer") + .HasColumnName("update_id"); + + b.Property("CreatedAt") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at") + .HasDefaultValueSql("CURRENT_TIMESTAMP"); + + b.Property("Message") + .HasMaxLength(128) + .HasColumnType("character varying(128)") + .HasColumnName("message"); + + b.Property("Status") + .HasColumnType("ota_update_status") + .HasColumnName("status"); + + b.Property("Version") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("character varying(64)") + .HasColumnName("version"); + + b.HasKey("DeviceId", "UpdateId") + .HasName("device_ota_updates_pkey"); + + b.HasIndex(new[] { "CreatedAt" }, "device_ota_updates_created_at_idx"); + + b.ToTable("device_ota_updates"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.DiscordWebhook", b => + { + b.Property("Name") + .HasMaxLength(64) + .HasColumnType("character varying(64)") + .HasColumnName("name"); + + b.Property("CreatedAt") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at") + .HasDefaultValueSql("CURRENT_TIMESTAMP"); + + b.Property("Id") + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("WebhookId") + .HasColumnType("bigint") + .HasColumnName("webhook_id"); + + b.Property("WebhookToken") + .IsRequired() + .HasMaxLength(255) + .HasColumnType("character varying(255)") + .HasColumnName("webhook_token"); + + b.HasKey("Name") + .HasName("discord_webhooks_pkey"); + + b.ToTable("discord_webhooks"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.EmailOutboxMessage", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("AttemptCount") + .IsConcurrencyToken() + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasDefaultValue(0) + .HasColumnName("attempt_count"); + + b.Property("CoalesceKey") + .HasMaxLength(128) + .HasColumnType("character varying(128)") + .HasColumnName("coalesce_key"); + + b.Property("CreatedAt") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at") + .HasDefaultValueSql("CURRENT_TIMESTAMP"); + + b.Property("FailedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("failed_at"); + + b.Property("LastError") + .HasMaxLength(1024) + .HasColumnType("character varying(1024)") + .HasColumnName("last_error"); + + b.Property("NextAttemptAt") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasColumnName("next_attempt_at") + .HasDefaultValueSql("CURRENT_TIMESTAMP"); + + b.Property>("Payload") + .IsRequired() + .HasColumnType("jsonb") + .HasColumnName("payload"); + + b.Property("Recipient") + .IsRequired() + .HasMaxLength(320) + .HasColumnType("character varying(320)") + .HasColumnName("recipient"); + + b.Property("RecipientName") + .HasMaxLength(32) + .HasColumnType("character varying(32)") + .HasColumnName("recipient_name"); + + b.Property("SentAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("sent_at"); + + b.Property("Status") + .HasColumnType("email_status") + .HasColumnName("status"); + + b.Property("Type") + .HasColumnType("email_type") + .HasColumnName("type"); + + b.HasKey("Id") + .HasName("email_outbox_pkey"); + + b.HasIndex("CoalesceKey"); + + b.HasIndex("Recipient"); + + b.HasIndex("Status", "NextAttemptAt"); + + b.ToTable("email_outbox"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.EmailProviderBlacklist", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("CreatedAt") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at") + .HasDefaultValueSql("CURRENT_TIMESTAMP"); + + b.Property("Domain") + .IsRequired() + .HasMaxLength(255) + .HasColumnType("character varying(255)") + .HasColumnName("domain") + .UseCollation("ndcoll"); + + b.HasKey("Id") + .HasName("email_provider_blacklist_pkey"); + + var index = b.HasIndex("Domain") + .IsUnique(); + + NpgsqlIndexBuilderExtensions.UseCollation(index, new[] { "ndcoll" }); + + b.ToTable("email_provider_blacklist"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.PublicShare", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("CreatedAt") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at") + .HasDefaultValueSql("CURRENT_TIMESTAMP"); + + b.Property("ExpiresAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("expires_at"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("character varying(64)") + .HasColumnName("name"); + + b.Property("OwnerId") + .HasColumnType("uuid") + .HasColumnName("owner_id"); + + b.HasKey("Id") + .HasName("public_shares_pkey"); + + b.HasIndex("OwnerId"); + + b.ToTable("public_shares"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.PublicShareShocker", b => + { + b.Property("PublicShareId") + .HasColumnType("uuid") + .HasColumnName("public_share_id"); + + b.Property("ShockerId") + .HasColumnType("uuid") + .HasColumnName("shocker_id"); + + b.Property("AllowLiveControl") + .ValueGeneratedOnAdd() + .HasColumnType("boolean") + .HasDefaultValue(false) + .HasColumnName("allow_livecontrol"); + + b.Property("AllowShock") + .ValueGeneratedOnAdd() + .HasColumnType("boolean") + .HasDefaultValue(true) + .HasColumnName("allow_shock"); + + b.Property("AllowSound") + .ValueGeneratedOnAdd() + .HasColumnType("boolean") + .HasDefaultValue(true) + .HasColumnName("allow_sound"); + + b.Property("AllowVibrate") + .ValueGeneratedOnAdd() + .HasColumnType("boolean") + .HasDefaultValue(true) + .HasColumnName("allow_vibrate"); + + b.Property("Cooldown") + .HasColumnType("integer") + .HasColumnName("cooldown"); + + b.Property("IsPaused") + .ValueGeneratedOnAdd() + .HasColumnType("boolean") + .HasDefaultValue(false) + .HasColumnName("is_paused"); + + b.Property("MaxDuration") + .HasColumnType("integer") + .HasColumnName("max_duration"); + + b.Property("MaxIntensity") + .HasColumnType("smallint") + .HasColumnName("max_intensity"); + + b.HasKey("PublicShareId", "ShockerId") + .HasName("public_share_shockers_pkey"); + + b.HasIndex("ShockerId"); + + b.ToTable("public_share_shockers"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.Shocker", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("CreatedAt") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at") + .HasDefaultValueSql("CURRENT_TIMESTAMP"); + + b.Property("DeviceId") + .HasColumnType("uuid") + .HasColumnName("device_id"); + + b.Property("IsPaused") + .ValueGeneratedOnAdd() + .HasColumnType("boolean") + .HasDefaultValue(false) + .HasColumnName("is_paused"); + + b.Property("Model") + .HasColumnType("shocker_model_type") + .HasColumnName("model"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("character varying(64)") + .HasColumnName("name"); + + b.Property("RfId") + .HasColumnType("integer") + .HasColumnName("rf_id"); + + b.HasKey("Id") + .HasName("shockers_pkey"); + + b.HasIndex("DeviceId"); + + b.ToTable("shockers"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.ShockerControlLog", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("CreatedAt") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at") + .HasDefaultValueSql("CURRENT_TIMESTAMP"); + + b.Property("ControlledByUserId") + .HasColumnType("uuid") + .HasColumnName("controlled_by_user_id"); + + b.Property("CustomName") + .HasMaxLength(64) + .HasColumnType("character varying(64)") + .HasColumnName("custom_name"); + + b.Property("Duration") + .HasColumnType("bigint") + .HasColumnName("duration"); + + b.Property("InsertedAt") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasColumnName("inserted_at") + .HasDefaultValueSql("now()"); + + b.Property("Intensity") + .HasColumnType("smallint") + .HasColumnName("intensity"); + + b.Property("LiveControl") + .ValueGeneratedOnAdd() + .HasColumnType("boolean") + .HasDefaultValue(false) + .HasColumnName("live_control"); + + b.Property("MaxIntensity") + .HasColumnType("smallint") + .HasColumnName("max_intensity"); + + b.Property("OwnerUserId") + .HasColumnType("uuid") + .HasColumnName("owner_user_id"); + + b.Property("ShockerId") + .HasColumnType("uuid") + .HasColumnName("shocker_id"); + + b.Property("Type") + .HasColumnType("control_type") + .HasColumnName("type"); + + b.HasKey("Id", "CreatedAt") + .HasName("shocker_control_logs_pkey"); + + b.HasIndex("ControlledByUserId"); + + b.HasIndex("OwnerUserId", "CreatedAt"); + + b.HasIndex("ShockerId", "CreatedAt"); + + b.ToTable("shocker_control_logs"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.ShockerShareCode", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("AllowLiveControl") + .ValueGeneratedOnAdd() + .HasColumnType("boolean") + .HasDefaultValue(true) + .HasColumnName("allow_livecontrol"); + + b.Property("AllowShock") + .ValueGeneratedOnAdd() + .HasColumnType("boolean") + .HasDefaultValue(true) + .HasColumnName("allow_shock"); + + b.Property("AllowSound") + .ValueGeneratedOnAdd() + .HasColumnType("boolean") + .HasDefaultValue(true) + .HasColumnName("allow_sound"); + + b.Property("AllowVibrate") + .ValueGeneratedOnAdd() + .HasColumnType("boolean") + .HasDefaultValue(true) + .HasColumnName("allow_vibrate"); + + b.Property("CreatedAt") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at") + .HasDefaultValueSql("CURRENT_TIMESTAMP"); + + b.Property("IsPaused") + .ValueGeneratedOnAdd() + .HasColumnType("boolean") + .HasDefaultValue(false) + .HasColumnName("is_paused"); + + b.Property("MaxDuration") + .HasColumnType("integer") + .HasColumnName("max_duration"); + + b.Property("MaxIntensity") + .HasColumnType("smallint") + .HasColumnName("max_intensity"); + + b.Property("ShockerId") + .HasColumnType("uuid") + .HasColumnName("shocker_id"); + + b.HasKey("Id") + .HasName("shocker_share_codes_pkey"); + + b.HasIndex("ShockerId"); + + b.ToTable("shocker_share_codes"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.User", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("ActivatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("activated_at"); + + b.Property("CreatedAt") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at") + .HasDefaultValueSql("CURRENT_TIMESTAMP"); + + b.Property("CreatedByAutomationTokenId") + .HasColumnType("uuid") + .HasColumnName("created_by_automation_token_id"); + + b.Property("Email") + .IsRequired() + .HasMaxLength(320) + .HasColumnType("character varying(320)") + .HasColumnName("email"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(32) + .HasColumnType("character varying(32)") + .HasColumnName("name") + .UseCollation("ndcoll"); + + b.Property("PasswordHash") + .HasMaxLength(100) + .HasColumnType("character varying(100)") + .HasColumnName("password_hash") + .UseCollation("C"); + + b.Property>("Roles") + .IsRequired() + .HasColumnType("role_type[]") + .HasColumnName("roles"); + + b.Property("SecurityStamp") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("security_stamp") + .HasDefaultValueSql("gen_random_uuid()"); + + b.HasKey("Id") + .HasName("users_pkey"); + + b.HasIndex("CreatedByAutomationTokenId"); + + b.HasIndex("Email") + .IsUnique(); + + var index = b.HasIndex("Name") + .IsUnique(); + + NpgsqlIndexBuilderExtensions.UseCollation(index, new[] { "ndcoll" }); + + b.ToTable("users"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.UserActivationRequest", b => + { + b.Property("UserId") + .HasColumnType("uuid") + .HasColumnName("user_id"); + + b.Property("CreatedAt") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at") + .HasDefaultValueSql("CURRENT_TIMESTAMP"); + + b.Property("EmailSendAttempts") + .HasColumnType("integer") + .HasColumnName("email_send_attempts"); + + b.Property("TokenHash") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("character varying(128)") + .HasColumnName("token_hash") + .UseCollation("C"); + + b.HasKey("UserId") + .HasName("user_activation_requests_pkey"); + + b.HasIndex("TokenHash") + .IsUnique(); + + b.ToTable("user_activation_requests"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.UserAuditLog", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("Action") + .HasColumnType("audit_action") + .HasColumnName("action"); + + b.Property("ActorId") + .HasColumnType("uuid") + .HasColumnName("actor_id"); + + b.Property("CreatedAt") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at") + .HasDefaultValueSql("CURRENT_TIMESTAMP"); + + b.Property("IpAddress") + .HasMaxLength(40) + .HasColumnType("character varying(40)") + .HasColumnName("ip_address"); + + b.Property("Metadata") + .HasColumnType("jsonb") + .HasColumnName("metadata"); + + b.Property("Reason") + .HasMaxLength(512) + .HasColumnType("character varying(512)") + .HasColumnName("reason"); + + b.Property("UserAgent") + .HasMaxLength(1024) + .HasColumnType("character varying(1024)") + .HasColumnName("user_agent"); + + b.Property("UserId") + .HasColumnType("uuid") + .HasColumnName("user_id"); + + b.HasKey("Id") + .HasName("user_audit_logs_pkey"); + + b.HasIndex("ActorId"); + + b.HasIndex("CreatedAt"); + + b.HasIndex("UserId"); + + b.ToTable("user_audit_logs"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.UserDeactivation", b => + { + b.Property("DeactivatedUserId") + .HasColumnType("uuid") + .HasColumnName("deactivated_user_id"); + + b.Property("CreatedAt") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at") + .HasDefaultValueSql("CURRENT_TIMESTAMP"); + + b.Property("DeactivatedByUserId") + .HasColumnType("uuid") + .HasColumnName("deactivated_by_user_id"); + + b.Property("DeleteLater") + .HasColumnType("boolean") + .HasColumnName("delete_later"); + + b.Property("UserModerationId") + .HasColumnType("uuid") + .HasColumnName("user_moderation_id"); + + b.HasKey("DeactivatedUserId") + .HasName("user_deactivations_pkey"); + + b.HasIndex("DeactivatedByUserId"); + + b.ToTable("user_deactivations"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.UserEmailChange", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("CreatedAt") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at") + .HasDefaultValueSql("CURRENT_TIMESTAMP"); + + b.Property("NewEmail") + .IsRequired() + .HasMaxLength(320) + .HasColumnType("character varying(320)") + .HasColumnName("email_new"); + + b.Property("OldEmail") + .IsRequired() + .HasMaxLength(320) + .HasColumnType("character varying(320)") + .HasColumnName("email_old"); + + b.Property("SecurityStampAtCreate") + .HasColumnType("uuid") + .HasColumnName("security_stamp_at_create"); + + b.Property("TokenHash") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("character varying(128)") + .HasColumnName("token_hash") + .UseCollation("C"); + + b.Property("UsedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("used_at"); + + b.Property("UserId") + .HasColumnType("uuid") + .HasColumnName("user_id"); + + b.HasKey("Id") + .HasName("user_email_changes_pkey"); + + b.HasIndex("CreatedAt"); + + b.HasIndex("UsedAt"); + + b.HasIndex("UserId"); + + b.ToTable("user_email_changes"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.UserNameBlacklist", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("CreatedAt") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at") + .HasDefaultValueSql("CURRENT_TIMESTAMP"); + + b.Property("MatchType") + .HasColumnType("match_type_enum") + .HasColumnName("match_type"); + + b.Property("Value") + .IsRequired() + .HasMaxLength(32) + .HasColumnType("character varying(32)") + .HasColumnName("value") + .UseCollation("ndcoll"); + + b.HasKey("Id") + .HasName("user_name_blacklist_pkey"); + + var index = b.HasIndex("Value") + .IsUnique(); + + NpgsqlIndexBuilderExtensions.UseCollation(index, new[] { "ndcoll" }); + + b.ToTable("user_name_blacklist"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.UserNameChange", b => + { + var id = b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityAlwaysColumn(id); + + b.Property("UserId") + .HasColumnType("uuid") + .HasColumnName("user_id"); + + b.Property("CreatedAt") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at") + .HasDefaultValueSql("CURRENT_TIMESTAMP"); + + b.Property("OldName") + .IsRequired() + .HasMaxLength(32) + .HasColumnType("character varying(32)") + .HasColumnName("old_name"); + + b.HasKey("Id", "UserId") + .HasName("user_name_changes_pkey"); + + b.HasIndex("CreatedAt"); + + b.HasIndex("OldName"); + + b.HasIndex("UserId"); + + b.ToTable("user_name_changes"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.UserOAuthConnection", b => + { + b.Property("ProviderKey") + .HasMaxLength(16) + .HasColumnType("character varying(16)") + .HasColumnName("provider_key") + .UseCollation("C"); + + b.Property("ExternalId") + .HasMaxLength(255) + .HasColumnType("character varying(255)") + .HasColumnName("external_id"); + + b.Property("CreatedAt") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at") + .HasDefaultValueSql("CURRENT_TIMESTAMP"); + + b.Property("DisplayName") + .HasMaxLength(64) + .HasColumnType("character varying(64)") + .HasColumnName("display_name"); + + b.Property("UserId") + .HasColumnType("uuid") + .HasColumnName("user_id"); + + b.HasKey("ProviderKey", "ExternalId") + .HasName("user_oauth_connections_pkey"); + + b.HasIndex("UserId"); + + b.ToTable("user_oauth_connections"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.UserPasswordReset", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("CreatedAt") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at") + .HasDefaultValueSql("CURRENT_TIMESTAMP"); + + b.Property("SecurityStampAtCreate") + .HasColumnType("uuid") + .HasColumnName("security_stamp_at_create"); + + b.Property("TokenHash") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("character varying(100)") + .HasColumnName("token_hash") + .UseCollation("C"); + + b.Property("UsedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("used_at"); + + b.Property("UserId") + .HasColumnType("uuid") + .HasColumnName("user_id"); + + b.HasKey("Id") + .HasName("user_password_resets_pkey"); + + b.HasIndex("UserId"); + + b.ToTable("user_password_resets"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.UserShare", b => + { + b.Property("SharedWithUserId") + .HasColumnType("uuid") + .HasColumnName("shared_with_user_id"); + + b.Property("ShockerId") + .HasColumnType("uuid") + .HasColumnName("shocker_id"); + + b.Property("AllowLiveControl") + .ValueGeneratedOnAdd() + .HasColumnType("boolean") + .HasDefaultValue(true) + .HasColumnName("allow_livecontrol"); + + b.Property("AllowShock") + .ValueGeneratedOnAdd() + .HasColumnType("boolean") + .HasDefaultValue(true) + .HasColumnName("allow_shock"); + + b.Property("AllowSound") + .ValueGeneratedOnAdd() + .HasColumnType("boolean") + .HasDefaultValue(true) + .HasColumnName("allow_sound"); + + b.Property("AllowVibrate") + .ValueGeneratedOnAdd() + .HasColumnType("boolean") + .HasDefaultValue(true) + .HasColumnName("allow_vibrate"); + + b.Property("CreatedAt") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at") + .HasDefaultValueSql("CURRENT_TIMESTAMP"); + + b.Property("IsPaused") + .ValueGeneratedOnAdd() + .HasColumnType("boolean") + .HasDefaultValue(false) + .HasColumnName("is_paused"); + + b.Property("MaxDuration") + .HasColumnType("integer") + .HasColumnName("max_duration"); + + b.Property("MaxIntensity") + .HasColumnType("smallint") + .HasColumnName("max_intensity"); + + b.HasKey("SharedWithUserId", "ShockerId") + .HasName("user_shares_pkey"); + + b.HasIndex("SharedWithUserId"); + + b.HasIndex("ShockerId"); + + b.ToTable("user_shares"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.UserShareInvite", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("CreatedAt") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at") + .HasDefaultValueSql("CURRENT_TIMESTAMP"); + + b.Property("OwnerId") + .HasColumnType("uuid") + .HasColumnName("owner_id"); + + b.Property("RecipientUserId") + .HasColumnType("uuid") + .HasColumnName("user_id"); + + b.HasKey("Id") + .HasName("user_share_invites_pkey"); + + b.HasIndex("OwnerId"); + + b.HasIndex("RecipientUserId"); + + b.ToTable("user_share_invites"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.UserShareInviteShocker", b => + { + b.Property("InviteId") + .HasColumnType("uuid") + .HasColumnName("invite_id"); + + b.Property("ShockerId") + .HasColumnType("uuid") + .HasColumnName("shocker_id"); + + b.Property("AllowLiveControl") + .ValueGeneratedOnAdd() + .HasColumnType("boolean") + .HasDefaultValue(true) + .HasColumnName("allow_livecontrol"); + + b.Property("AllowShock") + .ValueGeneratedOnAdd() + .HasColumnType("boolean") + .HasDefaultValue(true) + .HasColumnName("allow_shock"); + + b.Property("AllowSound") + .ValueGeneratedOnAdd() + .HasColumnType("boolean") + .HasDefaultValue(true) + .HasColumnName("allow_sound"); + + b.Property("AllowVibrate") + .ValueGeneratedOnAdd() + .HasColumnType("boolean") + .HasDefaultValue(true) + .HasColumnName("allow_vibrate"); + + b.Property("IsPaused") + .ValueGeneratedOnAdd() + .HasColumnType("boolean") + .HasDefaultValue(false) + .HasColumnName("is_paused"); + + b.Property("MaxDuration") + .HasColumnType("integer") + .HasColumnName("max_duration"); + + b.Property("MaxIntensity") + .HasColumnType("smallint") + .HasColumnName("max_intensity"); + + b.HasKey("InviteId", "ShockerId") + .HasName("user_share_invite_shockers_pkey"); + + b.HasIndex("ShockerId"); + + b.ToTable("user_share_invite_shockers"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.ApiToken", b => + { + b.HasOne("OpenShock.Common.OpenShockDb.User", "User") + .WithMany("ApiTokens") + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_api_tokens_user_id"); + + b.Navigation("User"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.ApiTokenReport", b => + { + b.HasOne("OpenShock.Common.OpenShockDb.User", "ReportedByUser") + .WithMany("ReportedApiTokens") + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_api_token_reports_reported_by_user_id"); + + b.Navigation("ReportedByUser"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.Device", b => + { + b.HasOne("OpenShock.Common.OpenShockDb.User", "Owner") + .WithMany("Devices") + .HasForeignKey("OwnerId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_devices_owner_id"); + + b.Navigation("Owner"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.DeviceOtaUpdate", b => + { + b.HasOne("OpenShock.Common.OpenShockDb.Device", "Device") + .WithMany("OtaUpdates") + .HasForeignKey("DeviceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_device_ota_updates_device_id"); + + b.Navigation("Device"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.PublicShare", b => + { + b.HasOne("OpenShock.Common.OpenShockDb.User", "Owner") + .WithMany("OwnedPublicShares") + .HasForeignKey("OwnerId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_public_shares_owner_id"); + + b.Navigation("Owner"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.PublicShareShocker", b => + { + b.HasOne("OpenShock.Common.OpenShockDb.PublicShare", "PublicShare") + .WithMany("ShockerMappings") + .HasForeignKey("PublicShareId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_public_share_shockers_public_share_id"); + + b.HasOne("OpenShock.Common.OpenShockDb.Shocker", "Shocker") + .WithMany("PublicShareMappings") + .HasForeignKey("ShockerId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_public_share_shockers_shocker_id"); + + b.Navigation("PublicShare"); + + b.Navigation("Shocker"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.Shocker", b => + { + b.HasOne("OpenShock.Common.OpenShockDb.Device", "Device") + .WithMany("Shockers") + .HasForeignKey("DeviceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_shockers_device_id"); + + b.Navigation("Device"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.ShockerShareCode", b => + { + b.HasOne("OpenShock.Common.OpenShockDb.Shocker", "Shocker") + .WithMany("ShockerShareCodes") + .HasForeignKey("ShockerId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_shocker_share_codes_shocker_id"); + + b.Navigation("Shocker"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.User", b => + { + b.HasOne("OpenShock.Common.OpenShockDb.AutomationToken", "CreatedByAutomationToken") + .WithMany("CreatedUsers") + .HasForeignKey("CreatedByAutomationTokenId") + .OnDelete(DeleteBehavior.Restrict) + .HasConstraintName("fk_users_created_by_automation_token_id"); + + b.Navigation("CreatedByAutomationToken"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.UserActivationRequest", b => + { + b.HasOne("OpenShock.Common.OpenShockDb.User", "User") + .WithOne("UserActivationRequest") + .HasForeignKey("OpenShock.Common.OpenShockDb.UserActivationRequest", "UserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_user_activation_requests_user_id"); + + b.Navigation("User"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.UserAuditLog", b => + { + b.HasOne("OpenShock.Common.OpenShockDb.User", "Actor") + .WithMany("ActorAuditLogs") + .HasForeignKey("ActorId") + .OnDelete(DeleteBehavior.SetNull) + .HasConstraintName("fk_user_audit_logs_actor_id"); + + b.HasOne("OpenShock.Common.OpenShockDb.User", "User") + .WithMany("AuditLogs") + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_user_audit_logs_user_id"); + + b.Navigation("Actor"); + + b.Navigation("User"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.UserDeactivation", b => + { + b.HasOne("OpenShock.Common.OpenShockDb.User", "DeactivatedByUser") + .WithMany() + .HasForeignKey("DeactivatedByUserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_user_deactivations_deactivated_by_user_id"); + + b.HasOne("OpenShock.Common.OpenShockDb.User", "DeactivatedUser") + .WithOne("UserDeactivation") + .HasForeignKey("OpenShock.Common.OpenShockDb.UserDeactivation", "DeactivatedUserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_user_deactivations_deactivated_user_id"); + + b.Navigation("DeactivatedByUser"); + + b.Navigation("DeactivatedUser"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.UserEmailChange", b => + { + b.HasOne("OpenShock.Common.OpenShockDb.User", "User") + .WithMany("EmailChanges") + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_user_email_changes_user_id"); + + b.Navigation("User"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.UserNameChange", b => + { + b.HasOne("OpenShock.Common.OpenShockDb.User", "User") + .WithMany("NameChanges") + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_user_name_changes_user_id"); + + b.Navigation("User"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.UserOAuthConnection", b => + { + b.HasOne("OpenShock.Common.OpenShockDb.User", "User") + .WithMany("OAuthConnections") + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_user_oauth_connections_user_id"); + + b.Navigation("User"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.UserPasswordReset", b => + { + b.HasOne("OpenShock.Common.OpenShockDb.User", "User") + .WithMany("PasswordResets") + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_user_password_resets_user_id"); + + b.Navigation("User"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.UserShare", b => + { + b.HasOne("OpenShock.Common.OpenShockDb.User", "SharedWithUser") + .WithMany("IncomingUserShares") + .HasForeignKey("SharedWithUserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_user_shares_shared_with_user_id"); + + b.HasOne("OpenShock.Common.OpenShockDb.Shocker", "Shocker") + .WithMany("UserShares") + .HasForeignKey("ShockerId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_user_shares_shocker_id"); + + b.Navigation("SharedWithUser"); + + b.Navigation("Shocker"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.UserShareInvite", b => + { + b.HasOne("OpenShock.Common.OpenShockDb.User", "Owner") + .WithMany("OutgoingUserShareInvites") + .HasForeignKey("OwnerId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_user_share_invites_owner_id"); + + b.HasOne("OpenShock.Common.OpenShockDb.User", "RecipientUser") + .WithMany("IncomingUserShareInvites") + .HasForeignKey("RecipientUserId") + .OnDelete(DeleteBehavior.Cascade) + .HasConstraintName("fk_user_share_invites_recipient_user_id"); + + b.Navigation("Owner"); + + b.Navigation("RecipientUser"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.UserShareInviteShocker", b => + { + b.HasOne("OpenShock.Common.OpenShockDb.UserShareInvite", "Invite") + .WithMany("ShockerMappings") + .HasForeignKey("InviteId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_user_share_invite_shockers_invite_id"); + + b.HasOne("OpenShock.Common.OpenShockDb.Shocker", "Shocker") + .WithMany("UserShareInviteShockerMappings") + .HasForeignKey("ShockerId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_user_share_invite_shockers_shocker_id"); + + b.Navigation("Invite"); + + b.Navigation("Shocker"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.AutomationToken", b => + { + b.Navigation("CreatedUsers"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.Device", b => + { + b.Navigation("OtaUpdates"); + + b.Navigation("Shockers"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.PublicShare", b => + { + b.Navigation("ShockerMappings"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.Shocker", b => + { + b.Navigation("PublicShareMappings"); + + b.Navigation("ShockerShareCodes"); + + b.Navigation("UserShareInviteShockerMappings"); + + b.Navigation("UserShares"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.User", b => + { + b.Navigation("ActorAuditLogs"); + + b.Navigation("ApiTokens"); + + b.Navigation("AuditLogs"); + + b.Navigation("Devices"); + + b.Navigation("EmailChanges"); + + b.Navigation("IncomingUserShareInvites"); + + b.Navigation("IncomingUserShares"); + + b.Navigation("NameChanges"); + + b.Navigation("OAuthConnections"); + + b.Navigation("OutgoingUserShareInvites"); + + b.Navigation("OwnedPublicShares"); + + b.Navigation("PasswordResets"); + + b.Navigation("ReportedApiTokens"); + + b.Navigation("UserActivationRequest"); + + b.Navigation("UserDeactivation"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.UserShareInvite", b => + { + b.Navigation("ShockerMappings"); + }); +#pragma warning restore 612, 618 + } +} diff --git a/Common/Migrations/20261120201415_MaterializeControlStats.cs b/Common/Migrations/20261120201415_MaterializeControlStats.cs new file mode 100644 index 000000000..0056a8562 --- /dev/null +++ b/Common/Migrations/20261120201415_MaterializeControlStats.cs @@ -0,0 +1,24 @@ +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace OpenShock.Common.Migrations; + +/// +/// Separate from as it can't run in a transaction. +/// +public partial class _20261120201415_MaterializeControlStats : Migration +{ + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.Sql( + "CALL refresh_continuous_aggregate('control_stats', NULL, now() - INTERVAL '1 hour');", + suppressTransaction: true); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + } +} diff --git a/Common/Migrations/OpenShockContextModelSnapshot.cs b/Common/Migrations/OpenShockContextModelSnapshot.cs index b7777ff19..8383175ec 100644 --- a/Common/Migrations/OpenShockContextModelSnapshot.cs +++ b/Common/Migrations/OpenShockContextModelSnapshot.cs @@ -18,7 +18,7 @@ partial class OpenShockContextModelSnapshot : ModelSnapshot // If you encounter a merge conflict in the line below, it means you need to // discard one of the migration branches and recreate its migrations on top of // the other branch. See https://aka.ms/efcore-docs-migrations-conflicts for more info. - public override string LastMigrationId => "20261113184758_AddStringLengthLimits"; + public override string LastMigrationId => "20261120201415_MaterializeControlStats"; protected override void BuildModel(ModelBuilder modelBuilder) { @@ -41,6 +41,7 @@ protected override void BuildModel(ModelBuilder modelBuilder) NpgsqlModelBuilderExtensions.HasPostgresEnum(modelBuilder, "permission_type", new[] { "shockers.use", "shockers.edit", "shockers.pause", "devices.edit", "devices.auth", "usershares.edit", "usershares.pause", "publicshares.pause", "publicshares.edit" }); NpgsqlModelBuilderExtensions.HasPostgresEnum(modelBuilder, "role_type", new[] { "support", "staff", "admin", "system" }); NpgsqlModelBuilderExtensions.HasPostgresEnum(modelBuilder, "shocker_model_type", new[] { "cai_xianlin", "petrainer", "petrainer_998dr", "wellturn_t330" }); + NpgsqlModelBuilderExtensions.HasPostgresExtension(modelBuilder, "timescaledb"); NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); modelBuilder.Entity("Microsoft.AspNetCore.DataProtection.EntityFrameworkCore.DataProtectionKey", b => @@ -400,10 +401,6 @@ protected override void BuildModel(ModelBuilder modelBuilder) modelBuilder.Entity("OpenShock.Common.OpenShockDb.ControlStat", b => { - b.Property("Id") - .HasColumnType("uuid") - .HasColumnName("id"); - b.Property("Bucket") .HasColumnType("timestamp with time zone") .HasColumnName("bucket"); @@ -436,7 +433,7 @@ protected override void BuildModel(ModelBuilder modelBuilder) .HasColumnType("uuid") .HasColumnName("owner_user_id"); - b.Property("ShockerId") + b.Property("ShockerId") .HasColumnType("uuid") .HasColumnName("shocker_id"); @@ -444,65 +441,9 @@ protected override void BuildModel(ModelBuilder modelBuilder) .HasColumnType("control_type") .HasColumnName("type"); - b.Property("UpdatedAt") - .HasColumnType("timestamp with time zone") - .HasColumnName("updated_at"); - - b.HasKey("Id") - .HasName("control_stats_pkey"); - - b.HasIndex("ShockerId"); - - b.HasIndex("UpdatedAt"); - - b.HasIndex("ControllerUserId", "Bucket"); - - b.HasIndex("OwnerUserId", "Bucket"); - - var index = b.HasIndex("OwnerUserId", "ShockerId", "ControllerUserId", "Bucket", "Type", "Live") - .IsUnique() - .HasDatabaseName("ix_control_stats_upsert_key") - .HasFilter("shocker_id IS NOT NULL"); - - NpgsqlIndexBuilderExtensions.AreNullsDistinct(index, false); - - b.ToTable("control_stats"); - }); - - modelBuilder.Entity("OpenShock.Common.OpenShockDb.ControlStatBatch", b => - { - b.Property("Id") - .HasColumnType("uuid") - .HasColumnName("id"); - - b.Property("AppliedAt") - .HasColumnType("timestamp with time zone") - .HasColumnName("applied_at"); - - b.HasKey("Id") - .HasName("control_stat_batches_pkey"); - - b.HasIndex("AppliedAt"); - - b.ToTable("control_stat_batches"); - }); - - modelBuilder.Entity("OpenShock.Common.OpenShockDb.ControlStatRemoval", b => - { - b.Property("UserId") - .HasColumnType("uuid") - .HasColumnName("user_id"); - - b.Property("RemovedAt") - .HasColumnType("timestamp with time zone") - .HasColumnName("removed_at"); - - b.HasKey("UserId") - .HasName("control_stat_removals_pkey"); - - b.HasIndex("RemovedAt"); + b.ToTable((string)null); - b.ToTable("control_stat_removals"); + b.ToView("control_stats", (string)null); }); modelBuilder.Entity("OpenShock.Common.OpenShockDb.Device", b => @@ -876,16 +817,16 @@ protected override void BuildModel(ModelBuilder modelBuilder) .HasColumnType("uuid") .HasColumnName("id"); - b.Property("ControlledByUserId") - .HasColumnType("uuid") - .HasColumnName("controlled_by_user_id"); - b.Property("CreatedAt") .ValueGeneratedOnAdd() .HasColumnType("timestamp with time zone") .HasColumnName("created_at") .HasDefaultValueSql("CURRENT_TIMESTAMP"); + b.Property("ControlledByUserId") + .HasColumnType("uuid") + .HasColumnName("controlled_by_user_id"); + b.Property("CustomName") .HasMaxLength(64) .HasColumnType("character varying(64)") @@ -895,6 +836,12 @@ protected override void BuildModel(ModelBuilder modelBuilder) .HasColumnType("bigint") .HasColumnName("duration"); + b.Property("InsertedAt") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasColumnName("inserted_at") + .HasDefaultValueSql("now()"); + b.Property("Intensity") .HasColumnType("smallint") .HasColumnName("intensity"); @@ -905,6 +852,14 @@ protected override void BuildModel(ModelBuilder modelBuilder) .HasDefaultValue(false) .HasColumnName("live_control"); + b.Property("MaxIntensity") + .HasColumnType("smallint") + .HasColumnName("max_intensity"); + + b.Property("OwnerUserId") + .HasColumnType("uuid") + .HasColumnName("owner_user_id"); + b.Property("ShockerId") .HasColumnType("uuid") .HasColumnName("shocker_id"); @@ -913,12 +868,14 @@ protected override void BuildModel(ModelBuilder modelBuilder) .HasColumnType("control_type") .HasColumnName("type"); - b.HasKey("Id") + b.HasKey("Id", "CreatedAt") .HasName("shocker_control_logs_pkey"); b.HasIndex("ControlledByUserId"); - b.HasIndex("ShockerId"); + b.HasIndex("OwnerUserId", "CreatedAt"); + + b.HasIndex("ShockerId", "CreatedAt"); b.ToTable("shocker_control_logs"); }); @@ -1552,26 +1509,6 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Navigation("ReportedByUser"); }); - modelBuilder.Entity("OpenShock.Common.OpenShockDb.ControlStat", b => - { - b.HasOne("OpenShock.Common.OpenShockDb.User", "Owner") - .WithMany("ControlStats") - .HasForeignKey("OwnerUserId") - .OnDelete(DeleteBehavior.Cascade) - .IsRequired() - .HasConstraintName("fk_control_stats_owner_user_id"); - - b.HasOne("OpenShock.Common.OpenShockDb.Shocker", "Shocker") - .WithMany("ControlStats") - .HasForeignKey("ShockerId") - .OnDelete(DeleteBehavior.SetNull) - .HasConstraintName("fk_control_stats_shocker_id"); - - b.Navigation("Owner"); - - b.Navigation("Shocker"); - }); - modelBuilder.Entity("OpenShock.Common.OpenShockDb.Device", b => { b.HasOne("OpenShock.Common.OpenShockDb.User", "Owner") @@ -1641,26 +1578,6 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Navigation("Device"); }); - modelBuilder.Entity("OpenShock.Common.OpenShockDb.ShockerControlLog", b => - { - b.HasOne("OpenShock.Common.OpenShockDb.User", "ControlledByUser") - .WithMany("ShockerControlLogs") - .HasForeignKey("ControlledByUserId") - .OnDelete(DeleteBehavior.Cascade) - .HasConstraintName("fk_shocker_control_logs_controlled_by_user_id"); - - b.HasOne("OpenShock.Common.OpenShockDb.Shocker", "Shocker") - .WithMany("ShockerControlLogs") - .HasForeignKey("ShockerId") - .OnDelete(DeleteBehavior.Cascade) - .IsRequired() - .HasConstraintName("fk_shocker_control_logs_shocker_id"); - - b.Navigation("ControlledByUser"); - - b.Navigation("Shocker"); - }); - modelBuilder.Entity("OpenShock.Common.OpenShockDb.ShockerShareCode", b => { b.HasOne("OpenShock.Common.OpenShockDb.Shocker", "Shocker") @@ -1866,12 +1783,8 @@ protected override void BuildModel(ModelBuilder modelBuilder) modelBuilder.Entity("OpenShock.Common.OpenShockDb.Shocker", b => { - b.Navigation("ControlStats"); - b.Navigation("PublicShareMappings"); - b.Navigation("ShockerControlLogs"); - b.Navigation("ShockerShareCodes"); b.Navigation("UserShareInviteShockerMappings"); @@ -1887,8 +1800,6 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Navigation("AuditLogs"); - b.Navigation("ControlStats"); - b.Navigation("Devices"); b.Navigation("EmailChanges"); @@ -1909,8 +1820,6 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Navigation("ReportedApiTokens"); - b.Navigation("ShockerControlLogs"); - b.Navigation("UserActivationRequest"); b.Navigation("UserDeactivation"); diff --git a/Common/OpenShockDb/Models/ControlStat.cs b/Common/OpenShockDb/Models/ControlStat.cs index 5ee5cdb62..1e13bfa28 100644 --- a/Common/OpenShockDb/Models/ControlStat.cs +++ b/Common/OpenShockDb/Models/ControlStat.cs @@ -1,34 +1,22 @@ namespace OpenShock.Common.OpenShockDb; /// -/// Rollup of shocker control activity in 15 minute buckets. Unlike these rows are -/// never trimmed, so they back lifetime statistics. 15 minutes since every time zone's UTC offset is a multiple of it, -/// so local days always start on a bucket boundary (an hour would split them in e.g. Asia/Kolkata, UTC+5:30). One row -/// per owner, shocker, controller, bucket, type and live flag. Live control -/// activity is only counted once the gateway has written it to the hub's websocket. +/// The control_stats continuous aggregate, s in 15 minute buckets. Outlives the +/// logs, so it backs lifetime statistics. Stop controls are not counted. /// public sealed class ControlStat { - public required Guid Id { get; init; } + public required DateTime Bucket { get; init; } - /// The user who owns the shocker (the one being controlled). public required Guid OwnerUserId { get; init; } - /// Null once the shocker has been deleted. - public Guid? ShockerId { get; init; } - - /// - /// The user who sent the control. Null for public shares / guests. Deliberately not a foreign key, so stats - /// of deleted users keep their (now dangling) id instead of colliding with the guest bucket. - /// - public Guid? ControllerUserId { get; init; } + public required Guid ShockerId { get; init; } - /// Start of the 15-minute UTC bucket this row aggregates, see . - public required DateTime Bucket { get; init; } + /// Null for public shares / guests. + public required Guid? ControllerUserId { get; init; } public required ControlType Type { get; init; } - /// Whether this row aggregates live control (gateway) activity. public required bool Live { get; init; } /// Number of controls, for live control the number of continuous bursts. @@ -40,11 +28,4 @@ public sealed class ControlStat public required long IntensityMs { get; init; } public required byte MaxIntensity { get; init; } - - /// When this row was last added to, change detection for integrations keys off this. - public required DateTime UpdatedAt { get; init; } - - // Navigations - public User Owner { get; init; } = null!; - public Shocker? Shocker { get; init; } } diff --git a/Common/OpenShockDb/Models/ControlStatBatch.cs b/Common/OpenShockDb/Models/ControlStatBatch.cs deleted file mode 100644 index a7ca6e68c..000000000 --- a/Common/OpenShockDb/Models/ControlStatBatch.cs +++ /dev/null @@ -1,13 +0,0 @@ -namespace OpenShock.Common.OpenShockDb; - -/// -/// Marks a buffered batch of deltas as applied, written in the same transaction as the -/// deltas. A batch retried after its commit went through but the acknowledgement got lost is then skipped instead of -/// being counted twice. -/// -public sealed class ControlStatBatch -{ - public required Guid Id { get; init; } - - public required DateTime AppliedAt { get; init; } -} diff --git a/Common/OpenShockDb/Models/ControlStatRemoval.cs b/Common/OpenShockDb/Models/ControlStatRemoval.cs deleted file mode 100644 index fdba3a769..000000000 --- a/Common/OpenShockDb/Models/ControlStatRemoval.cs +++ /dev/null @@ -1,14 +0,0 @@ -namespace OpenShock.Common.OpenShockDb; - -/// -/// When rows involving a user, as owner or controller, were last deleted, e.g. by the owner -/// account being deleted. Deletions leave no row behind to carry an , so this is -/// how integrations learn that a user's totals went down. Written by a database trigger, one row per user. -/// -public sealed class ControlStatRemoval -{ - /// Deliberately not a foreign key, the user may be the one that got deleted. - public required Guid UserId { get; init; } - - public required DateTime RemovedAt { get; init; } -} diff --git a/Common/OpenShockDb/Models/Shocker.cs b/Common/OpenShockDb/Models/Shocker.cs index a17d2c9be..3c6cc8576 100644 --- a/Common/OpenShockDb/Models/Shocker.cs +++ b/Common/OpenShockDb/Models/Shocker.cs @@ -19,8 +19,6 @@ public sealed class Shocker // Navigations public Device Device { get; init; } = null!; public ICollection UserShareInviteShockerMappings { get; } = []; - public ICollection ShockerControlLogs { get; } = []; - public ICollection ControlStats { get; } = []; public ICollection ShockerShareCodes { get; } = []; public ICollection UserShares { get; } = []; public ICollection PublicShareMappings { get; } = []; diff --git a/Common/OpenShockDb/Models/ShockerControlLog.cs b/Common/OpenShockDb/Models/ShockerControlLog.cs index cb8553252..965d9d825 100644 --- a/Common/OpenShockDb/Models/ShockerControlLog.cs +++ b/Common/OpenShockDb/Models/ShockerControlLog.cs @@ -1,26 +1,34 @@ namespace OpenShock.Common.OpenShockDb; +/// +/// No foreign keys to shockers and controllers, so their stats survive them being deleted. +/// public sealed class ShockerControlLog { - public Guid Id { get; init; } + public required Guid Id { get; init; } + + public required Guid OwnerUserId { get; init; } public required Guid ShockerId { get; init; } public required Guid? ControlledByUserId { get; init; } + /// For live control the burst's average intensity, weighted by duration. public required byte Intensity { get; init; } + public required byte MaxIntensity { get; init; } + public required uint Duration { get; init; } - + public required ControlType Type { get; init; } public required string? CustomName { get; init; } public bool LiveControl { get; init; } + /// For live control when the burst started. public required DateTime CreatedAt { get; init; } - // Navigations - public Shocker Shocker { get; init; } = null!; - public User? ControlledByUser { get; init; } + /// Set by the database. Live control bursts are only written once they end. + public DateTime InsertedAt { get; init; } } diff --git a/Common/OpenShockDb/Models/User.cs b/Common/OpenShockDb/Models/User.cs index 9b5cbbca5..2e3fad5d2 100644 --- a/Common/OpenShockDb/Models/User.cs +++ b/Common/OpenShockDb/Models/User.cs @@ -42,8 +42,6 @@ public sealed class User public ICollection OutgoingUserShareInvites { get; } = []; public ICollection IncomingUserShareInvites { get; } = []; public ICollection OwnedPublicShares { get; } = []; - public ICollection ShockerControlLogs { get; } = []; - public ICollection ControlStats { get; } = []; public ICollection NameChanges { get; } = []; public ICollection EmailChanges { get; } = []; public ICollection PasswordResets { get; } = []; diff --git a/Common/OpenShockDb/OpenShockContext.cs b/Common/OpenShockDb/OpenShockContext.cs index b28813927..5760ce1c4 100644 --- a/Common/OpenShockDb/OpenShockContext.cs +++ b/Common/OpenShockDb/OpenShockContext.cs @@ -85,10 +85,6 @@ public static void ConfigureOptionsBuilder(DbContextOptionsBuilder optionsBuilde public DbSet ControlStats { get; set; } - public DbSet ControlStatBatches { get; set; } - - public DbSet ControlStatRemovals { get; set; } - public DbSet UserShares { get; set; } public DbSet ShockerShareCodes { get; set; } @@ -139,6 +135,7 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) { modelBuilder .RegisterPgEnums() + .HasPostgresExtension("timescaledb") // Community edition .HasCollation("public", "ndcoll", "und-u-ks-level2", "icu", false); // Add case-insensitive, accent-sensitive comparison collation modelBuilder.Entity(entity => @@ -423,11 +420,14 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) modelBuilder.Entity(entity => { - entity.HasKey(e => e.Id).HasName("shocker_control_logs_pkey"); + // A hypertable's unique indexes must include the column it is partitioned by + entity.HasKey(e => new { e.Id, e.CreatedAt }).HasName("shocker_control_logs_pkey"); entity.ToTable("shocker_control_logs"); - entity.HasIndex(e => e.ShockerId); + entity.HasIndex(e => new { e.ShockerId, e.CreatedAt }); + entity.HasIndex(e => new { e.OwnerUserId, e.CreatedAt }); + entity.HasIndex(e => e.ControlledByUserId); entity.Property(e => e.Id) .ValueGeneratedOnAdd() @@ -441,93 +441,32 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) .HasColumnName("custom_name"); entity.Property(e => e.Duration).HasColumnName("duration"); entity.Property(e => e.Intensity).HasColumnName("intensity"); + entity.Property(e => e.MaxIntensity).HasColumnName("max_intensity"); entity.Property(e => e.LiveControl) .HasDefaultValue(false) .HasColumnName("live_control"); + entity.Property(e => e.OwnerUserId).HasColumnName("owner_user_id"); entity.Property(e => e.ShockerId).HasColumnName("shocker_id"); entity.Property(e => e.Type).HasColumnType("control_type").HasColumnName("type"); - - entity.HasOne(d => d.ControlledByUser).WithMany(p => p.ShockerControlLogs) - .HasForeignKey(d => d.ControlledByUserId) - .OnDelete(DeleteBehavior.Cascade) - .HasConstraintName("fk_shocker_control_logs_controlled_by_user_id"); - - entity.HasOne(d => d.Shocker).WithMany(p => p.ShockerControlLogs) - .HasForeignKey(d => d.ShockerId) - .HasConstraintName("fk_shocker_control_logs_shocker_id"); + entity.Property(e => e.InsertedAt) + .HasDefaultValueSql("now()") + .HasColumnName("inserted_at"); }); modelBuilder.Entity(entity => { - entity.HasKey(e => e.Id).HasName("control_stats_pkey"); - - entity.ToTable("control_stats"); - - // Upsert target. Rows orphaned by a shocker delete (shocker_id SET NULL) fall outside the filter, so - // they may duplicate each other without violating it; readers sum them anyway. - entity.HasIndex(e => new { e.OwnerUserId, e.ShockerId, e.ControllerUserId, e.Bucket, e.Type, e.Live }) - .IsUnique() - .AreNullsDistinct(false) - .HasFilter("shocker_id IS NOT NULL") - .HasDatabaseName("ix_control_stats_upsert_key"); - entity.HasIndex(e => new { e.OwnerUserId, e.Bucket }); - entity.HasIndex(e => new { e.ControllerUserId, e.Bucket }); - entity.HasIndex(e => e.ShockerId); - entity.HasIndex(e => e.UpdatedAt); + entity.HasNoKey().ToView("control_stats"); - entity.Property(e => e.Id) - .ValueGeneratedNever() - .HasColumnName("id"); + entity.Property(e => e.Bucket).HasColumnName("bucket"); entity.Property(e => e.OwnerUserId).HasColumnName("owner_user_id"); entity.Property(e => e.ShockerId).HasColumnName("shocker_id"); entity.Property(e => e.ControllerUserId).HasColumnName("controller_user_id"); - entity.Property(e => e.Bucket).HasColumnName("bucket"); entity.Property(e => e.Type).HasColumnType("control_type").HasColumnName("type"); entity.Property(e => e.Live).HasColumnName("live"); entity.Property(e => e.Count).HasColumnName("count"); entity.Property(e => e.DurationMs).HasColumnName("duration_ms"); entity.Property(e => e.IntensityMs).HasColumnName("intensity_ms"); entity.Property(e => e.MaxIntensity).HasColumnName("max_intensity"); - entity.Property(e => e.UpdatedAt).HasColumnName("updated_at"); - - entity.HasOne(e => e.Owner).WithMany(u => u.ControlStats) - .HasForeignKey(e => e.OwnerUserId) - .OnDelete(DeleteBehavior.Cascade) - .HasConstraintName("fk_control_stats_owner_user_id"); - - entity.HasOne(e => e.Shocker).WithMany(s => s.ControlStats) - .HasForeignKey(e => e.ShockerId) - .OnDelete(DeleteBehavior.SetNull) - .HasConstraintName("fk_control_stats_shocker_id"); - }); - - modelBuilder.Entity(entity => - { - entity.HasKey(e => e.Id).HasName("control_stat_batches_pkey"); - - entity.ToTable("control_stat_batches"); - - entity.HasIndex(e => e.AppliedAt); - - entity.Property(e => e.Id) - .ValueGeneratedNever() - .HasColumnName("id"); - entity.Property(e => e.AppliedAt).HasColumnName("applied_at"); - }); - - // Filled by the control_stats_record_removals trigger, see the AddControlStats migration - modelBuilder.Entity(entity => - { - entity.HasKey(e => e.UserId).HasName("control_stat_removals_pkey"); - - entity.ToTable("control_stat_removals"); - - entity.HasIndex(e => e.RemovedAt); - - entity.Property(e => e.UserId) - .ValueGeneratedNever() - .HasColumnName("user_id"); - entity.Property(e => e.RemovedAt).HasColumnName("removed_at"); }); modelBuilder.Entity(entity => diff --git a/Common/OpenShockDb/Queries/ControlStatQueries.cs b/Common/OpenShockDb/Queries/ControlStatQueries.cs deleted file mode 100644 index e2cf9ed5c..000000000 --- a/Common/OpenShockDb/Queries/ControlStatQueries.cs +++ /dev/null @@ -1,151 +0,0 @@ -using Microsoft.EntityFrameworkCore; - -namespace OpenShock.Common.OpenShockDb; - -/// -/// A pending increment for one bucket. -/// -public readonly record struct ControlStatDelta( - Guid OwnerUserId, - Guid ShockerId, - Guid? ControllerUserId, - DateTime Bucket, - ControlType Type, - bool Live, - long Count, - long DurationMs, - long IntensityMs, - byte MaxIntensity) -{ - /// Granularity of , every time zone's UTC offset is a multiple of it. - public static readonly TimeSpan BucketSize = TimeSpan.FromMinutes(15); - - /// Truncates a UTC timestamp to the start of its bucket. - public static DateTime ToBucket(DateTime utc) => - new(utc.Ticks - utc.Ticks % BucketSize.Ticks, DateTimeKind.Utc); - - /// Combines two deltas for the same bucket. - public ControlStatDelta Merge(ControlStatDelta other) => this with - { - Count = Count + other.Count, - DurationMs = DurationMs + other.DurationMs, - IntensityMs = IntensityMs + other.IntensityMs, - MaxIntensity = Math.Max(MaxIntensity, other.MaxIntensity) - }; -} - -/// -/// Write path for the control_stats rollup table. -/// -public static class ControlStatQueries -{ - /// - /// How long after its creation a buffered batch may still be applied, older ones are refused. This is what makes - /// pruning markers safe: once no retry of a batch can be applied anymore, its marker - /// has nothing left to deduplicate. - /// - public static readonly TimeSpan BatchReplayLifetime = TimeSpan.FromDays(30); - - /// - /// Markers outlive by this much, covering clock skew between the services that - /// create batches and the database that stamps the markers. - /// - private static readonly TimeSpan BatchMarkerMargin = TimeSpan.FromDays(1); - - /// - /// Adds the given deltas onto their buckets in a single statement, creating buckets as needed. Deltas of shockers - /// deleted in the meantime are kept with no shocker, like rows orphaned by the delete. Deltas of owners that no - /// longer exist are dropped, their stats are gone with them. - /// - public static Task UpsertControlStatsAsync(this OpenShockContext db, IEnumerable deltas, - CancellationToken cancellationToken = default) - { - // One row per conflict key (ON CONFLICT can't touch a row twice in one statement), in a stable order so - // concurrent upserts lock rows in the same order and can't deadlock. - var merged = deltas - .Where(d => d.Type != ControlType.Stop) - .Select(d => d with { Bucket = ControlStatDelta.ToBucket(d.Bucket) }) - .GroupBy(d => (d.ShockerId, d.ControllerUserId, d.Bucket, d.Type, d.Live)) - .Select(g => g.Aggregate((a, b) => a.Merge(b))) - .OrderBy(d => d.ShockerId) - .ThenBy(d => d.ControllerUserId) - .ThenBy(d => d.Bucket) - .ThenBy(d => d.Type) - .ThenBy(d => d.Live) - .ToArray(); - - if (merged.Length == 0) return Task.FromResult(0); - - var ids = merged.Select(_ => Guid.CreateVersion7()).ToArray(); - var ownerIds = merged.Select(d => d.OwnerUserId).ToArray(); - var shockerIds = merged.Select(d => d.ShockerId).ToArray(); - var controllerIds = merged.Select(d => d.ControllerUserId).ToArray(); - var buckets = merged.Select(d => d.Bucket).ToArray(); - var types = merged.Select(d => d.Type).ToArray(); - var lives = merged.Select(d => d.Live).ToArray(); - var counts = merged.Select(d => d.Count).ToArray(); - var durations = merged.Select(d => d.DurationMs).ToArray(); - var intensityMs = merged.Select(d => d.IntensityMs).ToArray(); - var maxIntensities = merged.Select(d => (short)d.MaxIntensity).ToArray(); - - // A deleted shocker resolves to a null shocker_id, which falls outside the upsert key's filter and is - // inserted as its own row instead of violating the foreign key. - return db.Database.ExecuteSqlAsync( - $""" - INSERT INTO control_stats (id, owner_user_id, shocker_id, controller_user_id, bucket, type, live, count, duration_ms, intensity_ms, max_intensity, updated_at) - SELECT v.id, v.owner_user_id, s.id, v.controller_user_id, v.bucket, v.type, v.live, v.count, v.duration_ms, v.intensity_ms, v.max_intensity, now() - FROM unnest({ids}, {ownerIds}, {shockerIds}, {controllerIds}, {buckets}, {types}, {lives}, {counts}, {durations}, {intensityMs}, {maxIntensities}) - AS v(id, owner_user_id, shocker_id, controller_user_id, bucket, type, live, count, duration_ms, intensity_ms, max_intensity) - JOIN users u ON u.id = v.owner_user_id - LEFT JOIN shockers s ON s.id = v.shocker_id - ON CONFLICT (owner_user_id, shocker_id, controller_user_id, bucket, type, live) WHERE shocker_id IS NOT NULL - DO UPDATE SET - count = control_stats.count + excluded.count, - duration_ms = control_stats.duration_ms + excluded.duration_ms, - intensity_ms = control_stats.intensity_ms + excluded.intensity_ms, - max_intensity = GREATEST(control_stats.max_intensity, excluded.max_intensity), - updated_at = excluded.updated_at - """, cancellationToken); - } - - /// - /// Applies a buffered batch of deltas exactly once. The batch is marked applied in the same transaction, so - /// retrying it with the same after an unknown commit outcome can't count it twice. - /// Batches older than are refused, as their marker may have been pruned. - /// - public static async Task ApplyControlStatBatchAsync(this OpenShockContext db, Guid batchId, - DateTime createdAt, IEnumerable deltas, CancellationToken cancellationToken = default) - { - if (DateTime.UtcNow - createdAt > BatchReplayLifetime) return ControlStatBatchResult.Expired; - - await using var transaction = await db.Database.BeginTransactionAsync(cancellationToken); - - var marked = await db.Database.ExecuteSqlAsync( - $"INSERT INTO control_stat_batches (id, applied_at) VALUES ({batchId}, now()) ON CONFLICT DO NOTHING", - cancellationToken); - if (marked == 0) return ControlStatBatchResult.AlreadyApplied; - - await db.UpsertControlStatsAsync(deltas, cancellationToken); - - // Only markers of batches that can no longer be applied, see BatchReplayLifetime - var prunableBefore = DateTime.UtcNow - BatchReplayLifetime - BatchMarkerMargin; - await db.ControlStatBatches.Where(x => x.AppliedAt < prunableBefore).ExecuteDeleteAsync(cancellationToken); - - await transaction.CommitAsync(cancellationToken); - return ControlStatBatchResult.Applied; - } -} - -/// -/// Outcome of . -/// -public enum ControlStatBatchResult -{ - Applied, - - /// A previous attempt with the same batch id was committed. - AlreadyApplied, - - /// Older than , not applied. - Expired -} diff --git a/Common/OpenShockDb/Queries/ShockerControlLogQueries.cs b/Common/OpenShockDb/Queries/ShockerControlLogQueries.cs index 04080dfa6..3a93f2a80 100644 --- a/Common/OpenShockDb/Queries/ShockerControlLogQueries.cs +++ b/Common/OpenShockDb/Queries/ShockerControlLogQueries.cs @@ -2,39 +2,36 @@ namespace OpenShock.Common.OpenShockDb; -/// -/// Reusable commands over the shocker_control_logs table. Kept here (rather than inline in the -/// Cron cleanup job) so the raw SQL is a single source of truth that integration tests can execute -/// directly - a table or column rename then breaks the test rather than surfacing only in production. -/// public static class ShockerControlLogQueries { /// - /// Retention trim: per owning user, keeps the newest control logs (by - /// created_at) and deletes the rest. Ownership is resolved through - /// shocker -> device -> owner, so the cap is per user, not per shocker or device. Runs as a - /// single set-based statement (a window function ranks each user's logs newest-first; anything past the - /// cap is deleted). Returns the number of rows deleted. + /// Skips logs whose id was already written, so retrying a batch can't count it twice, and logs of deleted owners. /// - public static Task DeleteControlLogsBeyondPerUserLimitAsync(this OpenShockContext db, int maxPerUser, + public static Task InsertControlLogsAsync(this OpenShockContext db, IReadOnlyCollection logs, CancellationToken cancellationToken = default) { - if (maxPerUser < 0) throw new ArgumentOutOfRangeException(nameof(maxPerUser)); + if (logs.Count == 0) return Task.FromResult(0); + + var ids = logs.Select(x => x.Id).ToArray(); + var ownerIds = logs.Select(x => x.OwnerUserId).ToArray(); + var shockerIds = logs.Select(x => x.ShockerId).ToArray(); + var controllerIds = logs.Select(x => x.ControlledByUserId).ToArray(); + var intensities = logs.Select(x => (short)x.Intensity).ToArray(); + var maxIntensities = logs.Select(x => (short)x.MaxIntensity).ToArray(); + var durations = logs.Select(x => (long)x.Duration).ToArray(); + var types = logs.Select(x => x.Type).ToArray(); + var customNames = logs.Select(x => x.CustomName).ToArray(); + var live = logs.Select(x => x.LiveControl).ToArray(); + var createdAts = logs.Select(x => x.CreatedAt).ToArray(); return db.Database.ExecuteSqlAsync( $""" - WITH ranked_logs AS ( - SELECT - l.id, - ROW_NUMBER() OVER (PARTITION BY d.owner_id ORDER BY l.created_at DESC) AS rn - FROM shocker_control_logs l - JOIN shockers s ON s.id = l.shocker_id - JOIN devices d ON d.id = s.device_id - ) - DELETE FROM shocker_control_logs l - USING ranked_logs rl - WHERE l.id = rl.id - AND rl.rn > {maxPerUser} + INSERT INTO shocker_control_logs (id, owner_user_id, shocker_id, controlled_by_user_id, intensity, max_intensity, duration, type, custom_name, live_control, created_at) + SELECT v.* + FROM unnest({ids}, {ownerIds}, {shockerIds}, {controllerIds}, {intensities}, {maxIntensities}, {durations}, {types}, {customNames}, {live}, {createdAts}) + AS v(id, owner_user_id, shocker_id, controlled_by_user_id, intensity, max_intensity, duration, type, custom_name, live_control, created_at) + WHERE EXISTS (SELECT 1 FROM users u WHERE u.id = v.owner_user_id) + ON CONFLICT DO NOTHING """, cancellationToken); } } diff --git a/Common/Services/BatchUpdate/BatchUpdateService.cs b/Common/Services/BatchUpdate/BatchUpdateService.cs index d361c66c8..38563d06c 100644 --- a/Common/Services/BatchUpdate/BatchUpdateService.cs +++ b/Common/Services/BatchUpdate/BatchUpdateService.cs @@ -1,11 +1,9 @@ -using System.Diagnostics; -using System.Text.Json; +using System.Text.Json; using Microsoft.EntityFrameworkCore; using NRedisStack.RedisStackCommands; using OpenShock.Common.Constants; using OpenShock.Common.OpenShockDb; using OpenShock.Common.Redis; -using OpenShock.Common.Services.ControlStats; using OpenShock.Common.Utils; using StackExchange.Redis; @@ -72,30 +70,13 @@ public sealed class BatchUpdateService : BackgroundService, IBatchUpdateService private readonly ConcurrentUniqueBatchQueue _tokenLastUsed = new(); private readonly ConcurrentUniqueBatchQueue _sessionLastUsed = new(); private readonly ConcurrentCountingBatchQueue<(Guid Id, string TokenHash)> _automationTokenUses = new(); - private readonly ControlStatsAccumulator _controlStats = new(); + private readonly Lock _controlLogsLock = new(); + private List _controlLogs = []; /// - /// Control stat batches not yet applied, oldest first. Only touched by the flush loop, a batch keeps its id - /// across retries so it is applied at most once. + /// Within the host's default 30 second shutdown timeout. /// - private readonly Queue _controlStatBatches = new(); - - /// - /// Redis list of control stat batches an instance couldn't apply before shutting down, any instance applies them. - /// - private const string ParkedControlStatBatchesKey = "control-stats:parked-batches"; - - /// - /// How long shutdown keeps trying to hand unapplied control stats to the database or Redis, within the host's - /// default 30 second shutdown timeout. - /// - private static readonly TimeSpan ShutdownHandOffTimeout = TimeSpan.FromSeconds(20); - - /// - /// How much of shutdown waits on the database, so a stalled write still - /// leaves time to park control stats in Redis. - /// - private static readonly TimeSpan ShutdownDatabaseTimeout = TimeSpan.FromSeconds(15); + private static readonly TimeSpan ShutdownDatabaseTimeout = TimeSpan.FromSeconds(20); public BatchUpdateService(IDbContextFactory dbFactory, ILogger logger, IConnectionMultiplexer connectionMultiplexer) { @@ -151,139 +132,54 @@ await db.AutomationTokens.Where(x => x.Id == automationTokenId && x.TokenHash == } } - private async Task UpdateControlStats() - { - EnqueueRecordedControlStats(); - await ApplyParkedControlStatBatches(); - await ApplyQueuedControlStatBatches(); - } - - private void EnqueueRecordedControlStats() - { - var deltas = _controlStats.DequeueAll(); - if (deltas.Length > 0) - _controlStatBatches.Enqueue(new PendingControlStatBatch(Guid.CreateVersion7(), DateTime.UtcNow, deltas)); - } - - private async Task ApplyQueuedControlStatBatches(CancellationToken cancellationToken = default) - { - // A batch that throws or is cancelled stays queued with its id. If its commit actually went through, the - // retry finds the batch marker and skips it instead of counting it twice. - while (_controlStatBatches.TryPeek(out var batch)) - { - await ApplyControlStatBatch(batch, cancellationToken); - _controlStatBatches.Dequeue(); - } - } - - private async Task ApplyControlStatBatch(PendingControlStatBatch batch, CancellationToken cancellationToken = default) + private async Task InsertControlLogs(CancellationToken cancellationToken = default) { - await using var db = await _dbFactory.CreateDbContextAsync(cancellationToken); - var result = await db.ApplyControlStatBatchAsync(batch.Id, batch.CreatedAt, batch.Deltas, cancellationToken); - if (result == ControlStatBatchResult.Expired) + List logs; + lock (_controlLogsLock) { - _logger.LogError("Dropped control stat batch {BatchId} from {CreatedAt}, it outlived the replay lifetime: {Batch}", - batch.Id, batch.CreatedAt, JsonSerializer.Serialize(batch)); + if (_controlLogs.Count == 0) return; + logs = _controlLogs; + _controlLogs = []; } - } - /// - /// Applies batches an instance parked in Redis on shutdown. Each one is only removed from Redis once applied, so - /// a crash in between leaves it parked. Several instances may apply the same batch concurrently, its marker makes - /// sure only one of them counts it. - /// - private async Task ApplyParkedControlStatBatches() - { - var redis = _connectionMultiplexer.GetDatabase(); - - RedisValue[] parked; try { - parked = await redis.ListRangeAsync(ParkedControlStatBatchesKey, 0, 99); - } - catch (Exception e) - { - // Still parked, the next flush tries again - _logger.LogWarning(e, "Failed to read parked control stat batches"); - return; + await using var db = await _dbFactory.CreateDbContextAsync(cancellationToken); + await db.InsertControlLogsAsync(logs, cancellationToken); } - - foreach (var value in parked) + catch { - PendingControlStatBatch? batch = null; - try - { - batch = JsonSerializer.Deserialize(value.ToString()); - } - catch (JsonException e) + // Retried on the next flush, the insert skips ids already written + lock (_controlLogsLock) { - // Can never be applied, so it must not block the others - _logger.LogError(e, "Discarding malformed parked control stat batch: {Value}", value.ToString()); + logs.AddRange(_controlLogs); + _controlLogs = logs; } - if (batch is not null) await ApplyControlStatBatch(batch); - await redis.ListRemoveAsync(ParkedControlStatBatchesKey, value, 1); + throw; } } - /// - /// Stats can't be rebuilt, so on shutdown whatever is unapplied keeps being offered to the database until - /// is cancelled, and to Redis for another instance, until one takes it. Only if - /// neither does within are the batches written to the log, the last place - /// they can be recovered from. - /// - private async Task HandOffControlStatsOnShutdown(CancellationToken databaseToken) + private async Task InsertControlLogsOnShutdown(CancellationToken cancellationToken) { - EnqueueRecordedControlStats(); - - var started = Stopwatch.GetTimestamp(); - while (_controlStatBatches.Count > 0) + try { - if (!databaseToken.IsCancellationRequested) - { - try - { - await ApplyQueuedControlStatBatches(databaseToken); - return; - } - catch (Exception e) - { - _logger.LogWarning(e, "Failed to apply {Count} control stat batches on shutdown", _controlStatBatches.Count); - } - } - - var remaining = ShutdownHandOffTimeout - Stopwatch.GetElapsedTime(started); - if (remaining <= TimeSpan.Zero) break; - - try - { - var values = _controlStatBatches.Select(b => (RedisValue)JsonSerializer.Serialize(b)).ToArray(); - // Redis calls can't be cancelled, only stopped waiting on. A push landing after we gave up parks - // batches that are also applied or logged here, their markers keep them from being counted twice. - await _connectionMultiplexer.GetDatabase().ListRightPushAsync(ParkedControlStatBatchesKey, values) - .WaitAsync(remaining); - _controlStatBatches.Clear(); - _logger.LogWarning("Parked {Count} unapplied control stat batches in Redis for another instance", values.Length); - return; - } - catch (Exception e) - { - _logger.LogWarning(e, "Failed to park {Count} control stat batches in Redis on shutdown", _controlStatBatches.Count); - } - - remaining = ShutdownHandOffTimeout - Stopwatch.GetElapsedTime(started); - if (remaining <= TimeSpan.Zero) break; - await Task.Delay(remaining < TimeSpan.FromSeconds(1) ? remaining : TimeSpan.FromSeconds(1)); + await InsertControlLogs(cancellationToken); + return; + } + catch (Exception e) + { + _logger.LogWarning(e, "Failed to insert control logs on shutdown"); } - if (_controlStatBatches.Count == 0) return; - - _logger.LogCritical("Lost {Count} control stat batches on shutdown, neither the database nor Redis took them: {Batches}", - _controlStatBatches.Count, JsonSerializer.Serialize(_controlStatBatches)); + lock (_controlLogsLock) + { + _logger.LogCritical("Lost {Count} control logs on shutdown, the database didn't take them: {Logs}", + _controlLogs.Count, JsonSerializer.Serialize(_controlLogs)); + _controlLogs = []; + } } - private sealed record PendingControlStatBatch(Guid Id, DateTime CreatedAt, ControlStatDelta[] Deltas); - private async Task UpdateSessions(CancellationToken cancellationToken = default) { var sessionsToUpdate = new List>(); @@ -314,9 +210,12 @@ public void UpdateAutomationTokenUsed(Guid automationTokenId, string tokenHash) _automationTokenUses.Increment((automationTokenId, tokenHash)); } - public void RecordControlStat(ControlStatDelta delta) + public void InsertControlLog(ShockerControlLog log) { - _controlStats.Add(delta); + lock (_controlLogsLock) + { + _controlLogs.Add(log); + } } public void UpdateSessionLastUsed(string sessionToken, DateTimeOffset lastUsed) @@ -347,15 +246,14 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) } // Drain what was queued since the last tick, otherwise every shutdown silently loses up to Interval worth of - // last used updates. Control stats are handed off alongside instead of flushed, leaving parked batches to - // other instances, and nothing waits on the database past ShutdownDatabaseTimeout. + // updates using var databaseCts = new CancellationTokenSource(ShutdownDatabaseTimeout); await Task.WhenAll( RunUpdates(UpdateTokens(databaseCts.Token), UpdateAutomationTokens(databaseCts.Token), UpdateSessions(databaseCts.Token)), - HandOffControlStatsOnShutdown(databaseCts.Token)); + InsertControlLogsOnShutdown(databaseCts.Token)); } - private Task Flush() => RunUpdates(UpdateTokens(), UpdateAutomationTokens(), UpdateControlStats(), UpdateSessions()); + private Task Flush() => RunUpdates(UpdateTokens(), UpdateAutomationTokens(), InsertControlLogs(), UpdateSessions()); private async Task RunUpdates(params Task[] updates) { diff --git a/Common/Services/BatchUpdate/IBatchUpdateService.cs b/Common/Services/BatchUpdate/IBatchUpdateService.cs index 9a7b513dd..40cbceab3 100644 --- a/Common/Services/BatchUpdate/IBatchUpdateService.cs +++ b/Common/Services/BatchUpdate/IBatchUpdateService.cs @@ -16,10 +16,9 @@ public interface IBatchUpdateService /// Hash of the secret presented; the use is dropped if the token was rotated since public void UpdateAutomationTokenUsed(Guid automationTokenId, string tokenHash); /// - /// Buffer control activity for the control_stats rollup, used by live control so the per-tick hot path - /// never touches the database + /// Buffer a control log for the next flush, written at most once per /// - /// - public void RecordControlStat(ControlStatDelta delta); + /// + public void InsertControlLog(ShockerControlLog log); public void UpdateSessionLastUsed(string sessionToken, DateTimeOffset lastUsed); } diff --git a/Common/Services/ControlSender.cs b/Common/Services/ControlSender.cs index 20c5851d4..124344656 100644 --- a/Common/Services/ControlSender.cs +++ b/Common/Services/ControlSender.cs @@ -109,7 +109,6 @@ private async Task ControlInternal(IReadOnlyList var messagesByDevice = new Dictionary>(); var logsByOwner = new Dictionary>(); - List statDeltas = [with(capacity: controls.Count)]; foreach (var control in controls.DistinctBy(x => x.Id)) { @@ -165,35 +164,21 @@ private async Task ControlInternal(IReadOnlyList _db.ShockerControlLogs.Add(new ShockerControlLog { + Id = Guid.CreateVersion7(), + OwnerUserId = shocker.OwnerId, ShockerId = shocker.ShockerId, ControlledByUserId = controllerUserId, Intensity = control.Intensity, + MaxIntensity = control.Intensity, Duration = control.Duration, Type = control.Type, CustomName = sender.CustomName, CreatedAt = now }); - - statDeltas.Add(new ControlStatDelta( - OwnerUserId: shocker.OwnerId, - ShockerId: shocker.ShockerId, - ControllerUserId: controllerUserId, - Bucket: now, - Type: control.Type, - Live: false, - Count: 1, - DurationMs: control.Duration, - IntensityMs: (long)control.Intensity * control.Duration, - MaxIntensity: control.Intensity)); } - // Save all db changes before continuing, logs and stats commit together - await using (var transaction = await _db.Database.BeginTransactionAsync()) - { - await _db.SaveChangesAsync(); - await _db.UpsertControlStatsAsync(statDeltas); - await transaction.CommitAsync(); - } + // Save all db changes before continuing + await _db.SaveChangesAsync(); // Then send all network events await Task.WhenAll([ diff --git a/Common/Services/ControlStats/ControlStatsAccumulator.cs b/Common/Services/ControlStats/ControlStatsAccumulator.cs deleted file mode 100644 index 665dfe191..000000000 --- a/Common/Services/ControlStats/ControlStatsAccumulator.cs +++ /dev/null @@ -1,39 +0,0 @@ -using OpenShock.Common.OpenShockDb; - -namespace OpenShock.Common.Services.ControlStats; - -/// -/// Thread-safe in-memory buffer of s, merged per bucket so its size is bounded by the -/// number of distinct buckets touched rather than the number of recorded events. -/// -public sealed class ControlStatsAccumulator -{ - private readonly Lock _lock = new(); - private Dictionary<(Guid ShockerId, Guid? ControllerUserId, DateTime Bucket, ControlType Type, bool Live), ControlStatDelta> _pending = []; - - public void Add(ControlStatDelta delta) - { - if (delta.Type == ControlType.Stop) return; - - delta = delta with { Bucket = ControlStatDelta.ToBucket(delta.Bucket) }; - var key = (delta.ShockerId, delta.ControllerUserId, delta.Bucket, delta.Type, delta.Live); - - lock (_lock) - { - _pending[key] = _pending.TryGetValue(key, out var existing) ? existing.Merge(delta) : delta; - } - } - - public ControlStatDelta[] DequeueAll() - { - Dictionary<(Guid, Guid?, DateTime, ControlType, bool), ControlStatDelta> items; - lock (_lock) - { - if (_pending.Count == 0) return []; - items = _pending; - _pending = []; - } - - return items.Values.ToArray(); - } -} diff --git a/Cron.IntegrationTests/Docker/InMemoryDatabase.cs b/Cron.IntegrationTests/Docker/InMemoryDatabase.cs index 34e3d0e7b..68832b27c 100644 --- a/Cron.IntegrationTests/Docker/InMemoryDatabase.cs +++ b/Cron.IntegrationTests/Docker/InMemoryDatabase.cs @@ -14,7 +14,7 @@ public PostgreSqlContainer Container { get { - _container ??= new PostgreSqlBuilder(image: "postgres:latest") + _container ??= new PostgreSqlBuilder(image: "timescale/timescaledb:latest-pg18") // Community edition, the schema needs it .WithNetwork(DockerNetwork.Instance) .WithName($"tunit-cron-postgresql-{Guid.CreateVersion7()}") .WithDatabase("openshock") diff --git a/Cron.IntegrationTests/Tests/ControlLogRetentionTests.cs b/Cron.IntegrationTests/Tests/ControlLogRetentionTests.cs deleted file mode 100644 index 471b6497a..000000000 --- a/Cron.IntegrationTests/Tests/ControlLogRetentionTests.cs +++ /dev/null @@ -1,136 +0,0 @@ -using Microsoft.EntityFrameworkCore; -using Microsoft.Extensions.DependencyInjection; -using OpenShock.Common.OpenShockDb; -using OpenShock.Internal.Common.Utils; - -namespace OpenShock.Cron.IntegrationTests.Tests; - -/// -/// End-to-end coverage for the Cron retention trim against real Postgres. Runs the cleanup job's *actual* -/// statement (, the single -/// source of truth the job uses) over a seeded graph, so the real table/column names, the -/// shocker -> device -> owner join, the per-user window function, and newest-first ordering are exercised. -/// A schema rename breaks this test rather than only the Cron host at runtime. -/// -public sealed class ControlLogRetentionTests -{ - [ClassDataSource(Shared = SharedType.PerTestSession)] - public required CronApplicationFactory Factory { get; init; } - - [Test] - public async Task DeleteControlLogsBeyondPerUserLimit_TrimsPerOwner_KeepsNewest_AndSparesOwnersUnderTheLimit() - { - const int keep = 10; - - // Owner A is over the limit; owner B is under it. A's logs are attributed to two *different* - // controller users (each under the limit on its own) and B's logs to B itself - so the trim is only - // correct if it partitions by the shocker's OWNER (d.owner_id). If it instead partitioned by the - // controller (l.controlled_by_user_id) it would delete nothing (no controller exceeds the limit); if - // it dropped PARTITION BY entirely it would rank globally and wrongly delete B's (older) logs too. - var ownerA = Guid.CreateVersion7(); - var ownerB = Guid.CreateVersion7(); - var controllerC = Guid.CreateVersion7(); - var controllerD = Guid.CreateVersion7(); - - var shockerA = Guid.CreateVersion7(); - var shockerB = Guid.CreateVersion7(); - - var baseTime = DateTime.UtcNow - TimeSpan.FromDays(1); - - // A: 15 logs (5 over). Newest 10 survive, oldest 5 go. Split across controllers C (first 8) and D. - List aOldestToNewest = [with(capacity: 15)]; - // B: 5 logs, all older than A's - a global (unpartitioned) trim would wrongly delete them. - List bIds = [with(capacity: 5)]; - - await using (var db = await Factory.DbContextFactory.CreateDbContextAsync()) - { - db.Users.Add(NewUser(ownerA, "own-a")); - db.Users.Add(NewUser(ownerB, "own-b")); - db.Users.Add(NewUser(controllerC, "ctl-c")); - db.Users.Add(NewUser(controllerD, "ctl-d")); - - AddShocker(db, shockerB, ownerB, rfId: 2000); - AddShocker(db, shockerA, ownerA, rfId: 1000); - - // B's logs sit at minutes 0..4 (oldest overall) so a global newest-10 trim would drop them. - for (var i = 0; i < 5; i++) - { - var logId = Guid.CreateVersion7(); - bIds.Add(logId); - db.ShockerControlLogs.Add(NewLog(logId, shockerB, ownerB, baseTime + TimeSpan.FromMinutes(i))); - } - - // A's logs sit at minutes 100..114 (newest overall), the first 8 by C and the last 7 by D. - for (var i = 0; i < 15; i++) - { - var logId = Guid.CreateVersion7(); - aOldestToNewest.Add(logId); - var controller = i < 8 ? controllerC : controllerD; - db.ShockerControlLogs.Add(NewLog(logId, shockerA, controller, baseTime + TimeSpan.FromMinutes(100 + i))); - } - - await db.SaveChangesAsync(); - } - - // Resolve the pooled OpenShockContext exactly as the Cron cleanup job does (DI-injected) so this - // covers the same registration path that runs in production. - await using var scope = Factory.Services.CreateAsyncScope(); - var db2 = scope.ServiceProvider.GetRequiredService(); - - var deleted = await db2.DeleteControlLogsBeyondPerUserLimitAsync(keep); - - // Only owner A is over the limit, so exactly its oldest 5 are removed. No other integration test - // writes control logs, so the global count equals A's deletion: 5. (Dropping PARTITION BY would - // delete 10; partitioning by controller instead of owner would delete 0.) - await Assert.That(deleted).IsEqualTo(5); - - var survivingA = (await db2.ShockerControlLogs.AsNoTracking() - .Where(l => l.ShockerId == shockerA).Select(l => l.Id).ToListAsync()).ToHashSet(); - var survivingB = await db2.ShockerControlLogs.AsNoTracking() - .Where(l => l.ShockerId == shockerB).Select(l => l.Id).ToListAsync(); - - // A keeps its newest 10; its oldest 5 are gone. - await Assert.That(survivingA.SetEquals(aOldestToNewest.Skip(5).ToHashSet())).IsTrue(); - await Assert.That(aOldestToNewest.Take(5).Any(survivingA.Contains)).IsFalse(); - - // B is under the limit, so every one of its (older) logs survives - a global trim would have deleted them. - await Assert.That(survivingB.Count).IsEqualTo(5); - } - - private static User NewUser(Guid id, string prefix) => new() - { - Id = id, - Name = $"{prefix}{id:N}"[..16], - Email = $"{prefix}-{id:N}@test.org", - SecurityStamp = Guid.CreateVersion7(), - CreatedAt = DateTime.UtcNow, - ActivatedAt = DateTime.UtcNow - }; - - private static void AddShocker(OpenShockContext db, Guid shockerId, Guid ownerId, ushort rfId) - { - var deviceId = Guid.CreateVersion7(); - db.Devices.Add(new Device - { - Id = deviceId, OwnerId = ownerId, Name = "LogTrimHub", - Token = CryptoUtils.RandomString(256), CreatedAt = DateTime.UtcNow - }); - db.Shockers.Add(new Shocker - { - Id = shockerId, Name = "LogTrimShocker", RfId = rfId, - DeviceId = deviceId, Model = ShockerModelType.CaiXianlin - }); - } - - private static ShockerControlLog NewLog(Guid id, Guid shockerId, Guid controllerId, DateTime createdAt) => new() - { - Id = id, - ShockerId = shockerId, - ControlledByUserId = controllerId, - Intensity = 50, - Duration = 1000, - Type = ControlType.Shock, - CustomName = null, - CreatedAt = createdAt - }; -} diff --git a/Cron/Jobs/ClearOldShockerControlLogs.cs b/Cron/Jobs/ClearOldShockerControlLogs.cs deleted file mode 100644 index 217446554..000000000 --- a/Cron/Jobs/ClearOldShockerControlLogs.cs +++ /dev/null @@ -1,41 +0,0 @@ -using OpenShock.Common.OpenShockDb; -using OpenShock.Cron.Attributes; -using OpenShock.Internal.Common.Constants; - -namespace OpenShock.Cron.Jobs; - -/// -/// Deletes shocker control logs by enforcing a maximum log count per user -/// -[CronJob("0 0 * * *")] // Every day at midnight (https://crontab.guru/) -public sealed class ClearOldShockerControlLogs -{ - private readonly OpenShockContext _db; - private readonly IHostApplicationLifetime _lifetime; - private readonly ILogger _logger; - - /// - /// DI constructor - /// - /// - /// Its stopping token cancels the job when the server shuts down. - /// - public ClearOldShockerControlLogs(OpenShockContext db, IHostApplicationLifetime lifetime, ILogger logger) - { - _db = db; - _lifetime = lifetime; - _logger = logger; - } - - public async Task Execute() - { - var cancellationToken = _lifetime.ApplicationStopping; - var deletedUserLimits = - await _db.DeleteControlLogsBeyondPerUserLimitAsync(HardLimits.MaxShockerControlLogsPerUser, cancellationToken); - - _logger.LogInformation("Deleted {deletedUserLimits} shocker control logs exceeding the per-user limit", - deletedUserLimits); - - return deletedUserLimits; - } -} diff --git a/Dev/docker-compose.yml b/Dev/docker-compose.yml index 14a1333b0..f48d89df2 100644 --- a/Dev/docker-compose.yml +++ b/Dev/docker-compose.yml @@ -36,7 +36,24 @@ x-openshock-svc: &openshock-svc services: postgres: - image: postgres:18 + # postgres:18 plus TimescaleDB Community, so existing postgres:18 data directories keep working + build: + dockerfile_inline: | + FROM postgres:18 + ADD https://packagecloud.io/timescale/timescaledb/gpgkey /etc/apt/keyrings/timescaledb.asc + RUN set -eux; \ + chmod 644 /etc/apt/keyrings/timescaledb.asc; \ + . /etc/os-release; \ + apt-get update; \ + apt-get install -y --no-install-recommends ca-certificates; \ + echo "deb [signed-by=/etc/apt/keyrings/timescaledb.asc] https://packagecloud.io/timescale/timescaledb/debian/ $$VERSION_CODENAME main" \ + > /etc/apt/sources.list.d/timescaledb.list; \ + apt-get update; \ + apt-get install -y --no-install-recommends timescaledb-2-postgresql-18; \ + rm -rf /var/lib/apt/lists/* + image: openshock-postgres-timescaledb:18 + pull_policy: build + command: postgres -c shared_preload_libraries=timescaledb -c timescaledb.telemetry_level=off container_name: openshock-postgres healthcheck: test: [ "CMD-SHELL", "pg_isready -d $${POSTGRES_DB} -U $${POSTGRES_USER}" ] diff --git a/LiveControlGateway/LifetimeManager/HubLifetime.cs b/LiveControlGateway/LifetimeManager/HubLifetime.cs index c0a7b1ee8..6aff55e20 100644 --- a/LiveControlGateway/LifetimeManager/HubLifetime.cs +++ b/LiveControlGateway/LifetimeManager/HubLifetime.cs @@ -394,7 +394,7 @@ private async Task Update() var frame = state.LastFrame; if (frame is null || frame.ActiveUntil < now || state.ExclusiveUntil >= now) { - state.EmittingOnLastTick = null; + EndBurst(state); continue; } @@ -412,29 +412,31 @@ private async Task Update() if (commandList.Count == 0) return; - // Only count what actually got written to the hub, a tick lost to a closing connection is not activity + // Only log what actually got written to the hub, a tick lost to a closing connection is not activity if (!await HubController.SendControl(commandList)) return; var tickMs = (long)_waitBetweenTicks.TotalMilliseconds; foreach (var (state, frame) in emittingFrames) { // Each sent tick is one tick interval of activity, a new type or controller starts a new burst - var emitting = (frame.Type, frame.ControllerId); - _batchUpdateService.RecordControlStat(new ControlStatDelta( - OwnerUserId: state.OwnerId, - ShockerId: state.Id, - ControllerUserId: frame.ControllerId, - Bucket: now.UtcDateTime, - Type: frame.Type, - Live: true, - Count: state.EmittingOnLastTick == emitting ? 0 : 1, - DurationMs: tickMs, - IntensityMs: frame.Intensity * tickMs, - MaxIntensity: frame.Intensity)); - state.EmittingOnLastTick = emitting; + if (state.Burst is not { } burst || burst.Type != frame.Type || burst.ControllerId != frame.ControllerId) + { + EndBurst(state); + burst = state.Burst = new LiveBurst(frame.Type, frame.ControllerId, now.UtcDateTime); + } + + burst.AddTick(frame.Intensity, tickMs); } } + private void EndBurst(ShockerState state) + { + if (state.Burst is not { } burst) return; + state.Burst = null; + + if (burst.Type != ControlType.Stop) _batchUpdateService.InsertControlLog(burst.ToLog(state)); + } + /// /// Update all shockers config /// @@ -454,13 +456,21 @@ private async Task UpdateShockers(OpenShockContext db, CancellationToken cancell { _logger.LogDebug("Updating shockers for device [{DeviceId}]", HubController.Id); - _shockerStates = await db.Shockers.Where(x => x.DeviceId == HubController.Id).Select(x => new ShockerState + var shockerStates = await db.Shockers.Where(x => x.DeviceId == HubController.Id).Select(x => new ShockerState { Id = x.Id, OwnerId = x.Device.OwnerId, Model = x.Model, RfId = x.RfId }).ToDictionaryAsync(x => x.Id, x => x, cancellationToken); + + foreach (var (id, state) in _shockerStates) + { + if (shockerStates.TryGetValue(id, out var updated)) updated.Burst = state.Burst; + else EndBurst(state); + } + + _shockerStates = shockerStates; } /// @@ -635,6 +645,8 @@ public async ValueTask DisposeAsync() // Cancelling ends UpdateLoop, which disposes its own timer on the way out await _cancellationSource.CancelAsync(); + foreach (var state in _shockerStates.Values) EndBurst(state); + if (_deviceMsgQueue is not null) { await _deviceMsgQueue.UnsubscribeAsync(); diff --git a/LiveControlGateway/LifetimeManager/LiveBurst.cs b/LiveControlGateway/LifetimeManager/LiveBurst.cs new file mode 100644 index 000000000..d2755d761 --- /dev/null +++ b/LiveControlGateway/LifetimeManager/LiveBurst.cs @@ -0,0 +1,48 @@ +using OpenShock.Common.OpenShockDb; + +namespace OpenShock.LiveControlGateway.LifetimeManager; + +/// +/// A continuous run of live control with the same type and controller, logged as one . +/// +public sealed class LiveBurst +{ + private readonly Guid _id = Guid.CreateVersion7(); + private readonly DateTime _startedAt; + private long _durationMs; + private long _intensityMs; + private byte _maxIntensity; + + public LiveBurst(ControlType type, Guid controllerId, DateTime startedAt) + { + Type = type; + ControllerId = controllerId; + _startedAt = startedAt; + } + + public ControlType Type { get; } + + public Guid ControllerId { get; } + + public void AddTick(byte intensity, long tickMs) + { + _durationMs += tickMs; + _intensityMs += intensity * tickMs; + _maxIntensity = Math.Max(_maxIntensity, intensity); + } + + public ShockerControlLog ToLog(ShockerState shocker) => new() + { + Id = _id, + OwnerUserId = shocker.OwnerId, + ShockerId = shocker.Id, + ControlledByUserId = ControllerId, + Intensity = (byte)Math.Round((double)_intensityMs / _durationMs), + MaxIntensity = _maxIntensity, + Duration = (uint)_durationMs, + Type = Type, + CustomName = null, + LiveControl = true, + CreatedAt = _startedAt + }; +} diff --git a/LiveControlGateway/LifetimeManager/ShockerState.cs b/LiveControlGateway/LifetimeManager/ShockerState.cs index 42568304d..4bf955d8b 100644 --- a/LiveControlGateway/LifetimeManager/ShockerState.cs +++ b/LiveControlGateway/LifetimeManager/ShockerState.cs @@ -40,10 +40,9 @@ public sealed class ShockerState public LiveFrame? LastFrame { get; set; } /// - /// Whether the shocker was emitting live control on the previous tick, with which type and by whom. - /// Used to count continuous bursts for stats. + /// The live control burst running as of the previous tick /// - public (ControlType Type, Guid ControllerId)? EmittingOnLastTick { get; set; } + public LiveBurst? Burst { get; set; } /// /// When an exclusive command is sent to the shocker, this is the time it will be exclusive until so we dont allow other live commands diff --git a/README.md b/README.md index 32f6c0263..509782d4a 100644 --- a/README.md +++ b/README.md @@ -117,7 +117,10 @@ They are only required on the gateway container and are configuration local to t The OpenShock stack consists of the following components: -- Postgres as database +- Postgres 18+ as database, with the [TimescaleDB](https://github.com/timescale/timescaledb) extension in its + Community edition (the Apache-2 only `-oss` builds lack features the schema needs). It has to be in + `shared_preload_libraries`. The `db` service in `docker-compose.yml` builds such an image on top of `postgres:18`, + so an existing `postgres:18` data directory can be reused as is. - Redis-Stack (with keyspace events KEA) (preferably [DragonflyDB](https://github.com/dragonflydb/dragonfly)) - One or multiple APIs (container, API) - One or multiple gateways (container, LCG) diff --git a/SeedE2E/Seeders/ControlLogSeeder.cs b/SeedE2E/Seeders/ControlLogSeeder.cs index 057642704..d84bc6d35 100644 --- a/SeedE2E/Seeders/ControlLogSeeder.cs +++ b/SeedE2E/Seeders/ControlLogSeeder.cs @@ -15,15 +15,31 @@ public static async Task SeedAsync(OpenShockContext db, ILogger logger) logger.LogInformation("Generating ControlLogs..."); - // Grab all shocker IDs and all user IDs - var allShockerIds = await db.Shockers.Select(s => s.Id).ToListAsync(); + // Grab all shockers with their owners and all user IDs + var allShockers = await db.Shockers.Select(s => new { s.Id, s.Device.OwnerId }).ToListAsync(); var allUserIds = await db.Users.Select(u => u.Id).ToListAsync(); var controlLogFaker = new Faker() - .RuleFor(l => l.Id, f => f.Random.Guid()) - .RuleFor(l => l.ShockerId, f => f.PickRandom(allShockerIds)) + .CustomInstantiator(f => + { + var shocker = f.PickRandom(allShockers); + return new ShockerControlLog + { + Id = f.Random.Guid(), + OwnerUserId = shocker.OwnerId, + ShockerId = shocker.Id, + ControlledByUserId = null, + Intensity = 0, + MaxIntensity = 0, + Duration = 0, + Type = ControlType.Stop, + CustomName = null, + CreatedAt = default + }; + }) .RuleFor(l => l.ControlledByUserId, f => f.PickRandom(allUserIds)) .RuleFor(l => l.Intensity, f => f.Random.Byte(1, 100)) + .RuleFor(l => l.MaxIntensity, (_, l) => l.Intensity) .RuleFor(l => l.Duration, f => f.Random.UInt(100, 60000)) .RuleFor(l => l.Type, f => f.PickRandom(ControlType.Sound, ControlType.Vibrate, ControlType.Shock, ControlType.Stop)) .RuleFor(l => l.LiveControl, f => f.Random.Bool(0.1f)) @@ -31,7 +47,7 @@ public static async Task SeedAsync(OpenShockContext db, ILogger logger) .RuleFor(l => l.CustomName, f => f.Random.Bool(0.3f) ? f.Commerce.ProductAdjective() : null); // Generate roughly 100 logs per Shocker - var bogusControlLogs = controlLogFaker.Generate(allShockerIds.Count * 100); + var bogusControlLogs = controlLogFaker.Generate(allShockers.Count * 100); db.ShockerControlLogs.AddRange(bogusControlLogs); await db.SaveChangesAsync(); diff --git a/docker-compose.yml b/docker-compose.yml index bdf08aadf..7f66eabd2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -77,8 +77,24 @@ x-openshock-svc: &openshock-svc services: - db: # We need a postgres database, preferably version 18+ - image: postgres:18 + db: # We need a postgres database, version 18+, with the TimescaleDB extension (Community edition) + build: + dockerfile_inline: | + FROM postgres:18 + ADD https://packagecloud.io/timescale/timescaledb/gpgkey /etc/apt/keyrings/timescaledb.asc + RUN set -eux; \ + chmod 644 /etc/apt/keyrings/timescaledb.asc; \ + . /etc/os-release; \ + apt-get update; \ + apt-get install -y --no-install-recommends ca-certificates; \ + echo "deb [signed-by=/etc/apt/keyrings/timescaledb.asc] https://packagecloud.io/timescale/timescaledb/debian/ $$VERSION_CODENAME main" \ + > /etc/apt/sources.list.d/timescaledb.list; \ + apt-get update; \ + apt-get install -y --no-install-recommends timescaledb-2-postgresql-18; \ + rm -rf /var/lib/apt/lists/* + image: openshock-postgres-timescaledb:18 + pull_policy: build + command: postgres -c shared_preload_libraries=timescaledb -c timescaledb.telemetry_level=off restart: unless-stopped container_name: openshock-postgres healthcheck: From 62a263075dd1eed92b7779e38fb64874526ae26b Mon Sep 17 00:00:00 2001 From: LucHeart Date: Sat, 21 Nov 2026 00:22:14 +0100 Subject: [PATCH 2/6] feat: refuse to migrate without TimescaleDB Community --- .../Tests/TimescaleDbRequirementTests.cs | 118 ++++++++++++++++++ .../20261120201327_AddControlLogHypertable.cs | 5 - .../20261120201415_MaterializeControlStats.cs | 12 ++ Common/OpenShockMiddlewareHelper.cs | 45 +++++++ 4 files changed, 175 insertions(+), 5 deletions(-) create mode 100644 API.IntegrationTests/Tests/TimescaleDbRequirementTests.cs diff --git a/API.IntegrationTests/Tests/TimescaleDbRequirementTests.cs b/API.IntegrationTests/Tests/TimescaleDbRequirementTests.cs new file mode 100644 index 000000000..09b0c26d6 --- /dev/null +++ b/API.IntegrationTests/Tests/TimescaleDbRequirementTests.cs @@ -0,0 +1,118 @@ +using Microsoft.AspNetCore.Builder; +using Microsoft.Extensions.DependencyInjection; +using Npgsql; +using OpenShock.Common; +using OpenShock.Common.Options; +using Testcontainers.PostgreSql; + +namespace OpenShock.API.IntegrationTests.Tests; + +public sealed class TimescaleDbRequirementTests +{ + private const string Community = "timescale/timescaledb:latest-pg18"; + + private static PostgreSqlContainer CreateContainer(string image) => new PostgreSqlBuilder(image: image).Build(); + + private static string WithDatabase(PostgreSqlContainer container, string database) => + new NpgsqlConnectionStringBuilder(container.GetConnectionString()) { Database = database }.ConnectionString; + + private static async Task Psql(PostgreSqlContainer container, string database, string sql) + { + var result = await container.ExecAsync(["psql", "-U", PostgreSqlBuilder.DefaultUsername, "-d", database, "-c", sql]); + if (result.ExitCode != 0) throw new InvalidOperationException(result.Stderr); + } + + private static async Task HasExtension(PostgreSqlContainer container, string database) + { + await using var connection = new NpgsqlConnection(WithDatabase(container, database)); + await connection.OpenAsync(); + await using var command = new NpgsqlCommand("SELECT count(*) FROM pg_extension WHERE extname = 'timescaledb'", connection); + return (long)(await command.ExecuteScalarAsync())! == 1; + } + + private static async Task HasControlStatsPolicy(PostgreSqlContainer container, string database) + { + await using var connection = new NpgsqlConnection(WithDatabase(container, database)); + await connection.OpenAsync(); + await using var command = new NpgsqlCommand( + """ + SELECT count(*) FROM timescaledb_information.jobs + WHERE proc_name = 'policy_refresh_continuous_aggregate' AND hypertable_name = 'control_stats' + """, connection); + return (long)(await command.ExecuteScalarAsync())! == 1; + } + + private static async Task HasMigrationHistory(PostgreSqlContainer container) + { + await using var connection = new NpgsqlConnection(container.GetConnectionString()); + await connection.OpenAsync(); + await using var command = new NpgsqlCommand("""SELECT to_regclass('"__EFMigrationsHistory"') IS NOT NULL""", connection); + return (bool)(await command.ExecuteScalarAsync())!; + } + + private static async Task ApplyMigrations(string connectionString) + { + await using var services = new ServiceCollection().AddLogging().BuildServiceProvider(); + await new ApplicationBuilder(services).ApplyPendingOpenShockMigrations(new DatabaseOptions { Conn = connectionString }); + } + + [Test] + public async Task Community_ApplyingMigrationsOnNewDatabase_Works() + { + await using var container = CreateContainer(Community); + await container.StartAsync(); + + await ApplyMigrations(WithDatabase(container, "created_on_startup")); + + await Assert.That(await HasExtension(container, "created_on_startup")).IsTrue(); + await Assert.That(await HasControlStatsPolicy(container, "created_on_startup")).IsTrue(); + } + + [Test] + public async Task Community_Check_LeavesNothingBehind() + { + await using var container = CreateContainer(Community); + await container.StartAsync(); + await Psql(container, PostgreSqlBuilder.DefaultDatabase, "CREATE DATABASE fresh TEMPLATE template0"); + + await OpenShockMiddlewareHelper.EnsureTimescaleDbAsync(WithDatabase(container, "fresh")); + + await Assert.That(await HasExtension(container, "fresh")).IsFalse(); + } + + [Test] + [Arguments("postgres:18", "is not available")] + [Arguments("timescale/timescaledb:latest-pg18-oss", "apache")] + public async Task WrongEdition_RefusesToMigrate(string image, string reason) + { + await using var container = CreateContainer(image); + await container.StartAsync(); + + var exception = await Assert.That(() => ApplyMigrations(container.GetConnectionString())) + .Throws(); + + await Assert.That(exception!.Message).Contains(reason); + await Assert.That(await HasMigrationHistory(container)).IsFalse(); + } + + [Test] + public async Task NotPreloaded_RefusesToMigrate() + { + await using var container = CreateContainer(Community); + await container.StartAsync(); + await Psql(container, PostgreSqlBuilder.DefaultDatabase, "CREATE DATABASE fresh TEMPLATE template0"); + await Psql(container, PostgreSqlBuilder.DefaultDatabase, "CREATE DATABASE has_extension TEMPLATE template0"); + await Psql(container, "has_extension", "CREATE EXTENSION timescaledb"); + await Psql(container, PostgreSqlBuilder.DefaultDatabase, "ALTER SYSTEM SET shared_preload_libraries = 'pg_stat_statements'"); + await container.StopAsync(); + await container.StartAsync(); + + foreach (var (database, reason) in new[] { ("fresh", "must be preloaded"), ("has_extension", "not preloaded") }) + { + var exception = await Assert.That(() => ApplyMigrations(WithDatabase(container, database))) + .Throws(); + + await Assert.That(exception!.Message).Contains(reason); + } + } +} diff --git a/Common/Migrations/20261120201327_AddControlLogHypertable.cs b/Common/Migrations/20261120201327_AddControlLogHypertable.cs index d3579b319..9781468fa 100644 --- a/Common/Migrations/20261120201327_AddControlLogHypertable.cs +++ b/Common/Migrations/20261120201327_AddControlLogHypertable.cs @@ -158,11 +158,6 @@ FROM shocker_control_logs WHERE type <> 'stop' GROUP BY bucket, owner_user_id, shocker_id, controlled_by_user_id, type, live_control WITH NO DATA; - - SELECT add_continuous_aggregate_policy('control_stats', - start_offset => INTERVAL '3 days', - end_offset => INTERVAL '1 hour', - schedule_interval => INTERVAL '5 minutes'); """); } diff --git a/Common/Migrations/20261120201415_MaterializeControlStats.cs b/Common/Migrations/20261120201415_MaterializeControlStats.cs index 0056a8562..e5f042401 100644 --- a/Common/Migrations/20261120201415_MaterializeControlStats.cs +++ b/Common/Migrations/20261120201415_MaterializeControlStats.cs @@ -15,6 +15,18 @@ protected override void Up(MigrationBuilder migrationBuilder) migrationBuilder.Sql( "CALL refresh_continuous_aggregate('control_stats', NULL, now() - INTERVAL '1 hour');", suppressTransaction: true); + + // Added after the refresh above, which fails if the policy's job is refreshing at the same time + migrationBuilder.Sql( + """ + SELECT add_continuous_aggregate_policy('control_stats', + start_offset => INTERVAL '3 days', + end_offset => INTERVAL '1 hour', + schedule_interval => INTERVAL '5 minutes', + if_not_exists => true, + initial_start => now() + INTERVAL '5 minutes'); + """, + suppressTransaction: true); } /// diff --git a/Common/OpenShockMiddlewareHelper.cs b/Common/OpenShockMiddlewareHelper.cs index 2eaefb271..60c3bce3d 100644 --- a/Common/OpenShockMiddlewareHelper.cs +++ b/Common/OpenShockMiddlewareHelper.cs @@ -3,6 +3,8 @@ using Microsoft.AspNetCore.Diagnostics.HealthChecks; using Microsoft.AspNetCore.HttpOverrides; using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Storage; using Microsoft.Extensions.Diagnostics.HealthChecks; using Npgsql; using OpenShock.Common.HealthChecks; @@ -220,6 +222,14 @@ public static async Task ApplyPendingOpenShockMigrations(th await using var migrationContext = new MigrationOpenShockContext(options.Conn, options.Debug, loggerFactory); + var databaseCreator = migrationContext.GetService(); + if (!await databaseCreator.ExistsAsync()) + { + await databaseCreator.CreateAsync(); + } + + await EnsureTimescaleDbAsync(options.Conn); + var pendingMigrations = (await migrationContext.Database.GetPendingMigrationsAsync()).ToArray(); if (pendingMigrations.Length > 0) @@ -244,6 +254,41 @@ public static async Task ApplyPendingOpenShockMigrations(th return app; } + /// + /// Creates the extension in a transaction that is rolled back: shared_preload_libraries is only readable + /// by privileged roles, and the extension only reports its license once created. + /// + public static async Task EnsureTimescaleDbAsync(string connectionString) + { + const string requirement = "The database requires the TimescaleDB Community edition, installed and listed in shared_preload_libraries."; + + var builder = new NpgsqlConnectionStringBuilder(connectionString) { Pooling = false }; + await using var connection = new NpgsqlConnection(builder.ConnectionString); + await connection.OpenAsync(); + await using var transaction = await connection.BeginTransactionAsync(); + + object? license; + try + { + await using var create = new NpgsqlCommand("CREATE EXTENSION IF NOT EXISTS timescaledb", connection, transaction); + await create.ExecuteNonQueryAsync(); + + await using var read = new NpgsqlCommand("SELECT current_setting('timescaledb.license', true)", connection, transaction); + license = await read.ExecuteScalarAsync(); + } + catch (PostgresException ex) + { + throw new InvalidOperationException($"{requirement} {ex.MessageText}", ex); + } + + if (license is not "timescale") + { + throw new InvalidOperationException(license is string edition + ? $"{requirement} Found the {edition} edition." + : $"{requirement} It is not preloaded."); + } + } + /// /// Forces the runtime pooled 's Npgsql data source to reload its Postgres /// type catalog. Called after migrations are applied so newly-added types (enums) are visible without a From f19bad69843ec3c2207556c5a316fa4bd8d60b66 Mon Sep 17 00:00:00 2001 From: LucHeart Date: Sat, 21 Nov 2026 00:22:14 +0100 Subject: [PATCH 3/6] chore: use the postgres-timescaledb image from infra --- Dev/docker-compose.yml | 19 +------------------ README.md | 25 +++++++++++++++++++++++-- docker-compose.yml | 18 +----------------- 3 files changed, 25 insertions(+), 37 deletions(-) diff --git a/Dev/docker-compose.yml b/Dev/docker-compose.yml index f48d89df2..05fbd4b1b 100644 --- a/Dev/docker-compose.yml +++ b/Dev/docker-compose.yml @@ -36,24 +36,7 @@ x-openshock-svc: &openshock-svc services: postgres: - # postgres:18 plus TimescaleDB Community, so existing postgres:18 data directories keep working - build: - dockerfile_inline: | - FROM postgres:18 - ADD https://packagecloud.io/timescale/timescaledb/gpgkey /etc/apt/keyrings/timescaledb.asc - RUN set -eux; \ - chmod 644 /etc/apt/keyrings/timescaledb.asc; \ - . /etc/os-release; \ - apt-get update; \ - apt-get install -y --no-install-recommends ca-certificates; \ - echo "deb [signed-by=/etc/apt/keyrings/timescaledb.asc] https://packagecloud.io/timescale/timescaledb/debian/ $$VERSION_CODENAME main" \ - > /etc/apt/sources.list.d/timescaledb.list; \ - apt-get update; \ - apt-get install -y --no-install-recommends timescaledb-2-postgresql-18; \ - rm -rf /var/lib/apt/lists/* - image: openshock-postgres-timescaledb:18 - pull_policy: build - command: postgres -c shared_preload_libraries=timescaledb -c timescaledb.telemetry_level=off + image: ghcr.io/openshock/postgres-timescaledb:2.31.0-18.7-trixie # postgres:18 + TimescaleDB Community, built in OpenShock/infra container_name: openshock-postgres healthcheck: test: [ "CMD-SHELL", "pg_isready -d $${POSTGRES_DB} -U $${POSTGRES_USER}" ] diff --git a/README.md b/README.md index 509782d4a..b704f1593 100644 --- a/README.md +++ b/README.md @@ -119,8 +119,9 @@ The OpenShock stack consists of the following components: - Postgres 18+ as database, with the [TimescaleDB](https://github.com/timescale/timescaledb) extension in its Community edition (the Apache-2 only `-oss` builds lack features the schema needs). It has to be in - `shared_preload_libraries`. The `db` service in `docker-compose.yml` builds such an image on top of `postgres:18`, - so an existing `postgres:18` data directory can be reused as is. + `shared_preload_libraries`, the API refuses to start otherwise. The `db` service in `docker-compose.yml` uses such an + image built on top of `postgres:18` (`ghcr.io/openshock/postgres-timescaledb`), so an existing `postgres:18` data directory can + be reused as is. - Redis-Stack (with keyspace events KEA) (preferably [DragonflyDB](https://github.com/dragonflydb/dragonfly)) - One or multiple APIs (container, API) - One or multiple gateways (container, LCG) @@ -151,6 +152,26 @@ Run with `docker compose up -d` You could also bring your own reverse proxy. You would need to remove traefik from the `docker-compose.yml` and route the traffic in your reverse proxy. +## Upgrading to TimescaleDB + +Control logs are stored in a TimescaleDB hypertable. When upgrading an existing instance: + +**With the provided `docker-compose.yml`:** pull the new compose file and run `docker compose up -d`. The `db` service +now uses `ghcr.io/openshock/postgres-timescaledb`, which is `postgres:18` with TimescaleDB on top, so your existing +`postgres-data` directory is reused and the API migrates it on startup. + +**With your own Postgres:** before upgrading OpenShock + +1. Install the TimescaleDB Community edition for your Postgres version (e.g. the `timescaledb-2-postgresql-18` package + from Timescale's repository). The `-oss` builds and managed offerings that only ship the Apache-2 edition do not + work. +2. Add `timescaledb` to `shared_preload_libraries` in `postgresql.conf` and restart Postgres. +3. Make sure the database user OpenShock connects with may run `CREATE EXTENSION timescaledb` (usually a superuser), + or create the extension in the OpenShock database yourself. + +The API checks this on startup and refuses to migrate otherwise. Existing control logs are converted in place, from +then on they are kept for a year by a retention policy. + ## Security Found a security vulnerability? **Please do not open a public issue.** Report it privately as described in diff --git a/docker-compose.yml b/docker-compose.yml index 7f66eabd2..bbb66d4bd 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -78,23 +78,7 @@ x-openshock-svc: &openshock-svc services: db: # We need a postgres database, version 18+, with the TimescaleDB extension (Community edition) - build: - dockerfile_inline: | - FROM postgres:18 - ADD https://packagecloud.io/timescale/timescaledb/gpgkey /etc/apt/keyrings/timescaledb.asc - RUN set -eux; \ - chmod 644 /etc/apt/keyrings/timescaledb.asc; \ - . /etc/os-release; \ - apt-get update; \ - apt-get install -y --no-install-recommends ca-certificates; \ - echo "deb [signed-by=/etc/apt/keyrings/timescaledb.asc] https://packagecloud.io/timescale/timescaledb/debian/ $$VERSION_CODENAME main" \ - > /etc/apt/sources.list.d/timescaledb.list; \ - apt-get update; \ - apt-get install -y --no-install-recommends timescaledb-2-postgresql-18; \ - rm -rf /var/lib/apt/lists/* - image: openshock-postgres-timescaledb:18 - pull_policy: build - command: postgres -c shared_preload_libraries=timescaledb -c timescaledb.telemetry_level=off + image: ghcr.io/openshock/postgres-timescaledb:2.31.0-18.7-trixie # postgres:18 + TimescaleDB Community, built in OpenShock/infra restart: unless-stopped container_name: openshock-postgres healthcheck: From b76669890fcdb392bb36f218ec65e14242656859 Mon Sep 17 00:00:00 2001 From: LucHeart Date: Sat, 21 Nov 2026 00:31:40 +0100 Subject: [PATCH 4/6] fix: bump TimescaleDB to 2.32.1 --- Dev/docker-compose.yml | 2 +- docker-compose.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dev/docker-compose.yml b/Dev/docker-compose.yml index 05fbd4b1b..f17775bda 100644 --- a/Dev/docker-compose.yml +++ b/Dev/docker-compose.yml @@ -36,7 +36,7 @@ x-openshock-svc: &openshock-svc services: postgres: - image: ghcr.io/openshock/postgres-timescaledb:2.31.0-18.7-trixie # postgres:18 + TimescaleDB Community, built in OpenShock/infra + image: ghcr.io/openshock/postgres-timescaledb:2.32.1-18.7-trixie # postgres:18 + TimescaleDB Community, built in OpenShock/infra container_name: openshock-postgres healthcheck: test: [ "CMD-SHELL", "pg_isready -d $${POSTGRES_DB} -U $${POSTGRES_USER}" ] diff --git a/docker-compose.yml b/docker-compose.yml index bbb66d4bd..9edeec7b3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -78,7 +78,7 @@ x-openshock-svc: &openshock-svc services: db: # We need a postgres database, version 18+, with the TimescaleDB extension (Community edition) - image: ghcr.io/openshock/postgres-timescaledb:2.31.0-18.7-trixie # postgres:18 + TimescaleDB Community, built in OpenShock/infra + image: ghcr.io/openshock/postgres-timescaledb:2.32.1-18.7-trixie # postgres:18 + TimescaleDB Community, built in OpenShock/infra restart: unless-stopped container_name: openshock-postgres healthcheck: From 1958e2b54ba5d7cc667322697eef0470f0ea7df8 Mon Sep 17 00:00:00 2001 From: LucHeart Date: Sat, 21 Nov 2026 20:05:10 +0100 Subject: [PATCH 5/6] fix: address review comments --- .../Helpers/ControlLogHelper.cs | 19 +++++++++++++++++-- .../BatchUpdate/BatchUpdateService.cs | 17 +++++++---------- SeedE2E/Program.cs | 6 +++++- 3 files changed, 29 insertions(+), 13 deletions(-) diff --git a/API.IntegrationTests/Helpers/ControlLogHelper.cs b/API.IntegrationTests/Helpers/ControlLogHelper.cs index cacb22849..e21429134 100644 --- a/API.IntegrationTests/Helpers/ControlLogHelper.cs +++ b/API.IntegrationTests/Helpers/ControlLogHelper.cs @@ -1,5 +1,6 @@ using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.DependencyInjection; +using Npgsql; using OpenShock.Common.OpenShockDb; namespace OpenShock.API.IntegrationTests.Helpers; @@ -61,8 +62,22 @@ public static async Task RefreshControlStats(WebApplicationFactory factory) try { await using var scope = factory.Services.CreateAsyncScope(); - await scope.ServiceProvider.GetRequiredService().Database.ExecuteSqlRawAsync( - "CALL refresh_continuous_aggregate('control_stats', NULL, now() - INTERVAL '1 hour')"); + var db = scope.ServiceProvider.GetRequiredService(); + + // The refresh policy job may hold the same range, TimescaleDB fails instead of waiting + for (var attempt = 1;; attempt++) + { + try + { + await db.Database.ExecuteSqlRawAsync( + "CALL refresh_continuous_aggregate('control_stats', NULL, now() - INTERVAL '1 hour')"); + return; + } + catch (PostgresException e) when (e.SqlState == PostgresErrorCodes.LockNotAvailable && attempt < 10) + { + await Task.Delay(TimeSpan.FromMilliseconds(200 * attempt)); + } + } } finally { diff --git a/Common/Services/BatchUpdate/BatchUpdateService.cs b/Common/Services/BatchUpdate/BatchUpdateService.cs index 38563d06c..de5368a68 100644 --- a/Common/Services/BatchUpdate/BatchUpdateService.cs +++ b/Common/Services/BatchUpdate/BatchUpdateService.cs @@ -1,5 +1,4 @@ -using System.Text.Json; -using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore; using NRedisStack.RedisStackCommands; using OpenShock.Common.Constants; using OpenShock.Common.OpenShockDb; @@ -169,14 +168,12 @@ private async Task InsertControlLogsOnShutdown(CancellationToken cancellationTok } catch (Exception e) { - _logger.LogWarning(e, "Failed to insert control logs on shutdown"); - } - - lock (_controlLogsLock) - { - _logger.LogCritical("Lost {Count} control logs on shutdown, the database didn't take them: {Logs}", - _controlLogs.Count, JsonSerializer.Serialize(_controlLogs)); - _controlLogs = []; + lock (_controlLogsLock) + { + _logger.LogCritical(e, "Lost {Count} control logs on shutdown, the database didn't take them", + _controlLogs.Count); + _controlLogs = []; + } } } diff --git a/SeedE2E/Program.cs b/SeedE2E/Program.cs index a620d06d7..6c736d2bc 100644 --- a/SeedE2E/Program.cs +++ b/SeedE2E/Program.cs @@ -1,4 +1,5 @@ -using Microsoft.Extensions.Configuration; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; @@ -63,6 +64,9 @@ await DiscordWebhookSeeder.SeedAsync(db, logger); await transaction.CommitAsync(); + + // The refresh policy only covers recent days, the seeded logs go back further + await db.Database.ExecuteSqlRawAsync("CALL refresh_continuous_aggregate('control_stats', NULL, NULL)"); } Console.WriteLine("Database seeding complete."); From 57ee3f43e94e701fd2f4cff8ef48acfbb1279f7c Mon Sep 17 00:00:00 2001 From: LucHeart Date: Sat, 21 Nov 2026 20:41:44 +0100 Subject: [PATCH 6/6] fix: cascade owner deletion through a foreign key instead of a trigger --- ...201327_AddControlLogHypertable.Designer.cs | 10 ++++++++ .../20261120201327_AddControlLogHypertable.cs | 23 +++++++------------ ...201415_MaterializeControlStats.Designer.cs | 10 ++++++++ .../OpenShockContextModelSnapshot.cs | 10 ++++++++ .../OpenShockDb/Models/ShockerControlLog.cs | 2 +- Common/OpenShockDb/OpenShockContext.cs | 6 +++++ 6 files changed, 45 insertions(+), 16 deletions(-) diff --git a/Common/Migrations/20261120201327_AddControlLogHypertable.Designer.cs b/Common/Migrations/20261120201327_AddControlLogHypertable.Designer.cs index cdfbcea18..5f973a6c3 100644 --- a/Common/Migrations/20261120201327_AddControlLogHypertable.Designer.cs +++ b/Common/Migrations/20261120201327_AddControlLogHypertable.Designer.cs @@ -1576,6 +1576,16 @@ protected override void BuildTargetModel(ModelBuilder modelBuilder) b.Navigation("Device"); }); + modelBuilder.Entity("OpenShock.Common.OpenShockDb.ShockerControlLog", b => + { + b.HasOne("OpenShock.Common.OpenShockDb.User", null) + .WithMany() + .HasForeignKey("OwnerUserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_shocker_control_logs_owner_user_id"); + }); + modelBuilder.Entity("OpenShock.Common.OpenShockDb.ShockerShareCode", b => { b.HasOne("OpenShock.Common.OpenShockDb.Shocker", "Shocker") diff --git a/Common/Migrations/20261120201327_AddControlLogHypertable.cs b/Common/Migrations/20261120201327_AddControlLogHypertable.cs index 9781468fa..89e17f2ed 100644 --- a/Common/Migrations/20261120201327_AddControlLogHypertable.cs +++ b/Common/Migrations/20261120201327_AddControlLogHypertable.cs @@ -122,21 +122,14 @@ ALTER TABLE shocker_control_logs SET ( SELECT add_retention_policy('shocker_control_logs', drop_after => INTERVAL '1 year'); """); - // Logs have no foreign keys, but the owner's logs go with their account - migrationBuilder.Sql( - """ - CREATE FUNCTION users_delete_shocker_control_logs() RETURNS trigger LANGUAGE plpgsql AS $$ - BEGIN - DELETE FROM shocker_control_logs l USING removed r WHERE l.owner_user_id = r.id; - RETURN NULL; - END - $$; - - CREATE TRIGGER users_delete_shocker_control_logs - AFTER DELETE ON users - REFERENCING OLD TABLE AS removed - FOR EACH STATEMENT EXECUTE FUNCTION users_delete_shocker_control_logs(); - """); + // Only the owner, the logs outlive their shocker and controller + migrationBuilder.AddForeignKey( + name: "fk_shocker_control_logs_owner_user_id", + table: "shocker_control_logs", + column: "owner_user_id", + principalTable: "users", + principalColumn: "id", + onDelete: ReferentialAction.Cascade); // 15 minute buckets, every time zone's UTC offset is a multiple of that. The refresh window must stay well // within the logs' retention, refreshing a bucket whose logs were dropped would wipe its stats. diff --git a/Common/Migrations/20261120201415_MaterializeControlStats.Designer.cs b/Common/Migrations/20261120201415_MaterializeControlStats.Designer.cs index 79fdeb8d0..89ec1fbd3 100644 --- a/Common/Migrations/20261120201415_MaterializeControlStats.Designer.cs +++ b/Common/Migrations/20261120201415_MaterializeControlStats.Designer.cs @@ -1576,6 +1576,16 @@ protected override void BuildTargetModel(ModelBuilder modelBuilder) b.Navigation("Device"); }); + modelBuilder.Entity("OpenShock.Common.OpenShockDb.ShockerControlLog", b => + { + b.HasOne("OpenShock.Common.OpenShockDb.User", null) + .WithMany() + .HasForeignKey("OwnerUserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_shocker_control_logs_owner_user_id"); + }); + modelBuilder.Entity("OpenShock.Common.OpenShockDb.ShockerShareCode", b => { b.HasOne("OpenShock.Common.OpenShockDb.Shocker", "Shocker") diff --git a/Common/Migrations/OpenShockContextModelSnapshot.cs b/Common/Migrations/OpenShockContextModelSnapshot.cs index 8383175ec..dc848102f 100644 --- a/Common/Migrations/OpenShockContextModelSnapshot.cs +++ b/Common/Migrations/OpenShockContextModelSnapshot.cs @@ -1578,6 +1578,16 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Navigation("Device"); }); + modelBuilder.Entity("OpenShock.Common.OpenShockDb.ShockerControlLog", b => + { + b.HasOne("OpenShock.Common.OpenShockDb.User", null) + .WithMany() + .HasForeignKey("OwnerUserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_shocker_control_logs_owner_user_id"); + }); + modelBuilder.Entity("OpenShock.Common.OpenShockDb.ShockerShareCode", b => { b.HasOne("OpenShock.Common.OpenShockDb.Shocker", "Shocker") diff --git a/Common/OpenShockDb/Models/ShockerControlLog.cs b/Common/OpenShockDb/Models/ShockerControlLog.cs index 965d9d825..9a71cfa9a 100644 --- a/Common/OpenShockDb/Models/ShockerControlLog.cs +++ b/Common/OpenShockDb/Models/ShockerControlLog.cs @@ -1,7 +1,7 @@ namespace OpenShock.Common.OpenShockDb; /// -/// No foreign keys to shockers and controllers, so their stats survive them being deleted. +/// No foreign keys to shockers and controllers, so their stats survive them being deleted. The owner's logs go with their account. /// public sealed class ShockerControlLog { diff --git a/Common/OpenShockDb/OpenShockContext.cs b/Common/OpenShockDb/OpenShockContext.cs index 5760ce1c4..0a8b4776e 100644 --- a/Common/OpenShockDb/OpenShockContext.cs +++ b/Common/OpenShockDb/OpenShockContext.cs @@ -451,6 +451,12 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) entity.Property(e => e.InsertedAt) .HasDefaultValueSql("now()") .HasColumnName("inserted_at"); + + // Only the owner, the logs outlive their shocker and controller + entity.HasOne().WithMany() + .HasForeignKey(e => e.OwnerUserId) + .OnDelete(DeleteBehavior.Cascade) + .HasConstraintName("fk_shocker_control_logs_owner_user_id"); }); modelBuilder.Entity(entity =>