Stabilise story intelligence global progress

PlotLine.Tests/Program.cs

@@ -236,6 +236,8 @@ var tests = new (string Name, Action Test)[]
     ("Story Intelligence recovered provider failures stay operational", StoryIntelligenceRecoveredProviderFailuresStayOperational),
     ("Story Intelligence recovered JSON repair stays operational", StoryIntelligenceRecoveredJsonRepairStaysOperational),
     ("Story Intelligence terminal failures remain user visible", StoryIntelligenceTerminalFailuresRemainUserVisible),
+    ("Story Intelligence global banner remains visible through finalising gaps", StoryIntelligenceGlobalBannerRemainsVisibleThroughFinalisingGaps),
+    ("Story Intelligence global banner rejects stale live snapshots", StoryIntelligenceGlobalBannerRejectsStaleLiveSnapshots),
     ("Story Intelligence active progress only appears in global strip", StoryIntelligenceActiveProgressOnlyAppearsInGlobalStrip),
     ("Story Intelligence book actions do not open review centre while running", StoryIntelligenceBookActionsDoNotOpenReviewCentreWhileRunning)
 };
@@ -285,6 +287,7 @@ static void StoryIntelligenceGlobalProgressUsesWholeRunSceneTotals()
     var layout = File.ReadAllText(Path.Combine(root, "Views/Shared/_Layout.cshtml"));
     var script = File.ReadAllText(Path.Combine(root, "wwwroot/js/story-intelligence-global-progress.js"));
     var migration = File.ReadAllText(Path.Combine(root, "Sql/170_Phase26F_StoryIntelligenceWholeRunProgress.sql"));
+    var authoritativeMigration = File.ReadAllText(Path.Combine(root, "Sql/171_Phase26G_StoryIntelligenceAuthoritativeGlobalProgress.sql"));
 
     var book = new StoryIntelligenceGlobalProgressBookViewModel
     {
@@ -322,6 +325,10 @@ static void StoryIntelligenceGlobalProgressUsesWholeRunSceneTotals()
     Assert(finalisingBook.ProgressPercent == 100, "Scene progress can be visually full during finalisation.");
     Assert(finalisingBook.EstimatedRemaining == "Finalising", "All scene analyses complete should show finalising until pipeline state is persisted.");
 
+    var finalisingModel = new StoryIntelligenceGlobalProgressViewModel { Books = [finalisingBook] };
+    Assert(finalisingModel.HasActiveWork, "Finalising should keep the global strip visible even when no chapter worker remains active.");
+    Assert(finalisingModel.Status == "Finalising", "Finalising pipeline state should be the aggregate status.");
+
     var failedBook = new StoryIntelligenceGlobalProgressBookViewModel
     {
         BookTitle = "Test Book",
@@ -345,6 +352,68 @@ static void StoryIntelligenceGlobalProgressUsesWholeRunSceneTotals()
     Assert(migration.Contains("CanonicalSceneScope", StringComparison.Ordinal), "Progress migration should derive expected scope from canonical scenes.");
     Assert(migration.Contains("ActiveScenes", StringComparison.Ordinal), "Progress migration should keep active scene count distinct.");
     Assert(migration.Contains("N'Finalising'", StringComparison.Ordinal), "Progress migration should expose a finalising state.");
+    Assert(authoritativeMigration.Contains("pipeline.Status = N'InProgress'", StringComparison.Ordinal), "Authoritative progress should include in-progress book pipelines.");
+    Assert(authoritativeMigration.Contains("LEFT JOIN ScopedRuns", StringComparison.Ordinal), "Finalising pipeline scopes should remain visible even if no active run row remains.");
+    Assert(authoritativeMigration.Contains("r.Status IN (N'Pending', N'Running')", StringComparison.Ordinal), "Queued and running work should keep active scene counts separate from completed scenes.");
+}
+
+static void StoryIntelligenceGlobalBannerRemainsVisibleThroughFinalisingGaps()
+{
+    var root = PlotLineRoot();
+    var viewModel = File.ReadAllText(Path.Combine(root, "ViewModels/StoryIntelligenceGlobalProgressViewModels.cs"));
+    var layout = File.ReadAllText(Path.Combine(root, "Views/Shared/_Layout.cshtml"));
+    var script = File.ReadAllText(Path.Combine(root, "wwwroot/js/story-intelligence-global-progress.js"));
+    var migration = File.ReadAllText(Path.Combine(root, "Sql/171_Phase26G_StoryIntelligenceAuthoritativeGlobalProgress.sql"));
+
+    var finalisingBook = new StoryIntelligenceGlobalProgressBookViewModel
+    {
+        BookTitle = "Finalising Book",
+        TotalExpectedScenes = 49,
+        CompletedScenes = 49,
+        Status = "Finalising",
+        UpdatedUtc = DateTime.UtcNow
+    };
+    var model = new StoryIntelligenceGlobalProgressViewModel { Books = [finalisingBook] };
+
+    Assert(model.HasActiveWork, "Finalising book lifecycle should render the shared banner.");
+    Assert(model.ActiveBookCount == 1, "Finalising should count as active visible work.");
+    Assert(model.SummaryText.Contains("49/49", StringComparison.Ordinal), "Finalising summary should remain book-scoped instead of becoming empty.");
+    Assert(viewModel.Contains("|| string.Equals(Status, \"Finalising\"", StringComparison.Ordinal), "Finalising should be treated as active in the shared status model.");
+    Assert(layout.Contains("data-story-intelligence-updated-utc", StringComparison.Ordinal), "Initial render should expose an aggregate freshness timestamp.");
+    Assert(!script.Contains("strip.hidden = true", StringComparison.Ordinal), "Live refresh should not hide the banner from an empty/finalising aggregate.");
+    Assert(!script.Contains("window.setTimeout(() =>", StringComparison.Ordinal), "Completion states should not auto-hide while final review routing may still be preparing.");
+    Assert(migration.Contains("OR pipeline.Status = N'InProgress'", StringComparison.Ordinal), "In-progress book pipeline state should keep the aggregate row visible.");
+}
+
+static void StoryIntelligenceGlobalBannerRejectsStaleLiveSnapshots()
+{
+    var root = PlotLineRoot();
+    var script = File.ReadAllText(Path.Combine(root, "wwwroot/js/story-intelligence-global-progress.js"));
+    var viewModel = File.ReadAllText(Path.Combine(root, "ViewModels/StoryIntelligenceGlobalProgressViewModels.cs"));
+
+    var runningWithTerminalSceneIssue = new StoryIntelligenceGlobalProgressViewModel
+    {
+        Books =
+        [
+            new StoryIntelligenceGlobalProgressBookViewModel
+            {
+                BookTitle = "Concurrent Book",
+                RunningRunCount = 1,
+                TotalExpectedScenes = 49,
+                CompletedScenes = 20,
+                FailedScenes = 1,
+                ActiveScenes = 11,
+                Status = StoryIntelligenceRunStatuses.Running,
+                UpdatedUtc = DateTime.UtcNow
+            }
+        ]
+    };
+
+    Assert(runningWithTerminalSceneIssue.Status == StoryIntelligenceRunStatuses.Running, "Active running work should outrank terminal scene warnings until finalisation.");
+    Assert(script.Contains("lastSnapshotTime", StringComparison.Ordinal), "Client should track the freshest aggregate snapshot.");
+    Assert(script.Contains("updatedTime < lastSnapshotTime", StringComparison.Ordinal), "Out-of-order SignalR refreshes should not regress the visible banner.");
+    Assert(script.Contains("strip.dataset.connection = \"waiting\"", StringComparison.Ordinal), "Empty aggregate refreshes should be treated as indeterminate rather than completion.");
+    Assert(viewModel.Contains("public DateTime? UpdatedUtc", StringComparison.Ordinal), "Aggregate payload should expose a timestamp for stale-snapshot checks.");
 }
 
 static void StoryIntelligenceRecoveredProviderFailuresStayOperational()

PlotLine/Sql/171_Phase26G_StoryIntelligenceAuthoritativeGlobalProgress.sql

@@ -0,0 +1,191 @@
+CREATE OR ALTER PROCEDURE dbo.StoryIntelligenceRun_ListActiveBookSummaryForUser
+    @UserID int
+AS
+BEGIN
+    SET NOCOUNT ON;
+
+    ;WITH RunScopes AS
+    (
+        SELECT
+            r.UserID,
+            r.ProjectID,
+            r.BookID,
+            LatestCreatedUtc = MAX(r.CreatedUtc),
+            LatestUpdatedUtc = MAX(r.UpdatedUtc)
+        FROM dbo.StoryIntelligenceRuns r
+        INNER JOIN dbo.Projects p ON p.ProjectID = r.ProjectID AND p.IsArchived = 0
+        INNER JOIN dbo.Books b ON b.BookID = r.BookID AND b.IsArchived = 0
+        INNER JOIN dbo.ProjectUserAccess access
+            ON access.ProjectID = r.ProjectID
+            AND access.UserID = @UserID
+            AND access.IsActive = 1
+        WHERE r.UserID = @UserID
+            AND r.BookID IS NOT NULL
+            AND r.Status IN (N'Pending', N'Running')
+        GROUP BY r.UserID, r.ProjectID, r.BookID
+    ),
+    PipelineScopes AS
+    (
+        SELECT
+            @UserID AS UserID,
+            p.ProjectID,
+            p.BookID,
+            LatestCreatedUtc = MAX(COALESCE(lastRun.CreatedUtc, p.CreatedUtc)),
+            LatestUpdatedUtc = MAX(CASE
+                WHEN lastRun.UpdatedUtc IS NOT NULL AND lastRun.UpdatedUtc > p.UpdatedUtc THEN lastRun.UpdatedUtc
+                ELSE p.UpdatedUtc
+            END)
+        FROM dbo.StoryIntelligenceBookPipelines p
+        INNER JOIN dbo.Projects project ON project.ProjectID = p.ProjectID AND project.IsArchived = 0
+        INNER JOIN dbo.Books book ON book.BookID = p.BookID AND book.IsArchived = 0
+        INNER JOIN dbo.ProjectUserAccess access
+            ON access.ProjectID = p.ProjectID
+            AND access.UserID = @UserID
+            AND access.IsActive = 1
+        LEFT JOIN dbo.StoryIntelligenceRuns lastRun ON lastRun.StoryIntelligenceRunID = p.LastRunID
+        WHERE p.Status = N'InProgress'
+        GROUP BY p.ProjectID, p.BookID
+    ),
+    ActiveScopes AS
+    (
+        SELECT
+            scopes.UserID,
+            scopes.ProjectID,
+            scopes.BookID,
+            LatestCreatedUtc = MAX(scopes.LatestCreatedUtc),
+            LatestUpdatedUtc = MAX(scopes.LatestUpdatedUtc)
+        FROM
+        (
+            SELECT UserID, ProjectID, BookID, LatestCreatedUtc, LatestUpdatedUtc FROM RunScopes
+            UNION ALL
+            SELECT UserID, ProjectID, BookID, LatestCreatedUtc, LatestUpdatedUtc FROM PipelineScopes
+        ) scopes
+        GROUP BY scopes.UserID, scopes.ProjectID, scopes.BookID
+    ),
+    ScopedRuns AS
+    (
+        SELECT r.*
+        FROM dbo.StoryIntelligenceRuns r
+        INNER JOIN ActiveScopes scope
+            ON scope.UserID = r.UserID
+            AND ISNULL(scope.ProjectID, -1) = ISNULL(r.ProjectID, -1)
+            AND ISNULL(scope.BookID, -1) = ISNULL(r.BookID, -1)
+            AND r.CreatedUtc >= DATEADD(minute, -30, scope.LatestCreatedUtc)
+        WHERE r.Status IN (N'Pending', N'Running', N'Completed', N'CompletedWithWarnings', N'Failed', N'Cancelled')
+    ),
+    CanonicalSceneScope AS
+    (
+        SELECT DISTINCT
+            scope.UserID,
+            scope.ProjectID,
+            scope.BookID,
+            s.SceneID
+        FROM ActiveScopes scope
+        INNER JOIN dbo.Chapters c
+            ON c.BookID = scope.BookID
+            AND c.IsArchived = 0
+        INNER JOIN dbo.Scenes s
+            ON s.ChapterID = c.ChapterID
+            AND s.IsArchived = 0
+    ),
+    RunSummary AS
+    (
+        SELECT
+            scope.UserID,
+            scope.ProjectID,
+            scope.BookID,
+            COUNT(r.StoryIntelligenceRunID) AS RunCount,
+            SUM(CASE WHEN r.Status = N'Pending' THEN 1 ELSE 0 END) AS PendingRunCount,
+            SUM(CASE WHEN r.Status = N'Running' THEN 1 ELSE 0 END) AS RunningRunCount,
+            SUM(CASE WHEN r.Status IN (N'Completed', N'CompletedWithWarnings') THEN 1 ELSE 0 END) AS CompletedRunCount,
+            SUM(CASE WHEN r.Status IN (N'Failed', N'Cancelled') THEN 1 ELSE 0 END) AS FailedRunCount,
+            SUM(ISNULL(r.TotalDetectedScenes, 0)) AS TotalDetectedScenes,
+            SUM(ISNULL(r.CompletedScenes, 0)) AS CompletedScenes,
+            SUM(ISNULL(r.FailedScenes, 0)) AS FailedScenes,
+            SUM(CASE
+                WHEN r.Status IN (N'Pending', N'Running')
+                    THEN CASE
+                        WHEN ISNULL(r.TotalDetectedScenes, 0) - ISNULL(r.CompletedScenes, 0) - ISNULL(r.FailedScenes, 0) > 0
+                            THEN ISNULL(r.TotalDetectedScenes, 0) - ISNULL(r.CompletedScenes, 0) - ISNULL(r.FailedScenes, 0)
+                        ELSE 0
+                    END
+                ELSE 0
+            END) AS ActiveScenes,
+            SUM(ISNULL(r.TotalDurationMs, 0)) AS TotalDurationMs,
+            MIN(COALESCE(r.CreatedUtc, scope.LatestCreatedUtc)) AS FirstCreatedUtc,
+            MAX(CASE
+                WHEN r.UpdatedUtc IS NOT NULL AND r.UpdatedUtc > scope.LatestUpdatedUtc THEN r.UpdatedUtc
+                ELSE scope.LatestUpdatedUtc
+            END) AS UpdatedUtc
+        FROM ActiveScopes scope
+        LEFT JOIN ScopedRuns r
+            ON r.UserID = scope.UserID
+            AND ISNULL(r.ProjectID, -1) = ISNULL(scope.ProjectID, -1)
+            AND ISNULL(r.BookID, -1) = ISNULL(scope.BookID, -1)
+        GROUP BY scope.UserID, scope.ProjectID, scope.BookID
+    ),
+    ExpectedSceneSummary AS
+    (
+        SELECT
+            scope.UserID,
+            scope.ProjectID,
+            scope.BookID,
+            COUNT(scope.SceneID) AS TotalExpectedScenes
+        FROM CanonicalSceneScope scope
+        GROUP BY scope.UserID, scope.ProjectID, scope.BookID
+    )
+    SELECT
+        summary.UserID,
+        summary.ProjectID,
+        p.ProjectName AS ProjectTitle,
+        summary.BookID,
+        b.BookTitle,
+        b.Subtitle AS BookSubtitle,
+        summary.RunCount,
+        summary.PendingRunCount,
+        summary.RunningRunCount,
+        summary.CompletedRunCount,
+        summary.FailedRunCount,
+        COALESCE(expected.TotalExpectedScenes, NULLIF(summary.TotalDetectedScenes, 0), 0) AS TotalExpectedScenes,
+        summary.TotalDetectedScenes,
+        summary.CompletedScenes,
+        summary.FailedScenes,
+        summary.ActiveScenes,
+        CASE
+            WHEN COALESCE(expected.TotalExpectedScenes, NULLIF(summary.TotalDetectedScenes, 0), 0) <= 0 THEN 0
+            ELSE CONVERT(int, ROUND(summary.CompletedScenes * 100.0 / COALESCE(expected.TotalExpectedScenes, NULLIF(summary.TotalDetectedScenes, 0), 1), 0))
+        END AS ProgressPercent,
+        CASE
+            WHEN pipeline.Status = N'InProgress'
+                AND summary.RunningRunCount = 0
+                AND summary.PendingRunCount = 0
+                THEN N'Finalising'
+            WHEN pipeline.Status = N'InProgress'
+                AND COALESCE(expected.TotalExpectedScenes, NULLIF(summary.TotalDetectedScenes, 0), 0) > 0
+                AND summary.CompletedScenes + summary.FailedScenes >= COALESCE(expected.TotalExpectedScenes, NULLIF(summary.TotalDetectedScenes, 0), 0)
+                THEN N'Finalising'
+            WHEN summary.RunningRunCount > 0 THEN N'Running'
+            WHEN summary.PendingRunCount > 0 THEN N'Pending'
+            WHEN summary.FailedRunCount > 0 THEN N'Failed'
+            WHEN summary.FailedScenes > 0 THEN N'CompletedWithWarnings'
+            ELSE N'Completed'
+        END AS Status,
+        summary.TotalDurationMs,
+        summary.FirstCreatedUtc,
+        summary.UpdatedUtc
+    FROM RunSummary summary
+    LEFT JOIN ExpectedSceneSummary expected
+        ON expected.UserID = summary.UserID
+        AND ISNULL(expected.ProjectID, -1) = ISNULL(summary.ProjectID, -1)
+        AND ISNULL(expected.BookID, -1) = ISNULL(summary.BookID, -1)
+    LEFT JOIN dbo.StoryIntelligenceBookPipelines pipeline
+        ON pipeline.BookID = summary.BookID
+        AND pipeline.ProjectID = summary.ProjectID
+    LEFT JOIN dbo.Projects p ON p.ProjectID = summary.ProjectID
+    LEFT JOIN dbo.Books b ON b.BookID = summary.BookID
+    WHERE summary.PendingRunCount > 0
+        OR summary.RunningRunCount > 0
+        OR pipeline.Status = N'InProgress'
+    ORDER BY summary.UpdatedUtc DESC, summary.FirstCreatedUtc;
+END;
+GO

PlotLine/ViewModels/StoryIntelligenceGlobalProgressViewModels.cs

@@ -16,12 +16,13 @@ public sealed class StoryIntelligenceGlobalProgressViewModel
     public int ProgressPercent => TotalExpectedScenes <= 0
         ? 0
         : Math.Clamp(Convert.ToInt32(Math.Round(CompletedScenes * 100m / TotalExpectedScenes)), 0, 100);
+    public DateTime? UpdatedUtc => Books.Count == 0 ? null : Books.Max(book => book.UpdatedUtc);
     public string Status => Books.Any(book => book.Status == "Finalising")
         ? "Finalising"
-        : Books.Any(book => book.HasFailures)
-            ? StoryIntelligenceRunStatuses.CompletedWithWarnings
         : Books.Any(book => book.HasActiveRuns)
             ? StoryIntelligenceRunStatuses.Running
+            : Books.Any(book => book.HasFailures)
+                ? StoryIntelligenceRunStatuses.CompletedWithWarnings
                 : StoryIntelligenceRunStatuses.Completed;
     public string SummaryText => ActiveBookCount switch
     {
@@ -50,7 +51,9 @@ public sealed class StoryIntelligenceGlobalProgressBookViewModel
     public long TotalDurationMs { get; init; }
     public DateTime UpdatedUtc { get; init; }
 
-    public bool HasActiveRuns => PendingRunCount > 0 || RunningRunCount > 0;
+    public bool HasActiveRuns => PendingRunCount > 0
+        || RunningRunCount > 0
+        || string.Equals(Status, "Finalising", StringComparison.OrdinalIgnoreCase);
     public bool HasFailures => FailedRunCount > 0 || FailedScenes > 0;
     public int ProgressPercent => TotalExpectedScenes <= 0
         ? 0

PlotLine/Views/Shared/_Layout.cshtml

@@ -231,11 +231,13 @@
         @if (storyIntelligenceProgress.HasActiveWork)
         {
             var primaryBook = storyIntelligenceProgress.Books.FirstOrDefault(book => book.HasActiveRuns);
+            var progressUpdatedUtc = storyIntelligenceProgress.UpdatedUtc?.ToUniversalTime().ToString("O", System.Globalization.CultureInfo.InvariantCulture);
             <div class="story-intelligence-global-strip"
                  data-story-intelligence-global-strip
                  data-progress-percent="@storyIntelligenceProgress.ProgressPercent"
                  data-total-expected-scenes="@storyIntelligenceProgress.TotalExpectedScenes"
                  data-active-books="@storyIntelligenceProgress.ActiveBookCount"
+                 data-story-intelligence-updated-utc="@progressUpdatedUtc"
                  data-book-id="@primaryBook?.BookID"
                  data-project-id="@primaryBook?.ProjectID"
                  role="status"

PlotLine/wwwroot/js/story-intelligence-global-progress.js

@@ -8,6 +8,7 @@
   const detail = strip.querySelector("[data-story-intelligence-global-detail]");
   const bar = strip.querySelector("[data-story-intelligence-global-bar]");
   let refreshTimer = null;
+  let lastSnapshotTime = Date.parse(strip.dataset.storyIntelligenceUpdatedUtc || "") || 0;
   const readNumber = (value) => {
     const parsed = Number.parseInt(value || "", 10);
     return Number.isInteger(parsed) ? parsed : null;
@@ -40,12 +41,24 @@
   const render = (progress) => {
     const books = progress?.books || progress?.Books || [];
     if (!books.length) {
-      strip.hidden = true;
+      strip.dataset.connection = "waiting";
       return;
     }
 
-    strip.hidden = false;
     const primary = books.find((book) => book.hasActiveRuns || book.HasActiveRuns) || books[0];
+    const updatedRaw = progress.updatedUtc || progress.UpdatedUtc || primary.updatedUtc || primary.UpdatedUtc || "";
+    const updatedTime = Date.parse(updatedRaw);
+    if (Number.isFinite(updatedTime) && updatedTime < lastSnapshotTime) {
+      return;
+    }
+
+    if (Number.isFinite(updatedTime)) {
+      lastSnapshotTime = updatedTime;
+      strip.dataset.storyIntelligenceUpdatedUtc = new Date(updatedTime).toISOString();
+    }
+
+    strip.hidden = false;
+    strip.dataset.connection = "live";
     const activeBookCount = readNumber(progress.activeBookCount ?? progress.ActiveBookCount) || books.length;
     const status = progress.status || progress.Status || primary.status || primary.Status || "";
     const total = readNumber(progress.totalExpectedScenes ?? progress.TotalExpectedScenes) || 0;
@@ -75,9 +88,6 @@
         title.textContent = "Story Intelligence cancelled";
       } else if (status === "Completed" || status === "CompletedWithWarnings") {
         title.textContent = failed > 0 ? "Story Intelligence completed with notes" : "Story Intelligence complete";
-        window.setTimeout(() => {
-          strip.hidden = true;
-        }, 10000);
       } else if (activeBookCount > 1) {
         title.textContent = `${formatCount(activeBookCount)} books analysing`;
       } else {