Phase 20R.1

PlotLine/Data/StoryIntelligenceResultRepository.cs

@@ -172,6 +172,7 @@ public sealed class StoryIntelligenceResultRepository(ISqlConnectionFactory conn
                 request.SourceFileSizeBytes,
                 request.SourceWordCount,
                 request.SourceCharacterCount,
+                request.SourceParagraphCount,
                 request.SourceChapterCount,
                 request.Model,
                 request.PromptVersionsSummary

PlotLine/Docs/AI/Chapter-Structure-Prompt-V1.md

@@ -37,11 +37,23 @@ Do not create boundaries for ordinary paragraph changes, small action beats, dia
 
 ## Paragraph Numbering
 
-Treat the supplied chapter text as a sequence of non-empty paragraphs.
+The supplied chapter text is already split into numbered paragraphs.
 
-The first non-empty paragraph is paragraph `1`.
+Each paragraph begins with an explicit marker such as:
 
-Each `startParagraph` and `endParagraph` must refer to these paragraph numbers.
+```text
+[1] First paragraph text...
+[2] Second paragraph text...
+[3] Third paragraph text...
+```
+
+Use only the supplied paragraph numbers.
+
+Each `startParagraph` and `endParagraph` must refer to those explicit paragraph numbers.
+
+Never invent paragraph numbers.
+
+Never return an `endParagraph` greater than the highest paragraph number supplied.
 
 Scene boundaries must:
 
@@ -49,6 +61,7 @@ Scene boundaries must:
 - cover the chapter in order;
 - not overlap;
 - not leave gaps;
+- be contiguous and non-overlapping where possible;
 - use increasing `sceneNumber` values starting at `1`.
 
 ## Output Requirements

PlotLine/Models/StoryIntelligencePersistenceModels.cs

@@ -32,6 +32,7 @@ public sealed class StoryIntelligenceRunSaveRequest
     public long? SourceFileSizeBytes { get; init; }
     public int? SourceWordCount { get; init; }
     public int? SourceCharacterCount { get; init; }
+    public int? SourceParagraphCount { get; init; }
     public int? SourceChapterCount { get; init; }
     public int? SourceDetectedImagesCount { get; init; }
     public int? SourceDetectedTablesCount { get; init; }
@@ -69,6 +70,7 @@ public sealed class StoryIntelligenceRunQueueRequest
     public long? SourceFileSizeBytes { get; init; }
     public int? SourceWordCount { get; init; }
     public int? SourceCharacterCount { get; init; }
+    public int? SourceParagraphCount { get; init; }
     public int? SourceChapterCount { get; init; }
     public string Model { get; init; } = string.Empty;
     public string PromptVersionsSummary { get; init; } = string.Empty;
@@ -156,6 +158,7 @@ public sealed class StoryIntelligenceSavedRun
     public long? SourceFileSizeBytes { get; init; }
     public int? SourceWordCount { get; init; }
     public int? SourceCharacterCount { get; init; }
+    public int? SourceParagraphCount { get; init; }
     public int? SourceChapterCount { get; init; }
     public int? SourceDetectedImagesCount { get; init; }
     public int? SourceDetectedTablesCount { get; init; }
@@ -200,6 +203,7 @@ public sealed class StoryIntelligenceQueuedRun
     public string? SourceText { get; init; }
     public int? SourceWordCount { get; init; }
     public int? SourceCharacterCount { get; init; }
+    public int? SourceParagraphCount { get; init; }
     public int? SourceChapterCount { get; init; }
     public string PromptVersion { get; init; } = string.Empty;
     public string? PromptVersionsSummary { get; init; }

PlotLine/Services/ChapterStoryIntelligenceDryRunService.cs

@@ -39,7 +39,7 @@ public sealed class ChapterStoryIntelligenceDryRunService(
         var chapterPromptVersion = versions.GetPromptVersion(ChapterPromptFile);
         var scenePromptVersion = versions.GetPromptVersion(ScenePromptFile);
         var clientStatus = client.GetConfigurationStatus();
-        var paragraphs = SplitParagraphs(form.ChapterText);
+        var paragraphs = StoryIntelligenceParagraphs.Split(form.ChapterText);
         var chapterContextJson = BuildChapterContextJson(form);
 
         try
@@ -50,7 +50,8 @@ public sealed class ChapterStoryIntelligenceDryRunService(
             }
 
             var chapterTemplate = await prompts.LoadPromptAsync(ChapterPromptFile, cancellationToken);
-            var chapterPrompt = BuildChapterPrompt(chapterTemplate, chapterContextJson, form.ChapterText);
+            var numberedChapterText = StoryIntelligenceParagraphs.Number(paragraphs);
+            var chapterPrompt = BuildChapterPrompt(chapterTemplate, chapterContextJson, numberedChapterText);
             var chapterResult = await client.ExecutePromptAsync(chapterPrompt, chapterPromptVersion, cancellationToken);
             var chapterJson = ExtractOutputText(chapterResult.RawResponseText);
             var parsedChapter = JsonSerializer.Deserialize<ChapterStructureModel>(chapterJson, JsonOptions);
@@ -313,12 +314,6 @@ public sealed class ChapterStoryIntelligenceDryRunService(
         return outputText.Trim();
     }
 
-    private static List<string> SplitParagraphs(string chapterText)
-        => chapterText.Split(["\r\n\r\n", "\n\n", "\r\r"], StringSplitOptions.RemoveEmptyEntries)
-            .Select(paragraph => paragraph.Trim())
-            .Where(paragraph => !string.IsNullOrWhiteSpace(paragraph))
-            .ToList();
-
     private static string Preview(string value)
     {
         var cleaned = value.ReplaceLineEndings(" ").Trim();

PlotLine/Services/PersistedStoryIntelligenceRunner.cs

@@ -66,7 +66,7 @@ public sealed class PersistedStoryIntelligenceRunner(
             }
 
             await ThrowIfCancellationRequestedAsync(run.StoryIntelligenceRunID, stopwatch.ElapsedMilliseconds, cancellationToken);
-            var paragraphs = SplitParagraphs(run.SourceText);
+            var paragraphs = StoryIntelligenceParagraphs.Split(run.SourceText);
             if (paragraphs.Count == 0)
             {
                 throw new StoryIntelligenceRunFailureException(
@@ -76,7 +76,8 @@ public sealed class PersistedStoryIntelligenceRunner(
 
             var chapterContextJson = BuildChapterContextJson(run);
             var chapterTemplate = await prompts.LoadPromptAsync(ChapterPromptFile, cancellationToken);
-            var chapterPrompt = BuildChapterPrompt(chapterTemplate, chapterContextJson, run.SourceText);
+            var numberedChapterText = StoryIntelligenceParagraphs.Number(paragraphs);
+            var chapterPrompt = BuildChapterPrompt(chapterTemplate, chapterContextJson, numberedChapterText);
             currentFailureStage = StoryIntelligenceFailureStages.ChapterStructure;
 
             await repository.UpdateProgressAsync(
@@ -386,12 +387,6 @@ public sealed class PersistedStoryIntelligenceRunner(
             : outputText.Trim();
     }
 
-    private static List<string> SplitParagraphs(string chapterText)
-        => chapterText.Split(["\r\n\r\n", "\n\n", "\r\r"], StringSplitOptions.RemoveEmptyEntries)
-            .Select(paragraph => paragraph.Trim())
-            .Where(paragraph => !string.IsNullOrWhiteSpace(paragraph))
-            .ToList();
-
     private static string SerializeParsed<T>(T? value)
         => value is null ? string.Empty : JsonSerializer.Serialize(value, JsonOptions);
 

PlotLine/Services/StoryIntelligenceExistingChapterQueueService.cs

@@ -70,6 +70,7 @@ public sealed class StoryIntelligenceExistingChapterQueueService(
             ChapterText = sourceText,
             SourceWordCount = selected.SourceWordCount ?? CountWords(sourceText),
             SourceCharacterCount = selected.SourceCharacterCount ?? sourceText.Length,
+            SourceParagraphCount = StoryIntelligenceParagraphs.Split(sourceText).Count,
             SourceChapterCount = 1,
             Model = clientStatus.Model,
             PromptVersionsSummary = "Chapter=Chapter-Structure-Prompt-V1; Scene=Scene-Prompt-V1"

PlotLine/Services/StoryIntelligenceFullChapterTestService.cs

@@ -73,6 +73,7 @@ public sealed class StoryIntelligenceFullChapterTestService(
             SourceFileSizeBytes = source.SourceFileSizeBytes,
             SourceWordCount = preflight.WordCount,
             SourceCharacterCount = preflight.CharacterCount,
+            SourceParagraphCount = preflight.ParagraphCount,
             SourceChapterCount = 1,
             Model = clientStatus.Model,
             PromptVersionsSummary = PromptVersionsSummary
@@ -117,7 +118,7 @@ public sealed class StoryIntelligenceFullChapterTestService(
     {
         var characterCount = text.Length;
         var wordCount = CountWords(text);
-        var paragraphCount = SplitParagraphs(text).Count;
+        var paragraphCount = StoryIntelligenceParagraphs.Split(text).Count;
         var estimatedInputTokens = EstimateTokens(text);
         var estimatedPipelineInputTokens = estimatedInputTokens * 2;
         var warnings = BuildWarnings(text, characterCount, paragraphCount);
@@ -211,12 +212,6 @@ public sealed class StoryIntelligenceFullChapterTestService(
             ? 0
             : value.Split((char[]?)null, StringSplitOptions.RemoveEmptyEntries).Length;
 
-    private static IReadOnlyList<string> SplitParagraphs(string value)
-        => value.Split(["\r\n\r\n", "\n\n", "\r\r"], StringSplitOptions.RemoveEmptyEntries)
-            .Select(paragraph => paragraph.Trim())
-            .Where(paragraph => !string.IsNullOrWhiteSpace(paragraph))
-            .ToList();
-
     private static int EstimateTokens(string value)
         => Math.Max(1, (int)Math.Ceiling(value.Length / 4m));
 

PlotLine/Services/StoryIntelligenceOpenAIInfrastructure.cs

@@ -517,7 +517,8 @@ public sealed class ChapterStructureDryRunService(
         var promptVersion = versions.GetPromptVersion(ChapterPromptFile);
         var clientStatus = client.GetConfigurationStatus();
         var contextJson = BuildChapterContextJson(form);
-        var paragraphCount = CountParagraphs(form.ChapterText);
+        var paragraphs = StoryIntelligenceParagraphs.Split(form.ChapterText);
+        var paragraphCount = paragraphs.Count;
 
         try
         {
@@ -527,7 +528,8 @@ public sealed class ChapterStructureDryRunService(
             }
 
             var template = await prompts.LoadPromptAsync(ChapterPromptFile, cancellationToken);
-            var completedPrompt = BuildChapterPrompt(template, contextJson, form.ChapterText);
+            var numberedChapterText = StoryIntelligenceParagraphs.Number(paragraphs);
+            var completedPrompt = BuildChapterPrompt(template, contextJson, numberedChapterText);
             var result = await client.ExecutePromptAsync(completedPrompt, promptVersion, cancellationToken);
             var chapterJson = ExtractOutputText(result.RawResponseText);
             var parsedChapter = DeserializeChapter(chapterJson);
@@ -649,10 +651,6 @@ public sealed class ChapterStructureDryRunService(
     private static ChapterStructureModel? DeserializeChapter(string chapterJson)
         => JsonSerializer.Deserialize<ChapterStructureModel>(chapterJson, JsonOptions);
 
-    private static int CountParagraphs(string chapterText)
-        => chapterText.Split(["\r\n\r\n", "\n\n", "\r\r"], StringSplitOptions.RemoveEmptyEntries)
-            .Count(paragraph => !string.IsNullOrWhiteSpace(paragraph));
-
     private static string FormatDuration(TimeSpan duration)
         => duration.TotalSeconds < 1
             ? $"{duration.TotalMilliseconds:0} ms"

PlotLine/Services/StoryIntelligenceParagraphs.cs

@@ -0,0 +1,33 @@
+using System.Text;
+
+namespace PlotLine.Services;
+
+public static class StoryIntelligenceParagraphs
+{
+    public static IReadOnlyList<string> Split(string? chapterText)
+        => (chapterText ?? string.Empty)
+            .Split(["\r\n\r\n", "\n\n", "\r\r"], StringSplitOptions.RemoveEmptyEntries)
+            .Select(paragraph => paragraph.Trim())
+            .Where(paragraph => !string.IsNullOrWhiteSpace(paragraph))
+            .ToList();
+
+    public static string Number(IReadOnlyList<string> paragraphs)
+    {
+        var builder = new StringBuilder();
+        for (var index = 0; index < paragraphs.Count; index++)
+        {
+            if (index > 0)
+            {
+                builder.AppendLine();
+                builder.AppendLine();
+            }
+
+            builder.Append('[')
+                .Append(index + 1)
+                .Append("] ")
+                .Append(paragraphs[index]);
+        }
+
+        return builder.ToString();
+    }
+}

PlotLine/Services/StoryIntelligenceResultPersistenceService.cs

@@ -81,6 +81,7 @@ public sealed class StoryIntelligenceResultPersistenceService(
             ChapterText = form.ChapterText,
             SourceWordCount = CountWords(form.ChapterText),
             SourceCharacterCount = form.ChapterText.Length,
+            SourceParagraphCount = StoryIntelligenceParagraphs.Split(form.ChapterText).Count,
             SourceChapterCount = 1,
             Model = clientStatus.Model,
             PromptVersionsSummary = promptVersionsSummary

PlotLine/Sql/120_Phase20R_DeterministicChapterParagraphs.sql

@@ -0,0 +1,102 @@
+IF COL_LENGTH(N'dbo.StoryIntelligenceRuns', N'SourceParagraphCount') IS NULL
+BEGIN
+    ALTER TABLE dbo.StoryIntelligenceRuns ADD SourceParagraphCount int NULL;
+END;
+GO
+
+CREATE OR ALTER PROCEDURE dbo.StoryIntelligenceRun_QueueAdminText
+    @UserID int,
+    @ProjectID int = NULL,
+    @BookID int = NULL,
+    @ChapterID int = NULL,
+    @ChapterNumber decimal(9, 2) = NULL,
+    @SourceType nvarchar(50),
+    @SourceLabel nvarchar(300),
+    @SourceText nvarchar(max),
+    @SourceFileName nvarchar(260) = NULL,
+    @SourceFileSizeBytes bigint = NULL,
+    @SourceWordCount int = NULL,
+    @SourceCharacterCount int = NULL,
+    @SourceParagraphCount int = NULL,
+    @SourceChapterCount int = NULL,
+    @Model nvarchar(100),
+    @PromptVersionsSummary nvarchar(500)
+AS
+BEGIN
+    SET NOCOUNT ON;
+
+    INSERT dbo.StoryIntelligenceRuns
+    (
+        UserID, ProjectID, BookID, ChapterID, ChapterNumber, Status, SourceType, SourceLabel, SourceText,
+        SourceFileName, SourceFileSizeBytes, SourceWordCount, SourceCharacterCount, SourceParagraphCount,
+        SourceChapterCount, Model, PromptVersion, PromptVersionsSummary, StartedUtc, CurrentStage, CurrentMessage,
+        CompletedScenes, FailedScenes
+    )
+    VALUES
+    (
+        @UserID, @ProjectID, @BookID, @ChapterID, @ChapterNumber, N'Pending', @SourceType, @SourceLabel, @SourceText,
+        @SourceFileName, @SourceFileSizeBytes, @SourceWordCount, @SourceCharacterCount, @SourceParagraphCount,
+        @SourceChapterCount, @Model, @PromptVersionsSummary, @PromptVersionsSummary, SYSUTCDATETIME(), N'Pending',
+        N'Queued for Story Intelligence processing.', 0, 0
+    );
+
+    SELECT CAST(SCOPE_IDENTITY() AS int) AS StoryIntelligenceRunID;
+END;
+GO
+
+CREATE OR ALTER PROCEDURE dbo.StoryIntelligenceRun_GetAdmin
+    @StoryIntelligenceRunID int
+AS
+BEGIN
+    SET NOCOUNT ON;
+
+    SELECT
+        r.StoryIntelligenceRunID,
+        r.UserID,
+        r.ProjectID,
+        p.ProjectName AS ProjectTitle,
+        r.BookID,
+        b.BookTitle,
+        r.ChapterID,
+        r.ChapterNumber,
+        r.Status,
+        r.SourceType,
+        r.SourceFileName,
+        r.SourceFileSizeBytes,
+        r.SourceWordCount,
+        r.SourceCharacterCount,
+        r.SourceParagraphCount,
+        r.SourceChapterCount,
+        r.SourceDetectedImagesCount,
+        r.SourceDetectedTablesCount,
+        r.SourceDetectedFootnotesCount,
+        r.SourceDetectedCommentsCount,
+        r.CurrentStage,
+        r.CurrentMessage,
+        r.TotalDetectedScenes,
+        r.CompletedScenes,
+        r.FailedScenes,
+        r.CancellationRequestedUtc,
+        r.CancelledUtc,
+        r.PromptVersion,
+        r.PromptVersionsSummary,
+        r.Model,
+        r.StartedUtc,
+        r.CompletedUtc,
+        r.FailureStage,
+        r.TotalInputTokens,
+        r.TotalOutputTokens,
+        r.TotalTokens,
+        r.TotalDurationMs,
+        r.EstimatedCostGBP,
+        r.EstimatedCostUSD,
+        r.ErrorMessage,
+        r.ErrorDetail,
+        r.CreatedUtc,
+        r.UpdatedUtc
+    FROM dbo.StoryIntelligenceRuns r
+    LEFT JOIN dbo.Projects p ON p.ProjectID = r.ProjectID
+    LEFT JOIN dbo.Books b ON b.BookID = r.BookID
+    WHERE r.StoryIntelligenceRunID = @StoryIntelligenceRunID;
+END;
+GO

PlotLine/Views/Admin/StoryIntelligenceRunDetails.cshtml

@@ -93,6 +93,8 @@ else
             <dd>@Display(run.SourceWordCount)</dd>
             <dt>Source character count</dt>
             <dd>@Display(run.SourceCharacterCount)</dd>
+            <dt>Submitted paragraph count</dt>
+            <dd>@Display(run.SourceParagraphCount)</dd>
             <dt>Source chapter count</dt>
             <dd>@Display(run.SourceChapterCount)</dd>
             <dt>Detected document features</dt>
@@ -169,6 +171,18 @@ else
             </dl>
 
             <label class="form-label" for="chapterParsedJson">Parsed JSON</label>
+            <dl>
+                <dt>Highest returned endParagraph</dt>
+                <dd>@Display(HighestReturnedEndParagraph(chapter))</dd>
+                <dt>Returned scene boundary ranges</dt>
+                <dd>@DisplayBoundaryRanges(chapter)</dd>
+            </dl>
+            @if (ReturnedEndParagraphExceedsSubmitted(chapter, run))
+            {
+                <div class="alert alert-warning">
+                    The model returned paragraph numbers beyond the submitted chapter. This usually means the prompt was not using explicit numbered paragraphs.
+                </div>
+            }
             <textarea class="form-control font-monospace" id="chapterParsedJson" rows="10" readonly>@chapter.ParsedJson</textarea>
             <label class="form-label mt-3" for="chapterOutputJson">Extracted output JSON</label>
             <textarea class="form-control font-monospace" id="chapterOutputJson" rows="10" readonly>@chapter.OutputTextJson</textarea>
@@ -300,4 +314,58 @@ else
 
         return "Scene failed. See parsed JSON for details.";
     }
+
+    private static int? HighestReturnedEndParagraph(StoryIntelligenceSavedChapterResult chapter)
+        => ReadBoundaryRanges(chapter).Select(range => range.End).DefaultIfEmpty().Max() is var max && max > 0 ? max : null;
+
+    private static string DisplayBoundaryRanges(StoryIntelligenceSavedChapterResult chapter)
+    {
+        var ranges = ReadBoundaryRanges(chapter).ToList();
+        return ranges.Count == 0
+            ? "None"
+            : string.Join(", ", ranges.Select(range => $"{range.Start}-{range.End}"));
+    }
+
+    private static bool ReturnedEndParagraphExceedsSubmitted(StoryIntelligenceSavedChapterResult chapter, StoryIntelligenceSavedRun run)
+    {
+        var highest = HighestReturnedEndParagraph(chapter);
+        var submitted = run.SourceParagraphCount;
+        return highest.HasValue && submitted.HasValue && highest.Value > submitted.Value;
+    }
+
+    private static IReadOnlyList<(int Start, int End)> ReadBoundaryRanges(StoryIntelligenceSavedChapterResult chapter)
+    {
+        if (string.IsNullOrWhiteSpace(chapter.ParsedJson))
+        {
+            return [];
+        }
+
+        try
+        {
+            using var document = System.Text.Json.JsonDocument.Parse(chapter.ParsedJson);
+            if (!document.RootElement.TryGetProperty("sceneBoundaries", out var boundaries)
+                || boundaries.ValueKind != System.Text.Json.JsonValueKind.Array)
+            {
+                return [];
+            }
+
+            var ranges = new List<(int Start, int End)>();
+            foreach (var boundary in boundaries.EnumerateArray())
+            {
+                var start = boundary.TryGetProperty("startParagraph", out var startElement) && startElement.TryGetInt32(out var startValue)
+                    ? startValue
+                    : 0;
+                var end = boundary.TryGetProperty("endParagraph", out var endElement) && endElement.TryGetInt32(out var endValue)
+                    ? endValue
+                    : 0;
+                ranges.Add((start, end));
+            }
+
+            return ranges;
+        }
+        catch (System.Text.Json.JsonException)
+        {
+            return [];
+        }
+    }
 }