Fix structural Word paragraph mapping

PlotLine.Tests/Program.cs

@@ -35,6 +35,10 @@ var tests = new (string Name, Action Test)[]
     ("Chapter Structure boundary normaliser replays full book terminal empty failure", ChapterStructureBoundaryNormaliserReplaysFullBookTerminalEmptyFailure),
     ("Chapter Structure boundary normaliser keeps meaningful end overflow repair", ChapterStructureBoundaryNormaliserKeepsMeaningfulEndOverflowRepair),
     ("Chapter Structure boundary normaliser preserves genuine inverted ranges for validation", ChapterStructureBoundaryNormaliserPreservesGenuineInvertedRanges),
+    ("Structural Word mapping uses paragraph map as authoritative text", StructuralWordMappingUsesParagraphMapAsAuthoritativeText),
+    ("Structural Word mapping includes filtered Word paragraphs in source coverage", StructuralWordMappingIncludesFilteredWordParagraphsInSourceCoverage),
+    ("Structural Word mapping fails precisely when map metadata is incomplete", StructuralWordMappingFailsPreciselyWhenMapMetadataIncomplete),
+    ("Structural Word mapping keeps trailing empty normalisation mapped", StructuralWordMappingKeepsTrailingEmptyNormalisationMapped),
     ("Shared parser reports raw chapter output on unrecoverable JSON", SharedParserReportsRawOutput),
     ("Character filtering rejects generic groups", CharacterFilteringRejectsGenericGroups),
     ("Character filtering preserves titled names", CharacterFilteringPreservesTitledNames),
@@ -1937,6 +1941,115 @@ static void ChapterStructureBoundaryNormaliserPreservesGenuineInvertedRanges()
     Assert(validation.Errors.Any(error => error.Message.Contains("Start paragraph cannot be after end paragraph", StringComparison.Ordinal)), "Validation should report the inverted range.");
 }
 
+static void StructuralWordMappingUsesParagraphMapAsAuthoritativeText()
+{
+    var request = MappedStructureRequest(
+        [
+            (1, 100, "Repeated words with an embedded line break are still one Word paragraph."),
+            (2, 101, "Repeated words with an embedded line break are still one Word paragraph."),
+            (3, 102, "Straight quote text stays mapped to a single Word paragraph."),
+            (4, 103, "Curly quote text stays mapped to a single Word paragraph."),
+            (5, 104, "Final paragraph maps directly back to Word.")
+        ],
+        sourceParagraphs:
+        [
+            SourceParagraph(99, "- A Date with Disappointment", include: false, heading: true),
+            SourceParagraph(100, "Repeated words with an embedded line break are still one Word paragraph."),
+            SourceParagraph(101, "Repeated words with an embedded line break are still one Word paragraph."),
+            SourceParagraph(102, "Straight quote text stays mapped to a single Word paragraph."),
+            SourceParagraph(103, "Curly quote text stays mapped to a single Word paragraph."),
+            SourceParagraph(104, "Final paragraph maps directly back to Word.")
+        ],
+        chapterText: string.Join("\n\n",
+        [
+            "Repeated words with an embedded\nline break are still one Word paragraph.",
+            "Repeated words with an embedded line break are still one Word paragraph.",
+            "Straight quote text stays mapped to a single Word paragraph.",
+            "Curly quote text stays mapped to a single Word paragraph.",
+            "Final paragraph maps directly back to Word."
+        ]));
+    var client = new FakeStoryIntelligenceClient(ChapterStructureResponse(Boundary(1, 1, 4), Boundary(2, 5, 5)));
+    var service = BuildMappedStructureAnalysisService(client);
+
+    var result = service.AnalyseChapterAsync(request).GetAwaiter().GetResult();
+
+    Assert(result.Success, result.ErrorMessage ?? "Mapped analysis should succeed.");
+    Assert(client.Prompts.Single().Contains("[5] Final paragraph maps directly back to Word.", StringComparison.Ordinal), "Prompt should be numbered from paragraph map metadata.");
+    Assert(!client.Prompts.Single().Contains("[6]", StringComparison.Ordinal), "Embedded line breaks inside a Word paragraph must not create unmapped structural paragraphs.");
+    Assert(result.Scenes.Count == 2, "Mapped analysis should return both scenes.");
+    Assert(result.Scenes[1].WordStartPosition == 104 && result.Scenes[1].WordEndPosition == 104, "Scene 2 should map structural paragraph 5 directly to Word paragraph 104.");
+}
+
+static void StructuralWordMappingIncludesFilteredWordParagraphsInSourceCoverage()
+{
+    var request = MappedStructureRequest(
+        [
+            (1, 50, "Scene opening paragraph with enough words for mapped structural analysis."),
+            (2, 53, "Scene closing paragraph with enough words for mapped structural analysis.")
+        ],
+        sourceParagraphs:
+        [
+            SourceParagraph(49, "Chapter title excluded from structure.", include: false, heading: true),
+            SourceParagraph(50, "Scene opening paragraph with enough words for mapped structural analysis."),
+            SourceParagraph(51, "", blank: true),
+            SourceParagraph(52, "***", include: false, separator: true),
+            SourceParagraph(53, "Scene closing paragraph with enough words for mapped structural analysis.")
+        ]);
+    var service = BuildMappedStructureAnalysisService(new FakeStoryIntelligenceClient(ChapterStructureResponse(Boundary(1, 1, 2))));
+
+    var result = service.AnalyseChapterAsync(request).GetAwaiter().GetResult();
+
+    Assert(result.Success, result.ErrorMessage ?? "Mapped analysis should succeed across filtered Word paragraphs.");
+    var scene = result.Scenes.Single();
+    Assert(scene.WordStartPosition == 50 && scene.WordEndPosition == 53, "Scene endpoint mapping should span filtered Word paragraphs between structural endpoints.");
+    Assert(scene.SourceText.Contains("Scene opening paragraph", StringComparison.Ordinal), "Source text should include the mapped start paragraph.");
+    Assert(scene.SourceText.Contains("Scene closing paragraph", StringComparison.Ordinal), "Source text should include the mapped end paragraph.");
+}
+
+static void StructuralWordMappingFailsPreciselyWhenMapMetadataIncomplete()
+{
+    var request = MappedStructureRequest(
+        [
+            (1, 10, "First mapped paragraph has enough words for the structure prompt."),
+            (3, 12, "Third mapped paragraph exposes the missing second structural index.")
+        ]);
+    var client = new FakeStoryIntelligenceClient(ChapterStructureResponse(Boundary(1, 1, 2)));
+    var service = BuildMappedStructureAnalysisService(client);
+
+    var result = service.AnalyseChapterAsync(request).GetAwaiter().GetResult();
+
+    Assert(!result.Success, "Missing structural paragraph map metadata should fail before invoking AI.");
+    Assert(client.Prompts.Count == 0, "Incomplete paragraph maps should not trigger OpenAI work.");
+    Assert(result.ErrorMessage?.Contains("missing structural paragraph 2", StringComparison.OrdinalIgnoreCase) == true, "Mapping failure should name the missing structural index.");
+    Assert(result.ErrorMessage?.Contains("nearest mapped neighbours", StringComparison.OrdinalIgnoreCase) == true, "Mapping failure should include nearest mapped neighbours.");
+}
+
+static void StructuralWordMappingKeepsTrailingEmptyNormalisationMapped()
+{
+    var request = MappedStructureRequest(
+        Enumerable.Range(1, 366)
+            .Select(index => (index, 990 + index, $"Paragraph {index} contains enough mapped manuscript words for structural analysis."))
+            .ToArray());
+    var service = BuildMappedStructureAnalysisService(new FakeStoryIntelligenceClient(ChapterStructureResponse(
+        Boundary(1, 1, 11),
+        Boundary(2, 12, 41),
+        Boundary(3, 42, 66),
+        Boundary(4, 67, 107),
+        Boundary(5, 108, 154),
+        Boundary(6, 155, 200),
+        Boundary(7, 201, 276),
+        Boundary(8, 277, 332),
+        Boundary(9, 333, 346),
+        Boundary(10, 347, 366),
+        Boundary(11, 367, 366))));
+
+    var result = service.AnalyseChapterAsync(request).GetAwaiter().GetResult();
+
+    Assert(result.Success, result.ErrorMessage ?? "Trailing empty normalisation should remain compatible with Word mapping.");
+    Assert(result.Scenes.Count == 10, "Trailing empty scene should be discarded before Word mapping.");
+    Assert(result.Scenes[9].WordStartPosition == 1337 && result.Scenes[9].WordEndPosition == 1356, "Final valid scene should preserve direct Word paragraph mapping.");
+}
+
 static ChapterStructureModel ChapterWithBoundaries(params ChapterSceneBoundary[] boundaries)
     => new()
     {
@@ -1957,6 +2070,74 @@ static ChapterSceneBoundary Boundary(int sceneNumber, int start, int end, string
         Reason = "Structural boundary identified from the submitted chapter text."
     };
 
+static ManuscriptStructureAnalysisService BuildMappedStructureAnalysisService(FakeStoryIntelligenceClient client)
+    => new(
+        new FakeStoryPromptRepository(),
+        new FakeStoryPromptVersionService(),
+        client,
+        new ChapterStructureValidator(),
+        NullLogger<ManuscriptStructureAnalysisService>.Instance);
+
+static ManuscriptStructureAnalysisRequest MappedStructureRequest(
+    IReadOnlyList<(int Structural, int Word, string Text)> map,
+    IReadOnlyList<ManuscriptScanSourceParagraph>? sourceParagraphs = null,
+    string? chapterText = null)
+    => new()
+    {
+        ProjectID = 3067,
+        BookID = 5075,
+        PreviewID = Guid.Parse("57eaf531-f483-45ae-87da-5cbb4a3ab16e"),
+        TemporaryChapterKey = "chapter-18",
+        ChapterNumber = 18,
+        SourceLabel = "Word manuscript: - A Date with Disappointment",
+        ChapterText = chapterText ?? string.Join(Environment.NewLine + Environment.NewLine, map.Select(item => item.Text)),
+        ParagraphMap = map
+            .Select(item => new ManuscriptScanParagraphMap
+            {
+                StructuralParagraphOrdinal = item.Structural,
+                WordParagraphIndex = item.Word,
+                Text = item.Text
+            })
+            .ToList(),
+        SourceParagraphs = sourceParagraphs ?? map.Select(item => SourceParagraph(item.Word, item.Text)).ToList()
+    };
+
+static ManuscriptScanSourceParagraph SourceParagraph(
+    int wordIndex,
+    string text,
+    bool include = true,
+    bool blank = false,
+    bool heading = false,
+    bool separator = false)
+    => new()
+    {
+        WordParagraphIndex = wordIndex,
+        Text = text,
+        IsBlank = blank,
+        IsChapterHeading = heading,
+        IsSceneSeparator = separator,
+        IncludeInSourceText = include
+    };
+
+static string ChapterStructureResponse(params ChapterSceneBoundary[] boundaries)
+    => JsonSerializer.Serialize(new
+    {
+        output = new[]
+        {
+            new
+            {
+                content = new[]
+                {
+                    new
+                    {
+                        type = "output_text",
+                        text = JsonSerializer.Serialize(ChapterWithBoundaries(boundaries), ManuscriptScanJsonOptions())
+                    }
+                }
+            }
+        }
+    });
+
 static void SharedParserReportsRawOutput()
 {
     const string raw = """{"schemaVersion":"1.0","sceneBoundaries":[""";
@@ -6273,6 +6454,51 @@ internal sealed class CaptureImageRequestHandler(Func<HttpRequestMessage, HttpRe
         => Task.FromResult(respond(request));
 }
 
+internal sealed class FakeStoryPromptRepository : IStoryPromptRepository
+{
+    public Task<string> LoadPromptAsync(string fileName, CancellationToken cancellationToken)
+        => Task.FromResult("Context: {{CHAPTER_CONTEXT_JSON}}\n\nText:\n{{CHAPTER_TEXT}}");
+}
+
+internal sealed class FakeStoryPromptVersionService : IStoryPromptVersionService
+{
+    public string GetPromptVersion(string fileName) => "Chapter-Structure-Prompt-V2";
+}
+
+internal sealed class FakeStoryIntelligenceClient(string rawResponseText) : IStoryIntelligenceClient
+{
+    public List<string> Prompts { get; } = [];
+
+    public Task<StoryIntelligenceClientResult> ExecutePromptAsync(
+        string completedPrompt,
+        string promptVersion,
+        CancellationToken cancellationToken,
+        string? modelOverride = null,
+        int? maxOutputTokensOverride = null,
+        StoryIntelligenceResponseContract? responseContract = null)
+    {
+        Prompts.Add(completedPrompt);
+        return Task.FromResult(new StoryIntelligenceClientResult
+        {
+            RawResponseText = rawResponseText,
+            Model = modelOverride ?? "test-model",
+            Duration = TimeSpan.FromMilliseconds(1)
+        });
+    }
+
+    public StoryIntelligenceClientConfigurationStatus GetConfigurationStatus()
+        => new()
+        {
+            ApiKeyConfigured = true,
+            Model = "test-model",
+            ChapterStructureModel = "test-structure-model",
+            SceneIntelligenceModel = "test-scene-model",
+            WholeBookPlotSynthesisModel = "test-whole-book-model",
+            ClientConstructed = true,
+            ConnectionStatus = "Configured"
+        };
+}
+
 internal sealed class FakeManuscriptTextStatisticsRepository(string sourceText) : IManuscriptTextStatisticsRepository
 {
     public int? SavedSceneID { get; private set; }

PlotLine/Services/ManuscriptStructureAnalysisService.cs

@@ -28,7 +28,15 @@ public sealed partial class ManuscriptStructureAnalysisService(
     public async Task<ManuscriptStructureAnalysisResult> AnalyseChapterAsync(ManuscriptStructureAnalysisRequest request, CancellationToken cancellationToken = default)
     {
         var text = request.ChapterText?.Trim() ?? string.Empty;
-        var paragraphs = StoryIntelligenceParagraphs.Split(text);
+        var structuralParagraphResult = BuildStructuralParagraphs(request, text);
+        if (!structuralParagraphResult.Success)
+        {
+            return ManuscriptStructureAnalysisResult.Failed(
+                request.TemporaryChapterKey,
+                structuralParagraphResult.ErrorMessage ?? "The approved chapter paragraph map is not reversible.");
+        }
+
+        var paragraphs = structuralParagraphResult.Paragraphs;
         if (paragraphs.Count == 0)
         {
             return ManuscriptStructureAnalysisResult.Failed(
@@ -141,9 +149,15 @@ public sealed partial class ManuscriptStructureAnalysisService(
                 var endMap = request.ParagraphMap.FirstOrDefault(item => item.StructuralParagraphOrdinal == end);
                 if (startMap is null || endMap is null || startMap.WordParagraphIndex < 0 || endMap.WordParagraphIndex < 0)
                 {
+                    var diagnostic = BuildMappingFailureDiagnostic(
+                        sceneNumber,
+                        start,
+                        end,
+                        request.ParagraphMap,
+                        request.SourceParagraphs);
                     return ManuscriptStructureAnalysisResult.Failed(
                         request.TemporaryChapterKey,
-                        $"Scene {sceneNumber:N0} could not be mapped back to a Word paragraph.",
+                        $"Scene {sceneNumber:N0} could not be mapped back to a Word paragraph. {diagnostic}",
                         persistedJson);
                 }
 
@@ -325,6 +339,66 @@ public sealed partial class ManuscriptStructureAnalysisService(
         return string.Join(Environment.NewLine + Environment.NewLine, lines);
     }
 
+    private static StructuralParagraphBuildResult BuildStructuralParagraphs(ManuscriptStructureAnalysisRequest request, string chapterText)
+    {
+        if (request.ParagraphMap.Count == 0)
+        {
+            return StructuralParagraphBuildResult.Succeeded(StoryIntelligenceParagraphs.Split(chapterText));
+        }
+
+        var ordered = request.ParagraphMap
+            .OrderBy(item => item.StructuralParagraphOrdinal)
+            .ToList();
+        var expectedOrdinal = 1;
+        var previousWordIndex = -1;
+        var paragraphs = new List<string>();
+        foreach (var item in ordered)
+        {
+            if (item.StructuralParagraphOrdinal != expectedOrdinal)
+            {
+                return StructuralParagraphBuildResult.Failed(
+                    $"Paragraph map is missing structural paragraph {expectedOrdinal:N0}. {BuildMapNeighbourDiagnostic(expectedOrdinal, ordered, request.SourceParagraphs)}");
+            }
+
+            if (item.WordParagraphIndex < 0)
+            {
+                return StructuralParagraphBuildResult.Failed(
+                    $"Structural paragraph {expectedOrdinal:N0} has no Word paragraph index. {BuildMapNeighbourDiagnostic(expectedOrdinal, ordered, request.SourceParagraphs)}");
+            }
+
+            if (item.WordParagraphIndex <= previousWordIndex)
+            {
+                return StructuralParagraphBuildResult.Failed(
+                    $"Structural paragraph {expectedOrdinal:N0} maps to non-monotonic Word paragraph {item.WordParagraphIndex:N0}. {BuildMapNeighbourDiagnostic(expectedOrdinal, ordered, request.SourceParagraphs)}");
+            }
+
+            var text = NormalizeMappedParagraphText(item.Text);
+            if (string.IsNullOrWhiteSpace(text))
+            {
+                return StructuralParagraphBuildResult.Failed(
+                    $"Structural paragraph {expectedOrdinal:N0} has no mapped text. {BuildMapNeighbourDiagnostic(expectedOrdinal, ordered, request.SourceParagraphs)}");
+            }
+
+            paragraphs.Add(text);
+            previousWordIndex = item.WordParagraphIndex;
+            expectedOrdinal++;
+        }
+
+        return StructuralParagraphBuildResult.Succeeded(paragraphs);
+    }
+
+    private static string NormalizeMappedParagraphText(string? value)
+        => Regex.Replace(
+            (value ?? string.Empty)
+                .Replace("\r\n", "\n", StringComparison.Ordinal)
+                .Replace('\r', '\n')
+                .Replace('\v', '\n')
+                .Replace('\f', '\n')
+                .Replace('\u2028', '\n')
+                .Replace('\u2029', '\n'),
+            @"\s*\n+\s*",
+            " ").Trim();
+
     private static string BuildValidationMessage(ValidationResult validation)
     {
         var first = validation.Errors.FirstOrDefault()?.Message
@@ -450,6 +524,54 @@ public sealed partial class ManuscriptStructureAnalysisService(
         return $"Structural paragraphs: 1..{structuralParagraphCount:N0}; final structural paragraph maps to Word paragraph {finalMap:N0}; returned {string.Join("; ", ranges)}.";
     }
 
+    private static string BuildMappingFailureDiagnostic(
+        int sceneNumber,
+        int structuralStart,
+        int structuralEnd,
+        IReadOnlyList<ManuscriptScanParagraphMap> paragraphMap,
+        IReadOnlyList<ManuscriptScanSourceParagraph> sourceParagraphs)
+    {
+        var startMap = paragraphMap.FirstOrDefault(item => item.StructuralParagraphOrdinal == structuralStart);
+        var endMap = paragraphMap.FirstOrDefault(item => item.StructuralParagraphOrdinal == structuralEnd);
+        var missing = new List<string>();
+        if (startMap is null || startMap.WordParagraphIndex < 0)
+        {
+            missing.Add($"start structural paragraph {structuralStart:N0}");
+        }
+
+        if (endMap is null || endMap.WordParagraphIndex < 0)
+        {
+            missing.Add($"end structural paragraph {structuralEnd:N0}");
+        }
+
+        var wordRange = sourceParagraphs.Count == 0
+            ? "unknown"
+            : $"{sourceParagraphs.Min(item => item.WordParagraphIndex):N0}..{sourceParagraphs.Max(item => item.WordParagraphIndex):N0}";
+        var nearestOrdinal = missing.Count > 0 && (endMap is null || endMap.WordParagraphIndex < 0)
+            ? structuralEnd
+            : structuralStart;
+        return $"Scene {sceneNumber:N0} structural range {structuralStart:N0}-{structuralEnd:N0}; missing {string.Join(" and ", missing)}; {BuildMapNeighbourDiagnostic(nearestOrdinal, paragraphMap, sourceParagraphs)}; chapter Word range {wordRange}.";
+    }
+
+    private static string BuildMapNeighbourDiagnostic(
+        int structuralOrdinal,
+        IReadOnlyList<ManuscriptScanParagraphMap> paragraphMap,
+        IReadOnlyList<ManuscriptScanSourceParagraph> sourceParagraphs)
+    {
+        var previous = paragraphMap
+            .Where(item => item.StructuralParagraphOrdinal < structuralOrdinal && item.WordParagraphIndex >= 0)
+            .OrderByDescending(item => item.StructuralParagraphOrdinal)
+            .FirstOrDefault();
+        var next = paragraphMap
+            .Where(item => item.StructuralParagraphOrdinal > structuralOrdinal && item.WordParagraphIndex >= 0)
+            .OrderBy(item => item.StructuralParagraphOrdinal)
+            .FirstOrDefault();
+        var wordRange = sourceParagraphs.Count == 0
+            ? "unknown"
+            : $"{sourceParagraphs.Min(item => item.WordParagraphIndex):N0}..{sourceParagraphs.Max(item => item.WordParagraphIndex):N0}";
+        return $"nearest mapped neighbours: previous {Display(previous?.StructuralParagraphOrdinal)}->{Display(previous?.WordParagraphIndex)}, next {Display(next?.StructuralParagraphOrdinal)}->{Display(next?.WordParagraphIndex)}, chapter Word range {wordRange}";
+    }
+
     private static string Display(int? value)
         => value.HasValue ? value.Value.ToString("N0") : "?";
 
@@ -612,6 +734,19 @@ public sealed class ManuscriptStructureAnalysisResult
         => new() { TemporaryChapterKey = temporaryChapterKey, Success = false, ErrorMessage = errorMessage, ParsedJson = parsedJson };
 }
 
+internal sealed class StructuralParagraphBuildResult
+{
+    public bool Success { get; init; }
+    public IReadOnlyList<string> Paragraphs { get; init; } = [];
+    public string? ErrorMessage { get; init; }
+
+    public static StructuralParagraphBuildResult Succeeded(IReadOnlyList<string> paragraphs)
+        => new() { Success = true, Paragraphs = paragraphs };
+
+    public static StructuralParagraphBuildResult Failed(string errorMessage)
+        => new() { Success = false, ErrorMessage = errorMessage };
+}
+
 public sealed class ManuscriptStructureSceneBoundary
 {
     public string TemporarySceneKey { get; init; } = string.Empty;

PlotLine/wwwroot/js/word-companion-host.js

@@ -2348,25 +2348,34 @@
             });
         };
 
+        const normalizeScannedParagraphText = (value) => String(value || "")
+            .replace(/[\r\n\v\f\u2028\u2029]+/g, " ")
+            .replace(/\s+/g, " ")
+            .trim();
+
         const addWords = (paragraph, text, words) => {
             if (!chapter || words <= 0) {
                 return;
             }
 
-            chapter.wordCount += words;
+            const structuralText = normalizeScannedParagraphText(text);
-            chapter.chapterTextParagraphs.push(text);
+            if (!structuralText) {
+                return;
+            }
+
+            chapter.wordCount += countWords(structuralText);
+            chapter.chapterTextParagraphs.push(structuralText);
             chapter.paragraphMap = chapter.paragraphMap || [];
             chapter.paragraphMap.push({
                 structuralParagraphOrdinal: chapter.chapterTextParagraphs.length,
                 wordParagraphIndex: Number.isInteger(paragraph?.index) ? paragraph.index : -1,
-                text
+                text: structuralText
             });
         };
 
         paragraphs.forEach((paragraph, index) => {
             paragraph.index = index;
-            const rawText = String(paragraph.text || "");
+            const text = normalizeScannedParagraphText(paragraph.text);
-            const text = rawText.trim();
             if (!text) {
                 if (chapter) {
                     recordSourceParagraph(paragraph, "", { isBlank: true, includeInSourceText: true });