Batch character enrichment by prompt size

PlotLine.Tests/Program.cs

@@ -338,6 +338,7 @@ var tests = new (string Name, Action Test)[]
     ("Story Intelligence character review queues enrichment beside plot synthesis", StoryIntelligenceCharacterReviewQueuesEnrichmentBesidePlotSynthesis),
     ("Story Intelligence character enrichment uses persisted SignalR progress", StoryIntelligenceCharacterEnrichmentUsesPersistedSignalRProgress),
     ("Story Intelligence character enrichment preserves author fields", StoryIntelligenceCharacterEnrichmentPreservesAuthorFields),
+    ("Story Intelligence character enrichment batches by prompt size", StoryIntelligenceCharacterEnrichmentBatchesByPromptSize),
     ("Character detail can queue single-character enrichment", CharacterDetailCanQueueSingleCharacterEnrichment),
     ("Single-character enrichment is persisted and duplicate safe", SingleCharacterEnrichmentIsPersistedAndDuplicateSafe),
     ("Character detail enrichment status uses scoped SignalR", CharacterDetailEnrichmentStatusUsesScopedSignalR),
@@ -1256,6 +1257,20 @@ static void StoryIntelligenceCharacterEnrichmentPreservesAuthorFields()
     Assert(migration.Contains("CREATE TABLE dbo.CharacterEnrichmentEvidence", StringComparison.Ordinal), "Generated facts should retain internal provenance/evidence.");
 }
 
+static void StoryIntelligenceCharacterEnrichmentBatchesByPromptSize()
+{
+    var root = PlotLineRoot();
+    var service = File.ReadAllText(Path.Combine(root, "Services/CharacterEnrichmentService.cs"));
+
+    Assert(service.Contains("BuildPromptBatches", StringComparison.Ordinal), "Character enrichment should batch by prompt size rather than fixed alphabetical chunks only.");
+    Assert(service.Contains("MainCharacterPromptCharacters", StringComparison.Ordinal), "Evidence-heavy characters should be processed singly before context is restrained.");
+    Assert(service.Contains("RestrainSingleCharacterContext", StringComparison.Ordinal), "Single-character prompts should be explicitly restrained only when one character exceeds the safe limit.");
+    Assert(service.Contains("SelectRepresentativeScenes", StringComparison.Ordinal), "Oversized single-character prompts should use deterministic whole-scene selection.");
+    Assert(!service.Contains("character.Scenes.Take(60)", StringComparison.Ordinal), "Prompt construction should not silently take the first 60 scenes for every character.");
+    Assert(service.Contains("OriginalPromptCharacters", StringComparison.Ordinal)
+        && service.Contains("IncludedScenes", StringComparison.Ordinal), "Restrained character context should be logged with auditable scene counts.");
+}
+
 static void CharacterDetailCanQueueSingleCharacterEnrichment()
 {
     var root = PlotLineRoot();

PlotLine/Services/CharacterEnrichmentService.cs

@@ -33,6 +33,8 @@ public sealed class CharacterEnrichmentService(
     private const string PromptVersion = "Character-Enrichment-V1";
     private const int BatchSize = 8;
     private const int MaxPromptCharacters = 350_000;
+    private const int MainCharacterPromptCharacters = 175_000;
+    private const int MinimumSingleCharacterScenes = 8;
     private const decimal SummaryMinimumConfidence = 0.65m;
     private const decimal FactMinimumConfidence = 0.9m;
     private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web)
@@ -134,29 +136,22 @@ public sealed class CharacterEnrichmentService(
                 characters.Count);
 
             await PublishProgressAsync(run, characters.Count, 0, "Analysing characters", "Analysing characters...", cancellationToken);
-            foreach (var batch in characters.Chunk(BatchSize))
+            foreach (var batchContext in BuildPromptBatches(context, characters, run))
             {
                 cancellationToken.ThrowIfCancellationRequested();
-                var batchContext = new CharacterEnrichmentContext
-                {
-                    ProjectID = context.ProjectID,
-                    BookID = context.BookID,
-                    BookTitle = context.BookTitle,
-                    StoryEra = context.StoryEra,
-                    SeriesStartDate = context.SeriesStartDate,
-                    Characters = batch
-                };
                 var prompt = BuildPrompt(batchContext);
                 if (prompt.Length > MaxPromptCharacters)
                 {
-                    throw new InvalidOperationException("Character enrichment prompt would exceed the configured safe context limit. No manuscript text was silently truncated.");
+                    throw new InvalidOperationException($"Character enrichment prompt would exceed the configured safe context limit. PromptCharacters={prompt.Length:N0} Limit={MaxPromptCharacters:N0}. No manuscript text was silently truncated.");
                 }
 
                 logger.LogInformation(
-                    "Character enrichment batch starting. RunID={RunID} CharacterID={CharacterID} BatchCharacters={BatchCharacters} Processed={Processed}/{Total}",
+                    "Character enrichment batch starting. RunID={RunID} CharacterID={CharacterID} BatchCharacters={BatchCharacters} PromptCharacters={PromptCharacters} SceneCount={SceneCount} Processed={Processed}/{Total}",
                     run.CharacterEnrichmentRunID,
                     run.CharacterID,
-                    batch.Length,
+                    batchContext.Characters.Count,
+                    prompt.Length,
+                    batchContext.Characters.Sum(character => character.Scenes.Count),
                     results.Count,
                     characters.Count);
                 await PublishProgressAsync(run, characters.Count, results.Count, "Creating summaries and extracting character details", "Creating summaries and extracting character details...", cancellationToken);
@@ -172,7 +167,7 @@ public sealed class CharacterEnrichmentService(
                 inputTokens += clientResult.InputTokens ?? 0;
                 outputTokens += clientResult.OutputTokens ?? 0;
                 var parsed = CharacterEnrichmentResult.FromJson(ExtractOutputText(clientResult.RawResponseText));
-                results.AddRange(FilterResults(parsed.Characters, batch.Select(character => character.CharacterID).ToHashSet()));
+                results.AddRange(FilterResults(parsed.Characters, batchContext.Characters.Select(character => character.CharacterID).ToHashSet()));
                 await PublishProgressAsync(run, characters.Count, Math.Min(characters.Count, results.Count), "Saving character analysis", "Saving character analysis...", cancellationToken);
             }
 
@@ -213,6 +208,186 @@ public sealed class CharacterEnrichmentService(
     public StoryIntelligenceResponseContract BuildResponseContractForTest()
         => CharacterEnrichmentStructuredOutputSchema.Contract;
 
+    private IReadOnlyList<CharacterEnrichmentContext> BuildPromptBatches(
+        CharacterEnrichmentContext context,
+        IReadOnlyList<CharacterEnrichmentCharacterContext> characters,
+        CharacterEnrichmentRun run)
+    {
+        var batches = new List<CharacterEnrichmentContext>();
+        var current = new List<CharacterEnrichmentCharacterContext>();
+
+        foreach (var character in characters)
+        {
+            var single = BuildContextForCharacters(context, [character]);
+            var singlePromptLength = BuildPrompt(single).Length;
+            if (singlePromptLength > MaxPromptCharacters)
+            {
+                var restrained = RestrainSingleCharacterContext(context, character, run, singlePromptLength);
+                AddCurrentBatch();
+                batches.Add(restrained);
+                continue;
+            }
+
+            if (singlePromptLength >= MainCharacterPromptCharacters)
+            {
+                AddCurrentBatch();
+                batches.Add(single);
+                continue;
+            }
+
+            var candidate = BuildContextForCharacters(context, current.Concat([character]).ToList());
+            if (current.Count > 0 && (current.Count >= BatchSize || BuildPrompt(candidate).Length > MaxPromptCharacters))
+            {
+                AddCurrentBatch();
+            }
+
+            current.Add(character);
+        }
+
+        AddCurrentBatch();
+        return batches;
+
+        void AddCurrentBatch()
+        {
+            if (current.Count == 0)
+            {
+                return;
+            }
+
+            batches.Add(BuildContextForCharacters(context, current.ToList()));
+            current.Clear();
+        }
+    }
+
+    private CharacterEnrichmentContext RestrainSingleCharacterContext(
+        CharacterEnrichmentContext context,
+        CharacterEnrichmentCharacterContext character,
+        CharacterEnrichmentRun run,
+        int originalPromptLength)
+    {
+        var scenes = SelectRepresentativeScenes(character.Scenes, character.Scenes.Count).ToList();
+        while (scenes.Count > MinimumSingleCharacterScenes)
+        {
+            var candidate = BuildContextForCharacters(context, [CopyCharacter(character, scenes)]);
+            if (BuildPrompt(candidate).Length <= MaxPromptCharacters)
+            {
+                logger.LogWarning(
+                    "Character enrichment constrained an evidence-heavy character to fit the safe context limit. RunID={RunID} ProjectID={ProjectID} BookID={BookID} CharacterID={CharacterID} CharacterName={CharacterName} OriginalPromptCharacters={OriginalPromptCharacters} FinalPromptCharacters={FinalPromptCharacters} OriginalScenes={OriginalScenes} IncludedScenes={IncludedScenes} Limit={Limit}",
+                    run.CharacterEnrichmentRunID,
+                    run.ProjectID,
+                    run.BookID,
+                    character.CharacterID,
+                    character.CharacterName,
+                    originalPromptLength,
+                    BuildPrompt(candidate).Length,
+                    character.Scenes.Count,
+                    scenes.Count,
+                    MaxPromptCharacters);
+                return candidate;
+            }
+
+            scenes = SelectRepresentativeScenes(scenes, Math.Max(MinimumSingleCharacterScenes, scenes.Count - 4)).ToList();
+        }
+
+        var minimum = BuildContextForCharacters(context, [CopyCharacter(character, scenes)]);
+        var minimumPromptLength = BuildPrompt(minimum).Length;
+        if (minimumPromptLength <= MaxPromptCharacters)
+        {
+            logger.LogWarning(
+                "Character enrichment constrained an evidence-heavy character to the minimum scene set. RunID={RunID} ProjectID={ProjectID} BookID={BookID} CharacterID={CharacterID} CharacterName={CharacterName} OriginalPromptCharacters={OriginalPromptCharacters} FinalPromptCharacters={FinalPromptCharacters} OriginalScenes={OriginalScenes} IncludedScenes={IncludedScenes} Limit={Limit}",
+                run.CharacterEnrichmentRunID,
+                run.ProjectID,
+                run.BookID,
+                character.CharacterID,
+                character.CharacterName,
+                originalPromptLength,
+                minimumPromptLength,
+                character.Scenes.Count,
+                scenes.Count,
+                MaxPromptCharacters);
+            return minimum;
+        }
+
+        throw new InvalidOperationException($"Character enrichment prompt would exceed the configured safe context limit for character {character.CharacterID} even when limited to {scenes.Count:N0} whole scenes. PromptCharacters={minimumPromptLength:N0} Limit={MaxPromptCharacters:N0}. No manuscript text was silently truncated.");
+    }
+
+    private static CharacterEnrichmentContext BuildContextForCharacters(
+        CharacterEnrichmentContext context,
+        IReadOnlyList<CharacterEnrichmentCharacterContext> characters)
+        => new()
+        {
+            ProjectID = context.ProjectID,
+            BookID = context.BookID,
+            BookTitle = context.BookTitle,
+            StoryEra = context.StoryEra,
+            SeriesStartDate = context.SeriesStartDate,
+            Characters = characters
+        };
+
+    private static CharacterEnrichmentCharacterContext CopyCharacter(
+        CharacterEnrichmentCharacterContext character,
+        IReadOnlyList<CharacterEnrichmentSceneContext> scenes)
+        => new()
+        {
+            CharacterID = character.CharacterID,
+            CharacterName = character.CharacterName,
+            BirthDate = character.BirthDate,
+            AgeAtSeriesStart = character.AgeAtSeriesStart,
+            Height = character.Height,
+            EyeColour = character.EyeColour,
+            DefaultDescription = character.DefaultDescription,
+            Aliases = character.Aliases,
+            Scenes = scenes
+        };
+
+    private static IReadOnlyList<CharacterEnrichmentSceneContext> SelectRepresentativeScenes(
+        IReadOnlyList<CharacterEnrichmentSceneContext> scenes,
+        int maxScenes)
+    {
+        if (scenes.Count <= maxScenes)
+        {
+            return scenes;
+        }
+
+        maxScenes = Math.Clamp(maxScenes, 1, scenes.Count);
+        var selected = new SortedDictionary<int, CharacterEnrichmentSceneContext>();
+        var firstCount = Math.Min(scenes.Count, Math.Min(12, Math.Max(1, maxScenes / 3)));
+        var lastCount = Math.Min(scenes.Count - firstCount, Math.Min(8, Math.Max(0, maxScenes / 4)));
+
+        for (var index = 0; index < firstCount; index++)
+        {
+            selected[index] = scenes[index];
+        }
+
+        for (var index = scenes.Count - lastCount; index < scenes.Count; index++)
+        {
+            if (index >= 0)
+            {
+                selected[index] = scenes[index];
+            }
+        }
+
+        var remaining = maxScenes - selected.Count;
+        if (remaining > 0)
+        {
+            var start = firstCount;
+            var endExclusive = scenes.Count - lastCount;
+            var span = Math.Max(0, endExclusive - start);
+            for (var slot = 1; slot <= remaining && span > 0; slot++)
+            {
+                var offset = (int)Math.Round(slot * (span - 1) / (double)(remaining + 1), MidpointRounding.AwayFromZero);
+                selected.TryAdd(start + offset, scenes[start + offset]);
+            }
+        }
+
+        for (var index = 0; selected.Count < maxScenes && index < scenes.Count; index++)
+        {
+            selected.TryAdd(index, scenes[index]);
+        }
+
+        return selected.Values.ToList();
+    }
+
     private async Task PublishProgressAsync(CharacterEnrichmentRun run, int total, int processed, string stage, string message, CancellationToken cancellationToken)
     {
         await repository.UpdateProgressAsync(new CharacterEnrichmentProgressUpdate
@@ -288,7 +463,7 @@ public sealed class CharacterEnrichmentService(
             builder.AppendLine($"Existing eye colour: {character.EyeColour}");
             builder.AppendLine($"Existing description: {character.DefaultDescription}");
             builder.AppendLine("[SCENES]");
-            foreach (var scene in character.Scenes.Take(60))
+            foreach (var scene in character.Scenes)
             {
                 builder.AppendLine($"SceneID: {scene.SceneID}; Chapter {scene.ChapterNumber:0.##} {scene.ChapterTitle}; Scene {scene.SceneNumber:0.##} {scene.SceneTitle}");
                 builder.AppendLine("<text>");