using System.Diagnostics; using System.Text; using System.Text.Json; using System.Text.Json.Serialization; using Microsoft.Extensions.Options; using PlotLine.Data; using PlotLine.Models; using PlotLine.ViewModels; namespace PlotLine.Services; public interface ICharacterEnrichmentService { Task QueueAfterCharacterResolutionAsync(int projectId, int bookId, int userId); Task QueueForCharacterAsync(int characterId, int userId, int? bookId = null); Task RetryAsync(int bookId, int userId); Task GetCurrentAsync(int bookId); Task GetCurrentForCharacterAsync(int characterId, int userId); Task GetCharacterStatusAsync(int characterId, int userId); Task ProcessNextAsync(CancellationToken cancellationToken); string BuildPromptForTest(CharacterEnrichmentContext context); StoryIntelligenceResponseContract BuildResponseContractForTest(); } public sealed class CharacterEnrichmentService( ICharacterEnrichmentRepository repository, IStoryIntelligencePipelineRepository pipelines, IStoryIntelligenceClient client, IOptions options, IStoryIntelligenceProgressNotifier notifier, ILogger logger) : ICharacterEnrichmentService { private const string PromptVersion = "Character-Enrichment-V1"; private const int BatchSize = 8; private const int MaxPromptCharacters = 350_000; private const int MainCharacterPromptCharacters = 175_000; private const int MinimumSingleCharacterScenes = 8; private const decimal SummaryMinimumConfidence = 0.65m; private const decimal FactMinimumConfidence = 0.9m; private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web) { WriteIndented = true }; private readonly StoryIntelligenceOptions settings = options.Value; public async Task QueueAfterCharacterResolutionAsync(int projectId, int bookId, int userId) { var pipeline = await pipelines.GetByBookForUserAsync(bookId, userId); return await repository.QueueAsync(new CharacterEnrichmentQueueRequest { ProjectID = projectId, BookID = bookId, UserID = userId, StoryIntelligenceBookPipelineID = pipeline?.StoryIntelligenceBookPipelineID, CharacterResolutionVersion = $"CharacterImport:{bookId}" }); } public async Task QueueForCharacterAsync(int characterId, int userId, int? bookId = null) { var run = await repository.QueueCharacterAsync(new CharacterEnrichmentCharacterQueueRequest { CharacterID = characterId, UserID = userId, BookID = bookId }); if (run.WasCoalesced) { logger.LogInformation( "Manual character enrichment request ignored because an active run already exists. RunID={RunID} ProjectID={ProjectID} BookID={BookID} CharacterID={CharacterID} Status={Status}", run.CharacterEnrichmentRunID, run.ProjectID, run.BookID, run.CharacterID, run.Status); } else { logger.LogInformation( "Manual character enrichment queued. RunID={RunID} ProjectID={ProjectID} BookID={BookID} CharacterID={CharacterID} Status={Status}", run.CharacterEnrichmentRunID, run.ProjectID, run.BookID, run.CharacterID, run.Status); } return run; } public Task RetryAsync(int bookId, int userId) => repository.RetryAsync(bookId, userId); public Task GetCurrentAsync(int bookId) => repository.GetCurrentAsync(bookId); public Task GetCurrentForCharacterAsync(int characterId, int userId) => repository.GetCurrentForCharacterAsync(characterId, userId); public Task GetCharacterStatusAsync(int characterId, int userId) => repository.GetCharacterStatusAsync(characterId, userId); public async Task ProcessNextAsync(CancellationToken cancellationToken) { var run = await repository.ClaimNextAsync(Math.Max(15, settings.ClaimLeaseMinutes)); if (run is null) { return false; } var stopwatch = Stopwatch.StartNew(); var results = new List(); var inputTokens = 0; var outputTokens = 0; var model = settings.EffectiveWholeBookPlotSynthesisModel; try { logger.LogInformation( "Character enrichment starting. RunID={RunID} ProjectID={ProjectID} BookID={BookID} CharacterID={CharacterID}", run.CharacterEnrichmentRunID, run.ProjectID, run.BookID, run.CharacterID); await PublishProgressAsync(run, 0, 0, "Preparing character analysis", "Preparing character analysis...", cancellationToken); var context = await repository.BuildContextAsync(run.ProjectID, run.BookID, run.CharacterID); var characters = context.Characters .Where(character => character.Scenes.Count > 0) .OrderBy(character => character.CharacterName, StringComparer.OrdinalIgnoreCase) .ToList(); logger.LogInformation( "Character enrichment context prepared. RunID={RunID} ProjectID={ProjectID} BookID={BookID} CharacterID={CharacterID} Characters={CharacterCount}", run.CharacterEnrichmentRunID, run.ProjectID, run.BookID, run.CharacterID, characters.Count); await PublishProgressAsync(run, characters.Count, 0, "Analysing characters", "Analysing characters...", cancellationToken); foreach (var batchContext in BuildPromptBatches(context, characters, run)) { cancellationToken.ThrowIfCancellationRequested(); var prompt = BuildPrompt(batchContext); if (prompt.Length > MaxPromptCharacters) { throw new InvalidOperationException($"Character enrichment prompt would exceed the configured safe context limit. PromptCharacters={prompt.Length:N0} Limit={MaxPromptCharacters:N0}. No manuscript text was silently truncated."); } logger.LogInformation( "Character enrichment batch starting. RunID={RunID} CharacterID={CharacterID} BatchCharacters={BatchCharacters} PromptCharacters={PromptCharacters} SceneCount={SceneCount} Processed={Processed}/{Total}", run.CharacterEnrichmentRunID, run.CharacterID, batchContext.Characters.Count, prompt.Length, batchContext.Characters.Sum(character => character.Scenes.Count), results.Count, characters.Count); await PublishProgressAsync(run, characters.Count, results.Count, "Creating summaries and extracting character details", "Creating summaries and extracting character details...", cancellationToken); var clientResult = await client.ExecutePromptAsync( prompt, PromptVersion, cancellationToken, settings.EffectiveWholeBookPlotSynthesisModel, settings.WholeBookPlotSynthesisMaxOutputTokens.GetValueOrDefault(settings.MaxOutputTokens), responseContract: CharacterEnrichmentStructuredOutputSchema.Contract); model = clientResult.Model; inputTokens += clientResult.InputTokens ?? 0; outputTokens += clientResult.OutputTokens ?? 0; var parsed = CharacterEnrichmentResult.FromJson(ExtractOutputText(clientResult.RawResponseText)); results.AddRange(FilterResults(parsed.Characters, batchContext.Characters.Select(character => character.CharacterID).ToHashSet())); await PublishProgressAsync(run, characters.Count, Math.Min(characters.Count, results.Count), "Saving character analysis", "Saving character analysis...", cancellationToken); } stopwatch.Stop(); await repository.CompleteAsync(new CharacterEnrichmentCompletionRequest { CharacterEnrichmentRunID = run.CharacterEnrichmentRunID, Model = model, InputTokens = inputTokens == 0 ? null : inputTokens, OutputTokens = outputTokens == 0 ? null : outputTokens, TotalTokens = inputTokens + outputTokens == 0 ? null : inputTokens + outputTokens, DurationMs = stopwatch.ElapsedMilliseconds }, results); await notifier.PublishCharacterEnrichmentAsync(ToProgress(run, CharacterEnrichmentStatuses.Completed, results.Count, results.Count, "Complete", "Character summaries are ready.")); logger.LogInformation( "Character enrichment completed. RunID={RunID} ProjectID={ProjectID} BookID={BookID} CharacterID={CharacterID} Characters={CharacterCount} DurationMs={DurationMs}", run.CharacterEnrichmentRunID, run.ProjectID, run.BookID, run.CharacterID, results.Count, stopwatch.ElapsedMilliseconds); return true; } catch (Exception ex) when (ex is not OperationCanceledException) { stopwatch.Stop(); await repository.FailAsync(run.CharacterEnrichmentRunID, ex.Message, ex.ToString(), stopwatch.ElapsedMilliseconds); await notifier.PublishCharacterEnrichmentAsync(ToProgress(run, CharacterEnrichmentStatuses.Failed, run.TotalCharacters, run.ProcessedCharacters, "Failed", "Character enrichment failed.", ex.Message)); logger.LogError(ex, "Character enrichment failed. RunID={RunID} ProjectID={ProjectID} BookID={BookID} CharacterID={CharacterID}", run.CharacterEnrichmentRunID, run.ProjectID, run.BookID, run.CharacterID); return true; } } public string BuildPromptForTest(CharacterEnrichmentContext context) => BuildPrompt(context); public StoryIntelligenceResponseContract BuildResponseContractForTest() => CharacterEnrichmentStructuredOutputSchema.Contract; private IReadOnlyList BuildPromptBatches( CharacterEnrichmentContext context, IReadOnlyList characters, CharacterEnrichmentRun run) { var batches = new List(); var current = new List(); foreach (var character in characters) { var single = BuildContextForCharacters(context, [character]); var singlePromptLength = BuildPrompt(single).Length; if (singlePromptLength > MaxPromptCharacters) { var restrained = RestrainSingleCharacterContext(context, character, run, singlePromptLength); AddCurrentBatch(); batches.Add(restrained); continue; } if (singlePromptLength >= MainCharacterPromptCharacters) { AddCurrentBatch(); batches.Add(single); continue; } var candidate = BuildContextForCharacters(context, current.Concat([character]).ToList()); if (current.Count > 0 && (current.Count >= BatchSize || BuildPrompt(candidate).Length > MaxPromptCharacters)) { AddCurrentBatch(); } current.Add(character); } AddCurrentBatch(); return batches; void AddCurrentBatch() { if (current.Count == 0) { return; } batches.Add(BuildContextForCharacters(context, current.ToList())); current.Clear(); } } private CharacterEnrichmentContext RestrainSingleCharacterContext( CharacterEnrichmentContext context, CharacterEnrichmentCharacterContext character, CharacterEnrichmentRun run, int originalPromptLength) { var scenes = SelectRepresentativeScenes(character.Scenes, character.Scenes.Count).ToList(); while (scenes.Count > MinimumSingleCharacterScenes) { var candidate = BuildContextForCharacters(context, [CopyCharacter(character, scenes)]); if (BuildPrompt(candidate).Length <= MaxPromptCharacters) { logger.LogWarning( "Character enrichment constrained an evidence-heavy character to fit the safe context limit. RunID={RunID} ProjectID={ProjectID} BookID={BookID} CharacterID={CharacterID} CharacterName={CharacterName} OriginalPromptCharacters={OriginalPromptCharacters} FinalPromptCharacters={FinalPromptCharacters} OriginalScenes={OriginalScenes} IncludedScenes={IncludedScenes} Limit={Limit}", run.CharacterEnrichmentRunID, run.ProjectID, run.BookID, character.CharacterID, character.CharacterName, originalPromptLength, BuildPrompt(candidate).Length, character.Scenes.Count, scenes.Count, MaxPromptCharacters); return candidate; } scenes = SelectRepresentativeScenes(scenes, Math.Max(MinimumSingleCharacterScenes, scenes.Count - 4)).ToList(); } var minimum = BuildContextForCharacters(context, [CopyCharacter(character, scenes)]); var minimumPromptLength = BuildPrompt(minimum).Length; if (minimumPromptLength <= MaxPromptCharacters) { logger.LogWarning( "Character enrichment constrained an evidence-heavy character to the minimum scene set. RunID={RunID} ProjectID={ProjectID} BookID={BookID} CharacterID={CharacterID} CharacterName={CharacterName} OriginalPromptCharacters={OriginalPromptCharacters} FinalPromptCharacters={FinalPromptCharacters} OriginalScenes={OriginalScenes} IncludedScenes={IncludedScenes} Limit={Limit}", run.CharacterEnrichmentRunID, run.ProjectID, run.BookID, character.CharacterID, character.CharacterName, originalPromptLength, minimumPromptLength, character.Scenes.Count, scenes.Count, MaxPromptCharacters); return minimum; } throw new InvalidOperationException($"Character enrichment prompt would exceed the configured safe context limit for character {character.CharacterID} even when limited to {scenes.Count:N0} whole scenes. PromptCharacters={minimumPromptLength:N0} Limit={MaxPromptCharacters:N0}. No manuscript text was silently truncated."); } private static CharacterEnrichmentContext BuildContextForCharacters( CharacterEnrichmentContext context, IReadOnlyList characters) => new() { ProjectID = context.ProjectID, BookID = context.BookID, BookTitle = context.BookTitle, StoryEra = context.StoryEra, SeriesStartDate = context.SeriesStartDate, Characters = characters }; private static CharacterEnrichmentCharacterContext CopyCharacter( CharacterEnrichmentCharacterContext character, IReadOnlyList scenes) => new() { CharacterID = character.CharacterID, CharacterName = character.CharacterName, BirthDate = character.BirthDate, AgeAtSeriesStart = character.AgeAtSeriesStart, Height = character.Height, EyeColour = character.EyeColour, DefaultDescription = character.DefaultDescription, Aliases = character.Aliases, Scenes = scenes }; private static IReadOnlyList SelectRepresentativeScenes( IReadOnlyList scenes, int maxScenes) { if (scenes.Count <= maxScenes) { return scenes; } maxScenes = Math.Clamp(maxScenes, 1, scenes.Count); var selected = new SortedDictionary(); var firstCount = Math.Min(scenes.Count, Math.Min(12, Math.Max(1, maxScenes / 3))); var lastCount = Math.Min(scenes.Count - firstCount, Math.Min(8, Math.Max(0, maxScenes / 4))); for (var index = 0; index < firstCount; index++) { selected[index] = scenes[index]; } for (var index = scenes.Count - lastCount; index < scenes.Count; index++) { if (index >= 0) { selected[index] = scenes[index]; } } var remaining = maxScenes - selected.Count; if (remaining > 0) { var start = firstCount; var endExclusive = scenes.Count - lastCount; var span = Math.Max(0, endExclusive - start); for (var slot = 1; slot <= remaining && span > 0; slot++) { var offset = (int)Math.Round(slot * (span - 1) / (double)(remaining + 1), MidpointRounding.AwayFromZero); selected.TryAdd(start + offset, scenes[start + offset]); } } for (var index = 0; selected.Count < maxScenes && index < scenes.Count; index++) { selected.TryAdd(index, scenes[index]); } return selected.Values.ToList(); } private async Task PublishProgressAsync(CharacterEnrichmentRun run, int total, int processed, string stage, string message, CancellationToken cancellationToken) { await repository.UpdateProgressAsync(new CharacterEnrichmentProgressUpdate { CharacterEnrichmentRunID = run.CharacterEnrichmentRunID, TotalCharacters = total, ProcessedCharacters = processed, CurrentStage = stage, CurrentMessage = message }); await notifier.PublishCharacterEnrichmentAsync(ToProgress(run, CharacterEnrichmentStatuses.Running, total, processed, stage, message)); } private static IReadOnlyList FilterResults( IReadOnlyList results, HashSet allowedCharacterIds) => results .Where(result => allowedCharacterIds.Contains(result.CharacterID)) .Select(result => new CharacterEnrichmentCharacterResult { CharacterID = result.CharacterID, Summary = Clamp(result.Confidence) >= SummaryMinimumConfidence ? Clean(result.Summary, 2500) : string.Empty, Confidence = Clamp(result.Confidence), DateOfBirth = StrongFact(result.DateOfBirth), AgeAtStartOfSeries = StrongFact(result.AgeAtStartOfSeries), Height = StrongFact(result.Height), EyeColour = StrongFact(result.EyeColour), EvidenceSceneIDs = result.EvidenceSceneIDs.Where(id => id > 0).Distinct().Take(20).ToList() }) .Where(result => !string.IsNullOrWhiteSpace(result.Summary) || !string.IsNullOrWhiteSpace(result.DateOfBirth.Value) || !string.IsNullOrWhiteSpace(result.AgeAtStartOfSeries.Value) || !string.IsNullOrWhiteSpace(result.Height.Value) || !string.IsNullOrWhiteSpace(result.EyeColour.Value)) .ToList(); private static CharacterEnrichmentFactResult StrongFact(CharacterEnrichmentFactResult fact) => Clamp(fact.Confidence) >= FactMinimumConfidence && !string.IsNullOrWhiteSpace(fact.Value) && fact.SceneID is > 0 ? new CharacterEnrichmentFactResult { Value = Clean(fact.Value, 120), Confidence = Clamp(fact.Confidence), Evidence = Clean(fact.Evidence, 700), SceneID = fact.SceneID } : new CharacterEnrichmentFactResult(); private static string BuildPrompt(CharacterEnrichmentContext context) { var builder = new StringBuilder(); builder.AppendLine("You are PlotDirector's Character Enrichment engine."); builder.AppendLine("Create concise author-facing character summaries and extract only strongly evidenced character facts."); builder.AppendLine("Use only the supplied manuscript scenes. Do not invent, infer from behaviour alone, speculate about future events, or list every appearance."); builder.AppendLine("For dateOfBirth, ageAtStartOfSeries, height, and eyeColour, return a value only when direct manuscript wording strongly supports it. Otherwise return null value and confidence 0."); builder.AppendLine("If a character already has a stored value, preserve it by returning null unless the manuscript contains stronger direct evidence; the save layer will still refuse to overwrite existing author data."); builder.AppendLine(); builder.AppendLine("[BOOK]"); builder.AppendLine($"BookID: {context.BookID}"); builder.AppendLine($"Title: {context.BookTitle}"); builder.AppendLine($"Story era: {context.StoryEra}"); builder.AppendLine($"Series start date: {context.SeriesStartDate:yyyy-MM-dd}"); builder.AppendLine(); builder.AppendLine("[CHARACTERS]"); foreach (var character in context.Characters) { builder.AppendLine(); builder.AppendLine($"CharacterID: {character.CharacterID}"); builder.AppendLine($"Name: {character.CharacterName}"); builder.AppendLine($"Aliases: {string.Join(", ", character.Aliases)}"); builder.AppendLine($"Existing birth date: {character.BirthDate:yyyy-MM-dd}"); builder.AppendLine($"Existing age at series start: {character.AgeAtSeriesStart}"); builder.AppendLine($"Existing height: {character.Height}"); builder.AppendLine($"Existing eye colour: {character.EyeColour}"); builder.AppendLine($"Existing description: {character.DefaultDescription}"); builder.AppendLine("[SCENES]"); foreach (var scene in character.Scenes) { builder.AppendLine($"SceneID: {scene.SceneID}; Chapter {scene.ChapterNumber:0.##} {scene.ChapterTitle}; Scene {scene.SceneNumber:0.##} {scene.SceneTitle}"); builder.AppendLine(""); builder.AppendLine(scene.SourceText); builder.AppendLine(""); } } return builder.ToString(); } private static CharacterEnrichmentProgressEvent ToProgress(CharacterEnrichmentRun run, string status, int total, int processed, string stage, string message, string? error = null) => new() { CharacterEnrichmentRunID = run.CharacterEnrichmentRunID, UserID = run.UserID, ProjectID = run.ProjectID, BookID = run.BookID, CharacterID = run.CharacterID, Status = status, TotalCharacters = total, ProcessedCharacters = processed, CurrentStage = stage, CurrentMessage = message, ErrorMessage = error, UpdatedUtc = DateTime.UtcNow }; private static string ExtractOutputText(string rawResponseText) { using var response = JsonDocument.Parse(rawResponseText); foreach (var output in response.RootElement.GetProperty("output").EnumerateArray()) { if (!output.TryGetProperty("content", out var content)) { continue; } foreach (var item in content.EnumerateArray()) { if (item.TryGetProperty("type", out var type) && string.Equals(type.GetString(), "output_text", StringComparison.OrdinalIgnoreCase) && item.TryGetProperty("text", out var text) && !string.IsNullOrWhiteSpace(text.GetString())) { return text.GetString()!.Trim(); } } } throw new JsonException("OpenAI response did not contain output_text content."); } private static decimal Clamp(decimal value) => Math.Clamp(value, 0m, 1m); private static string Clean(string? value, int maxLength) { var clean = value?.Trim(); if (string.IsNullOrWhiteSpace(clean)) { return string.Empty; } return clean.Length <= maxLength ? clean : clean[..maxLength].TrimEnd(); } } public static class CharacterEnrichmentStructuredOutputSchema { public static readonly StoryIntelligenceResponseContract Contract = new() { Name = "character_enrichment", Strict = true, Schema = new Dictionary { ["type"] = "object", ["additionalProperties"] = false, ["required"] = new[] { "schemaVersion", "characters" }, ["properties"] = new Dictionary { ["schemaVersion"] = StringSchema(), ["characters"] = ArraySchema(ObjectSchema(new Dictionary { ["characterID"] = IntegerSchema(), ["summary"] = StringSchema(), ["confidence"] = NumberSchema(), ["dateOfBirth"] = FactSchema(), ["ageAtStartOfSeries"] = FactSchema(), ["height"] = FactSchema(), ["eyeColour"] = FactSchema(), ["evidenceSceneIDs"] = ArraySchema(IntegerSchema()) }, "characterID", "summary", "confidence", "dateOfBirth", "ageAtStartOfSeries", "height", "eyeColour", "evidenceSceneIDs")) } } }; private static Dictionary FactSchema() => ObjectSchema(new Dictionary { ["value"] = NullableStringSchema(), ["confidence"] = NumberSchema(), ["evidence"] = NullableStringSchema(), ["sceneID"] = NullableIntegerSchema() }, "value", "confidence", "evidence", "sceneID"); private static Dictionary ObjectSchema(Dictionary properties, params string[] required) => new() { ["type"] = "object", ["additionalProperties"] = false, ["required"] = required, ["properties"] = properties }; private static Dictionary ArraySchema(object items) => new() { ["type"] = "array", ["items"] = items }; private static Dictionary StringSchema() => new() { ["type"] = "string" }; private static Dictionary NullableStringSchema() => new() { ["type"] = new[] { "string", "null" } }; private static Dictionary IntegerSchema() => new() { ["type"] = "integer" }; private static Dictionary NullableIntegerSchema() => new() { ["type"] = new[] { "integer", "null" } }; private static Dictionary NumberSchema() => new() { ["type"] = "number", ["minimum"] = 0, ["maximum"] = 1 }; }