using System.Diagnostics; using System.Text; using System.Text.Json; using System.Text.Json.Serialization; using Microsoft.Extensions.Options; using PlotLine.Data; using PlotLine.Models; using PlotLine.ViewModels; namespace PlotLine.Services; public interface IWholeBookPlotIntelligenceService { Task GetAuthorPlotContextAsync(int? bookId = null); Task SaveAuthorPlotContextAsync(int projectId, int bookId, IReadOnlyList descriptions); Task QueueAfterCharacterResolutionAsync(int projectId, int bookId, int userId); Task RetryAsync(int bookId, int userId); Task GetCurrentAsync(int bookId); Task ProcessNextAsync(CancellationToken cancellationToken); string BuildPromptForTest(WholeBookPlotSynthesisContext context); StoryIntelligenceResponseContract BuildResponseContractForTest(); } public sealed class WholeBookPlotIntelligenceService( IWholeBookPlotIntelligenceRepository repository, IStoryIntelligencePipelineRepository pipelines, IBookRepository books, IStoryIntelligenceClient client, IOptions options, IStoryIntelligenceProgressNotifier notifier, ICurrentUserService currentUser, ILogger logger) : IWholeBookPlotIntelligenceService { private const string PromptVersion = "Whole-Book-Plot-Synthesis-V1"; private const int MaxPromptCharacters = 900_000; private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web) { WriteIndented = true }; private readonly StoryIntelligenceOptions settings = options.Value; public async Task GetAuthorPlotContextAsync(int? bookId = null) { var userId = currentUser.UserId; if (!userId.HasValue) { return null; } var book = bookId.HasValue ? await books.GetAsync(bookId.Value) : null; if (book is null && bookId.HasValue) { return null; } var descriptions = book is null ? new List() : (await repository.ListAuthorSeedsAsync(book.BookID)).Select(seed => seed.Description).ToList(); while (descriptions.Count < 3) { descriptions.Add(string.Empty); } return new StoryIntelligenceAuthorPlotContextViewModel { ProjectID = book?.ProjectID ?? 0, BookID = book?.BookID ?? 0, BookTitle = book?.BookDisplayTitle ?? string.Empty, Descriptions = descriptions }; } public Task SaveAuthorPlotContextAsync(int projectId, int bookId, IReadOnlyList descriptions) => repository.SaveAuthorSeedsAsync(new AuthorPlotSeedSaveRequest { ProjectID = projectId, BookID = bookId, Descriptions = descriptions }); public async Task QueueAfterCharacterResolutionAsync(int projectId, int bookId, int userId) { var pipeline = await pipelines.GetByBookForUserAsync(bookId, userId); var latestRunId = pipeline?.LastRunID ?? (await pipelines.ListCommittedRunsByBookAsync(bookId, userId)) .OrderByDescending(run => run.StoryIntelligenceRunID) .FirstOrDefault() ?.StoryIntelligenceRunID; return await repository.QueueAsync(new WholeBookPlotSynthesisQueueRequest { ProjectID = projectId, BookID = bookId, UserID = userId, SourceStoryIntelligenceRunID = latestRunId, CharacterResolutionVersion = $"CharacterImport:{bookId}" }); } public Task RetryAsync(int bookId, int userId) => repository.RetryAsync(bookId, userId); public Task GetCurrentAsync(int bookId) => repository.GetCurrentAsync(bookId); public async Task ProcessNextAsync(CancellationToken cancellationToken) { var run = await repository.ClaimNextAsync(Math.Max(15, settings.ClaimLeaseMinutes)); if (run is null) { return false; } var stopwatch = Stopwatch.StartNew(); try { var context = await repository.BuildContextAsync(run.ProjectID, run.BookID); if (!context.SourceAudit.CanReconstruct) { throw new InvalidOperationException( $"Scene manuscript source audit failed. Scenes={context.SourceAudit.SceneCount}, sources={context.SourceAudit.SourceRowCount}, missing={context.SourceAudit.MissingSourceCount}, duplicateSourceScenes={context.SourceAudit.DuplicateSourceSceneCount}."); } var prompt = BuildPrompt(context); if (prompt.Length > MaxPromptCharacters) { prompt = BuildPrompt(context, includeOptionalContext: false); } if (prompt.Length > MaxPromptCharacters) { throw new InvalidOperationException("Whole-book Plot Intelligence prompt would exceed the configured safe context limit. No manuscript text was silently truncated."); } var result = await client.ExecutePromptAsync( prompt, PromptVersion, cancellationToken, settings.EffectiveWholeBookPlotSynthesisModel, WholeBookPlotSynthesisMaxOutputTokens(), responseContract: WholeBookPlotSynthesisStructuredOutputSchema.Contract); var outputJson = ExtractOutputText(result.RawResponseText); var parsed = PreserveUnmatchedAuthorSeedsForReview( WholeBookPlotSynthesisResult.FromJson(outputJson), context.AuthorSeeds); stopwatch.Stop(); await repository.CompleteAsync(new WholeBookPlotSynthesisCompletionRequest { WholeBookPlotSynthesisRunID = run.WholeBookPlotSynthesisRunID, Model = result.Model, OpenAIResponseID = ExtractResponseId(result.RawResponseText), InputTokens = result.InputTokens, OutputTokens = result.OutputTokens, CachedInputTokens = ExtractCachedTokens(result.RawResponseText), TotalTokens = (result.InputTokens ?? 0) + (result.OutputTokens ?? 0) > 0 ? (result.InputTokens ?? 0) + (result.OutputTokens ?? 0) : null, DurationMs = stopwatch.ElapsedMilliseconds, ResultJson = outputJson }, parsed); await notifier.PublishGlobalAsync(run.UserID); logger.LogInformation( "Whole-book Plot Intelligence completed. RunID={RunID} ProjectID={ProjectID} BookID={BookID} PlotLines={PlotLineCount} DurationMs={DurationMs}", run.WholeBookPlotSynthesisRunID, run.ProjectID, run.BookID, parsed.PlotLines.Count, stopwatch.ElapsedMilliseconds); return true; } catch (Exception ex) when (ex is not OperationCanceledException) { stopwatch.Stop(); await repository.FailAsync(run.WholeBookPlotSynthesisRunID, ex.Message, ex.ToString(), stopwatch.ElapsedMilliseconds); await notifier.PublishGlobalAsync(run.UserID); logger.LogError(ex, "Whole-book Plot Intelligence failed. RunID={RunID} BookID={BookID}", run.WholeBookPlotSynthesisRunID, run.BookID); return true; } } public string BuildPromptForTest(WholeBookPlotSynthesisContext context) => BuildPrompt(context); public StoryIntelligenceResponseContract BuildResponseContractForTest() => WholeBookPlotSynthesisStructuredOutputSchema.Contract; private int WholeBookPlotSynthesisMaxOutputTokens() => settings.WholeBookPlotSynthesisMaxOutputTokens.GetValueOrDefault(settings.MaxOutputTokens) > 0 ? settings.WholeBookPlotSynthesisMaxOutputTokens.GetValueOrDefault(settings.MaxOutputTokens) : settings.MaxOutputTokens; public static WholeBookPlotSynthesisResult PreserveUnmatchedAuthorSeedsForReviewForTest( WholeBookPlotSynthesisResult result, IReadOnlyList authorSeeds) => PreserveUnmatchedAuthorSeedsForReview(result, authorSeeds); private static WholeBookPlotSynthesisResult PreserveUnmatchedAuthorSeedsForReview( WholeBookPlotSynthesisResult result, IReadOnlyList authorSeeds) { var matchedSeedIds = result.PlotLines .Where(plotLine => plotLine.AuthorPlotSeedID.HasValue) .Select(plotLine => plotLine.AuthorPlotSeedID!.Value) .ToHashSet(); var unmatchedSeeds = authorSeeds .Where(seed => !matchedSeedIds.Contains(seed.AuthorPlotSeedID)) .ToList(); if (unmatchedSeeds.Count == 0) { return result; } var plotLines = result.PlotLines.ToList(); foreach (var seed in unmatchedSeeds) { plotLines.Add(new WholeBookPlotSynthesisPlotLineResult { Key = $"author-seed-{seed.AuthorPlotSeedID}", ProposedTitle = "Author-provided Plot Line", Description = seed.Description, Significance = "Author intent", BookStatus = "Limited current-book evidence", ExistingPlotLineID = seed.MatchedPlotLineID, ExistingPlotLineMatchConfidence = seed.MatchedPlotLineID.HasValue ? 1m : null, ExistingPlotLineMatchReason = seed.MatchedPlotLineID.HasValue ? "Author seed is already linked to an existing Project Plot Line." : null, AuthorPlotSeedID = seed.AuthorPlotSeedID, SeedMatchConfidence = null, SeedMatchReason = "Author-provided seed retained for review; no detected whole-book Plot Line referenced this seed.", Confidence = null, Threads = [] }); } return new WholeBookPlotSynthesisResult { Metadata = result.Metadata, PlotLines = plotLines }; } private static string BuildPrompt(WholeBookPlotSynthesisContext context, bool includeOptionalContext = true) { var builder = new StringBuilder(); builder.AppendLine("You are PlotDirector's Whole-Book Plot Intelligence engine."); builder.AppendLine("Read the complete manuscript and identify sustained plot lines, subordinate threads, and every meaningful scene touch."); builder.AppendLine("Return only data that conforms to the strict schema. Reference only SceneIDs, PlotLineIDs, PlotThreadIDs, and AuthorPlotSeedIDs supplied in this request."); builder.AppendLine("A major Plot Line is a sustained narrative arc, mystery, conflict, goal, hidden truth, character struggle, investigation, relationship arc, or continuing question."); builder.AppendLine("Do not promote isolated incidents, simple presence, locations, or recurring props without narrative development into major Plot Lines."); builder.AppendLine("A Thread is a meaningful subordinate strand within a Plot Line. Do not fragment every scene development into a separate Thread."); builder.AppendLine("Create an event for a scene when the scene meaningfully mentions, reinforces, advances, complicates, foreshadows, questions, misdirects, reveals, pays off, contradicts, or resolves the thread."); builder.AppendLine("Do not attach a scene merely because associated characters are present."); builder.AppendLine("Use Mentioned only as the fallback for genuine reinforcement when no more informative event type applies."); builder.AppendLine(); builder.AppendLine("[BOOK]"); builder.AppendLine($"BookID: {context.BookID}"); builder.AppendLine($"Title: {context.BookTitle}"); builder.AppendLine(); if (includeOptionalContext) { builder.AppendLine("[CANONICAL CHARACTERS]"); foreach (var character in context.Characters) { builder.AppendLine($"CharacterID: {character.CharacterID}; CanonicalName: {character.CanonicalName}; Aliases: {string.Join(", ", character.Aliases)}"); } builder.AppendLine(); builder.AppendLine("[EXISTING PROJECT PLOT LINES]"); foreach (var plotLine in context.ExistingPlotLines) { builder.AppendLine($"PlotLineID: {plotLine.PlotLineID}; Name: {plotLine.Name}; Importance: {plotLine.Importance}; BookID: {plotLine.BookID?.ToString() ?? "series"}"); if (!string.IsNullOrWhiteSpace(plotLine.Description)) { builder.AppendLine($"Description: {plotLine.Description}"); } foreach (var thread in plotLine.Threads) { builder.AppendLine($" ThreadID: {thread.PlotThreadID}; Title: {thread.Title}; Status: {thread.Status}"); } } builder.AppendLine(); builder.AppendLine("[AUTHOR PLOT SEEDS]"); foreach (var seed in context.AuthorSeeds) { builder.AppendLine($"AuthorPlotSeedID: {seed.AuthorPlotSeedID}; Description: {seed.Description}"); } builder.AppendLine(); } builder.AppendLine("[MANUSCRIPT]"); foreach (var chapter in context.Chapters) { builder.AppendLine(); builder.AppendLine($"[CHAPTER {chapter.ChapterNumber:0.##}]"); builder.AppendLine($"ChapterID: {chapter.ChapterID}"); builder.AppendLine($"Title: {chapter.ChapterTitle}"); foreach (var scene in chapter.Scenes) { builder.AppendLine(); builder.AppendLine("[SCENE]"); builder.AppendLine($"SceneID: {scene.SceneID}"); builder.AppendLine($"SceneNumber: {scene.SceneNumber:0.##}"); builder.AppendLine($"Title: {scene.SceneTitle}"); builder.AppendLine($"POV: {scene.PovCharacterName}"); builder.AppendLine($"Setting: {scene.StructuralSetting}"); builder.AppendLine(""); builder.AppendLine(scene.SourceText); builder.AppendLine(""); } } return builder.ToString(); } private static string ExtractOutputText(string rawResponseText) { var response = JsonSerializer.Deserialize(rawResponseText, JsonOptions) ?? throw new JsonException("OpenAI response envelope was empty."); return response.Output? .SelectMany(item => item.Content ?? []) .FirstOrDefault(content => string.Equals(content.Type, "output_text", StringComparison.OrdinalIgnoreCase) && !string.IsNullOrWhiteSpace(content.Text)) ?.Text ?.Trim() ?? throw new JsonException("OpenAI response did not contain output_text content."); } private static string? ExtractResponseId(string rawResponseText) { try { return JsonSerializer.Deserialize(rawResponseText, JsonOptions)?.Id; } catch (JsonException) { return null; } } private static int? ExtractCachedTokens(string rawResponseText) { try { return JsonSerializer.Deserialize(rawResponseText, JsonOptions)?.Usage?.InputTokenDetails?.CachedTokens; } catch (JsonException) { return null; } } private sealed class WholeBookOpenAIResponseEnvelope { public string? Id { get; init; } public List? Output { get; init; } public WholeBookOpenAIResponseUsage? Usage { get; init; } } private sealed class WholeBookOpenAIResponseOutputItem { public List? Content { get; init; } } private sealed class WholeBookOpenAIResponseContentItem { public string? Type { get; init; } public string? Text { get; init; } } private sealed class WholeBookOpenAIResponseUsage { [JsonPropertyName("input_tokens")] public int? InputTokens { get; init; } [JsonPropertyName("output_tokens")] public int? OutputTokens { get; init; } [JsonPropertyName("input_tokens_details")] public WholeBookOpenAIInputTokenDetails? InputTokenDetails { get; init; } } private sealed class WholeBookOpenAIInputTokenDetails { [JsonPropertyName("cached_tokens")] public int? CachedTokens { get; init; } } } public static class WholeBookPlotSynthesisStructuredOutputSchema { public static readonly StoryIntelligenceResponseContract Contract = new() { Name = "whole_book_plot_synthesis", Strict = true, Schema = new Dictionary { ["type"] = "object", ["additionalProperties"] = false, ["required"] = new[] { "metadata", "plotLines" }, ["properties"] = new Dictionary { ["metadata"] = ObjectSchema(new Dictionary { ["bookStatus"] = StringSchema(), ["summary"] = StringSchema() }, "bookStatus", "summary"), ["plotLines"] = ArraySchema(ObjectSchema(new Dictionary { ["key"] = StringSchema(), ["proposedTitle"] = StringSchema(), ["description"] = StringSchema(), ["significance"] = StringSchema(), ["bookStatus"] = StringSchema(), ["existingPlotLineID"] = NullableIntegerSchema(), ["existingPlotLineMatchConfidence"] = NullableNumberSchema(), ["existingPlotLineMatchReason"] = NullableStringSchema(), ["authorPlotSeedID"] = NullableIntegerSchema(), ["seedMatchConfidence"] = NullableNumberSchema(), ["seedMatchReason"] = NullableStringSchema(), ["confidence"] = NullableNumberSchema(), ["threads"] = ArraySchema(ObjectSchema(new Dictionary { ["key"] = StringSchema(), ["proposedTitle"] = StringSchema(), ["description"] = StringSchema(), ["status"] = StringSchema(), ["existingPlotThreadID"] = NullableIntegerSchema(), ["existingThreadMatchConfidence"] = NullableNumberSchema(), ["matchReason"] = NullableStringSchema(), ["confidence"] = NullableNumberSchema(), ["events"] = ArraySchema(ObjectSchema(new Dictionary { ["key"] = StringSchema(), ["sceneID"] = IntegerSchema(), ["eventType"] = EnumSchema("Introduced", "Mentioned", "Developed", "Complicated", "Clue Planted", "Question Raised", "Question Answered", "False Lead", "Revealed", "Payoff", "Resolved", "Contradicted", "Retconned", "Note"), ["title"] = StringSchema(), ["description"] = StringSchema(), ["evidence"] = StringSchema(), ["confidence"] = NullableNumberSchema() }, "key", "sceneID", "eventType", "title", "description", "evidence", "confidence")) }, "key", "proposedTitle", "description", "status", "existingPlotThreadID", "existingThreadMatchConfidence", "matchReason", "confidence", "events")) }, "key", "proposedTitle", "description", "significance", "bookStatus", "existingPlotLineID", "existingPlotLineMatchConfidence", "existingPlotLineMatchReason", "authorPlotSeedID", "seedMatchConfidence", "seedMatchReason", "confidence", "threads")) } } }; private static Dictionary ObjectSchema(Dictionary properties, params string[] required) => new() { ["type"] = "object", ["additionalProperties"] = false, ["required"] = required, ["properties"] = properties }; private static Dictionary ArraySchema(object items) => new() { ["type"] = "array", ["items"] = items }; private static Dictionary StringSchema() => new() { ["type"] = "string" }; private static Dictionary IntegerSchema() => new() { ["type"] = "integer" }; private static Dictionary NullableIntegerSchema() => new() { ["type"] = new[] { "integer", "null" } }; private static Dictionary NullableNumberSchema() => new() { ["type"] = new[] { "number", "null" }, ["minimum"] = 0, ["maximum"] = 1 }; private static Dictionary NullableStringSchema() => new() { ["type"] = new[] { "string", "null" } }; private static Dictionary EnumSchema(params string[] values) => new() { ["type"] = "string", ["enum"] = values }; }