From b639b12163ee36fbab380e4ae7c5035137b2459c Mon Sep 17 00:00:00 2001 From: Nick Beckley Date: Sat, 29 Aug 2026 18:57:46 +0000 Subject: [PATCH] Implement whole-book plot intelligence --- PlotLine.Tests/Program.cs | 156 +++- PlotLine/Controllers/OnboardingController.cs | 113 ++- PlotLine/Data/Repositories.cs | 14 + .../WholeBookPlotIntelligenceRepository.cs | 365 ++++++++ PlotLine/Models/CoreModels.cs | 4 + PlotLine/Models/StoryIntelligenceModels.cs | 2 + .../StoryIntelligenceNarrativeModels.cs | 38 + .../Models/WholeBookPlotIntelligenceModels.cs | 211 +++++ PlotLine/Program.cs | 2 + .../OnboardingStoryIntelligenceService.cs | 23 + .../PersistedStoryIntelligenceWorker.cs | 12 + PlotLine/Services/ProjectAccessFilter.cs | 1 + ...StoryIntelligenceCharacterImportService.cs | 2 + ...StoryIntelligenceNarrativeReviewService.cs | 167 +++- .../StoryIntelligenceOpenAIInfrastructure.cs | 2 + .../WholeBookPlotIntelligenceService.cs | 414 +++++++++ .../174_Phase27_WholeBookPlotIntelligence.sql | 817 ++++++++++++++++++ PlotLine/ViewModels/OnboardingViewModels.cs | 57 ++ PlotLine/Views/Books/Details.cshtml | 8 +- .../Views/Onboarding/BuildComplete.cshtml | 4 +- .../StoryIntelligenceAuthorPlotContext.cshtml | 69 ++ .../StoryIntelligenceNarrative.cshtml | 63 +- .../StoryIntelligenceReviewCentre.cshtml | 80 +- PlotLine/appsettings.json | 1 + 24 files changed, 2561 insertions(+), 64 deletions(-) create mode 100644 PlotLine/Data/WholeBookPlotIntelligenceRepository.cs create mode 100644 PlotLine/Models/WholeBookPlotIntelligenceModels.cs create mode 100644 PlotLine/Services/WholeBookPlotIntelligenceService.cs create mode 100644 PlotLine/Sql/174_Phase27_WholeBookPlotIntelligence.sql create mode 100644 PlotLine/Views/Onboarding/StoryIntelligenceAuthorPlotContext.cshtml diff --git a/PlotLine.Tests/Program.cs b/PlotLine.Tests/Program.cs index 7bd6522..c8a65da 100644 --- a/PlotLine.Tests/Program.cs +++ b/PlotLine.Tests/Program.cs @@ -202,6 +202,9 @@ var tests = new (string Name, Action Test)[] ("Narrative candidate merge and parent fields are persisted", NarrativeCandidateMergeAndParentFieldsArePersisted), ("Narrative prompt uses one scene call with compact schema", NarrativePromptUsesOneSceneCallWithCompactSchema), ("Narrative production code remains fixture agnostic", NarrativeProductionCodeRemainsFixtureAgnostic), + ("Whole-book Plot Intelligence prompt carries full manuscript context", WholeBookPlotIntelligencePromptCarriesFullManuscriptContext), + ("Whole-book Plot Intelligence schema requires timeline event types", WholeBookPlotIntelligenceSchemaRequiresTimelineEventTypes), + ("Whole-book Plot Intelligence production code remains fixture agnostic", WholeBookPlotIntelligenceProductionCodeRemainsFixtureAgnostic), ("Story Intelligence narrative review creates canonical records via existing procs", StoryIntelligenceNarrativeReviewCreatesCanonicalRecordsViaExistingProcs), ("Story Intelligence narrative stage follows knowledge", StoryIntelligenceNarrativeStageFollowsKnowledge), ("Word Companion replays live current scene to writing panes", WordCompanionReplaysLiveCurrentSceneToWritingPanes), @@ -690,7 +693,7 @@ static void StoryIntelligenceReviewCentreUsesIndependentTileGrid() "\"Assets\"", "\"Knowledge\"", "\"Locations\"", - "\"Plot Lines & Threads\"", + "ReviewPlotSynthesis()", "\"Relationships\"", "ReviewMetrics" }; @@ -3820,12 +3823,12 @@ static void StoryIntelligenceStartsFromCoreImportComplete() var controller = File.ReadAllText(Path.Combine(root, "Controllers/OnboardingController.cs")); var service = File.ReadAllText(Path.Combine(root, "Services/OnboardingStoryIntelligenceService.cs")); - Assert(complete.Contains("asp-action=\"StartStoryIntelligence\"", StringComparison.Ordinal), "Core Import Complete should start Story Intelligence directly."); + Assert(complete.Contains("asp-action=\"StoryIntelligencePlotContext\"", StringComparison.Ordinal), "Core Import Complete should open the optional author Plot Line context screen before analysis."); Assert(!complete.Contains("asp-action=\"StoryIntelligence\"", StringComparison.Ordinal), "Core Import Complete should not route through the redundant ready page."); Assert(controller.Contains("public async Task StoryIntelligence()\n {", StringComparison.Ordinal) && controller.Contains("return RedirectToAction(nameof(BuildComplete)", StringComparison.Ordinal), "Old ready route should be a compatibility redirect."); Assert(controller.Contains("public async Task StoryIntelligenceContext()\n {\n return await StoryIntelligence();", StringComparison.Ordinal), "Old context route should not remain an active wizard page."); - Assert(controller.Contains("var job = await storyIntelligence.StartAsync();", StringComparison.Ordinal), "Story Intelligence start should not require portrait/context form fields."); + Assert(controller.Contains("StartStoryIntelligenceWithPlotContext", StringComparison.Ordinal), "Story Intelligence start should save optional Plot Line context before queuing analysis."); Assert(service.Contains("BookContextOptions.StoryEraLabel(book?.StoryEra)", StringComparison.Ordinal), "Story Intelligence should derive era from Book context."); Assert(service.Contains("BookContextOptions.PrimaryLocaleLabel(book?.PrimaryLocaleCode)", StringComparison.Ordinal), "Story Intelligence should derive locale from Book context."); } @@ -4542,7 +4545,7 @@ static void PermanentAnalyseMyBookStartsFromSavedBook() var service = File.ReadAllText(Path.Combine(root, "Services/OnboardingStoryIntelligenceService.cs")); var core = File.ReadAllText(Path.Combine(root, "Services/CoreServices.cs")); - Assert(bookView.Contains("StartBookStoryIntelligence", StringComparison.Ordinal), "Book details should expose a permanent Analyse My Book action."); + Assert(bookView.Contains("StoryIntelligenceBookPlotContext", StringComparison.Ordinal), "Book details should expose a permanent Analyse My Book action through author Plot Line context."); Assert(controller.Contains("story-intelligence/book/{bookId:int}/start", StringComparison.Ordinal), "Permanent Story Intelligence start should have a book-scoped route."); Assert(service.Contains("Task StartBookAsync(int bookId)", StringComparison.Ordinal), "Story Intelligence service should support starting from a saved book."); Assert(service.Contains("sources.GetChapterSourceAsync(chapter.ChapterID)", StringComparison.Ordinal), "Saved-book analysis should use durable persisted chapter text."); @@ -4985,7 +4988,7 @@ static void StoryIntelligenceStabilisationRoutesNormalUsersToBook() var startAction = ExtractBetween(controller, "public async Task StartStoryIntelligence()", "[HttpPost(\"story-intelligence/book/{bookId:int}/start\")]"); Assert(startAction.Contains("return RedirectToBook(progress);", StringComparison.Ordinal), "Fresh onboarding Story Intelligence start should return to the Book page."); Assert(!startAction.Contains("RedirectToAction(nameof(StoryIntelligenceProgress)", StringComparison.Ordinal), "Fresh onboarding start must not route normal users to the old progress page."); - Assert(buildComplete.Contains("asp-action=\"StartStoryIntelligence\"", StringComparison.Ordinal), "Core Import Complete should still start the analysis."); + Assert(buildComplete.Contains("asp-action=\"StoryIntelligencePlotContext\"", StringComparison.Ordinal), "Core Import Complete should open optional plot context before analysis."); Assert(overview.Contains("asp-controller=\"Books\" asp-action=\"Details\"", StringComparison.Ordinal), "Existing analysis overview should return to Book details."); Assert(context.Contains("asp-controller=\"Books\" asp-action=\"Details\"", StringComparison.Ordinal), "Existing analysis context page should return to Book details."); } @@ -5249,6 +5252,149 @@ static void NarrativeProductionCodeRemainsFixtureAgnostic() } } +static void WholeBookPlotIntelligencePromptCarriesFullManuscriptContext() +{ + var context = new WholeBookPlotSynthesisContext + { + ProjectID = 71, + BookID = 72, + BookTitle = "The Glass Meridian", + Characters = + [ + new WholeBookPlotSynthesisCharacter + { + CharacterID = 101, + CanonicalName = "Nora Ell", + Aliases = ["Nora", "Doctor Ell"] + } + ], + ExistingPlotLines = + [ + new WholeBookPlotSynthesisExistingPlotLine + { + PlotLineID = 201, + Name = "The eclipse ledger", + BookID = null, + Threads = + [ + new WholeBookPlotSynthesisExistingThread + { + PlotThreadID = 301, + Title = "The missing observatory pages", + Status = "Open" + } + ] + } + ], + AuthorSeeds = + [ + new AuthorPlotSeed + { + AuthorPlotSeedID = 401, + Description = "A public research rivalry hides a private collaboration that changes the expedition." + } + ], + Chapters = + [ + new WholeBookPlotSynthesisChapter + { + ChapterID = 501, + ChapterNumber = 1, + ChapterTitle = "North Window", + Scenes = + [ + new WholeBookPlotSynthesisScene + { + SceneID = 601, + SceneNumber = 1, + SceneTitle = "A coded invitation", + PovCharacterName = "Nora Ell", + StructuralSetting = "Hilltop observatory", + SourceText = "Nora finds a coded invitation tucked inside the observatory ledger." + }, + new WholeBookPlotSynthesisScene + { + SceneID = 602, + SceneNumber = 2, + SceneTitle = "The false chart", + PovCharacterName = "Nora Ell", + StructuralSetting = "Map room", + SourceText = "The false chart sends the expedition toward the wrong meridian." + } + ] + } + ], + SourceAudit = new WholeBookPlotSynthesisSourceAudit + { + SceneCount = 2, + SourceRowCount = 2, + MissingSourceCount = 0, + DuplicateSourceSceneCount = 0 + } + }; + + var prompt = InvokeWholeBookPrompt(context); + Assert(prompt.Contains("[BOOK]", StringComparison.Ordinal), "Whole-book prompt should include book marker."); + Assert(prompt.Contains("BookID: 72", StringComparison.Ordinal), "Whole-book prompt should include BookID."); + Assert(prompt.Contains("[CHAPTER 1]", StringComparison.Ordinal), "Whole-book prompt should include deterministic chapter markers."); + Assert(prompt.Contains("SceneID: 601", StringComparison.Ordinal) && prompt.Contains("SceneID: 602", StringComparison.Ordinal), "Whole-book prompt should include canonical SceneIDs."); + Assert(prompt.Contains("Nora finds a coded invitation", StringComparison.Ordinal), "Whole-book prompt should include full scene source text."); + Assert(prompt.Contains("AuthorPlotSeedID: 401", StringComparison.Ordinal), "Whole-book prompt should include persisted author seeds."); + Assert(prompt.Contains("ThreadID: 301", StringComparison.Ordinal), "Whole-book prompt should include existing series/project thread context."); +} + +static void WholeBookPlotIntelligenceSchemaRequiresTimelineEventTypes() +{ + var schema = JsonSerializer.SerializeToElement(WholeBookPlotSynthesisStructuredOutputSchema.Contract.Schema); + var eventSchema = schema.GetProperty("properties") + .GetProperty("plotLines").GetProperty("items").GetProperty("properties") + .GetProperty("threads").GetProperty("items").GetProperty("properties") + .GetProperty("events").GetProperty("items").GetProperty("properties"); + var eventTypes = eventSchema.GetProperty("eventType").GetProperty("enum") + .EnumerateArray() + .Select(item => item.GetString()) + .ToHashSet(StringComparer.OrdinalIgnoreCase); + + foreach (var expected in new[] { "Introduced", "Mentioned", "Developed", "Complicated", "Clue Planted", "Question Raised", "Question Answered", "False Lead", "Revealed", "Payoff", "Resolved", "Contradicted", "Retconned", "Note" }) + { + Assert(eventTypes.Contains(expected), $"Whole-book schema should permit ThreadEvent type {expected}."); + } + + Assert(WholeBookPlotSynthesisStructuredOutputSchema.Contract.Strict, "Whole-book schema should use strict structured output."); +} + +static void WholeBookPlotIntelligenceProductionCodeRemainsFixtureAgnostic() +{ + var root = Path.Combine(AppContext.BaseDirectory, "../../../../PlotLine"); + var productionFiles = new[] + { + "Services/WholeBookPlotIntelligenceService.cs", + "Data/WholeBookPlotIntelligenceRepository.cs", + "Views/Onboarding/StoryIntelligenceAuthorPlotContext.cshtml", + "Views/Onboarding/StoryIntelligenceReviewCentre.cshtml", + "Views/Onboarding/StoryIntelligenceNarrative.cshtml", + "Sql/174_Phase27_WholeBookPlotIntelligence.sql" + }; + var forbidden = new[] { "Memory Tin", "Triumph TR6", "Mrs Patterson", "Maggie", "Beth", "Rob", "Rosie", "Sophie", "Cindy", "Rick", "Alpha Flame" }; + + foreach (var relativePath in productionFiles) + { + var text = File.ReadAllText(Path.Combine(root, relativePath)); + foreach (var term in forbidden) + { + Assert(!text.Contains(term, StringComparison.OrdinalIgnoreCase), $"Whole-book production file {relativePath} must not contain fixture term {term}."); + } + } +} + +static string InvokeWholeBookPrompt(WholeBookPlotSynthesisContext context) +{ + var method = typeof(WholeBookPlotIntelligenceService).GetMethod( + "BuildPrompt", + System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Static); + return (string)(method?.Invoke(null, [context, true]) ?? throw new InvalidOperationException("Whole-book prompt builder was not found.")); +} + static void StoryIntelligenceSceneCallUsesStrictStructuredOutput() { string? requestBody = null; diff --git a/PlotLine/Controllers/OnboardingController.cs b/PlotLine/Controllers/OnboardingController.cs index 747ff62..90b219f 100644 --- a/PlotLine/Controllers/OnboardingController.cs +++ b/PlotLine/Controllers/OnboardingController.cs @@ -8,7 +8,11 @@ namespace PlotLine.Controllers; [Authorize] [Route("onboarding")] -public sealed class OnboardingController(IOnboardingService onboarding, IOnboardingStoryIntelligenceService storyIntelligence) : Controller +public sealed class OnboardingController( + IOnboardingService onboarding, + IOnboardingStoryIntelligenceService storyIntelligence, + IWholeBookPlotIntelligenceService wholeBookPlotIntelligence, + ICurrentUserService currentUser) : Controller { [HttpGet("")] public async Task Index() @@ -134,14 +138,80 @@ public sealed class OnboardingController(IOnboardingService onboarding, IOnboard [HttpPost("story-intelligence/start")] [ValidateAntiForgeryToken] public async Task StartStoryIntelligence() + => RedirectToAction(nameof(StoryIntelligencePlotContext)); + + [HttpGet("story-intelligence/plot-context")] + public async Task StoryIntelligencePlotContext() { - var job = await storyIntelligence.StartAsync(); - if (job is null) + var overview = await storyIntelligence.GetOverviewAsync(); + if (!overview.BookID.HasValue || !overview.CanStart) { TempData["OnboardingStoryIntelligenceError"] = "Review the manuscript scan before starting Story Intelligence."; return RedirectToAction(nameof(Index)); } + var model = await wholeBookPlotIntelligence.GetAuthorPlotContextAsync(overview.BookID.Value); + if (model is null) + { + return RedirectToAction(nameof(Index)); + } + + return View("StoryIntelligenceAuthorPlotContext", model); + } + + [HttpGet("story-intelligence/book/{bookId:int}/plot-context")] + public async Task StoryIntelligenceBookPlotContext(int bookId) + { + var existing = await storyIntelligence.ResumeBookAsync(bookId); + if (existing is not null) + { + TempData["ArchiveMessage"] = "Story Intelligence is already available for this book. Progress is shown in the page header."; + return RedirectToAction("Details", "Books", new { id = bookId }); + } + + var model = await wholeBookPlotIntelligence.GetAuthorPlotContextAsync(bookId); + if (model is null) + { + TempData["ArchiveError"] = "Story Intelligence could not find that book."; + return RedirectToAction("Details", "Books", new { id = bookId }); + } + + return View("StoryIntelligenceAuthorPlotContext", model); + } + + [HttpPost("story-intelligence/plot-context/start")] + [ValidateAntiForgeryToken] + public async Task StartStoryIntelligenceWithPlotContext(StoryIntelligenceAuthorPlotContextForm form) + { + if (form.ProjectID <= 0 || form.BookID <= 0) + { + TempData["OnboardingStoryIntelligenceError"] = "Story Intelligence could not find the selected book."; + return RedirectToAction(nameof(Index)); + } + + await wholeBookPlotIntelligence.SaveAuthorPlotContextAsync(form.ProjectID, form.BookID, form.PlotSeeds); + + OnboardingStoryIntelligenceBatch? job; + try + { + var overview = await storyIntelligence.GetOverviewAsync(); + var isCurrentOnboardingBook = overview.BookID == form.BookID && overview.PreviewID != Guid.Empty && overview.CanStart; + job = isCurrentOnboardingBook + ? await storyIntelligence.StartAsync() + : await storyIntelligence.StartBookAsync(form.BookID); + } + catch (InvalidOperationException ex) + { + TempData["ArchiveError"] = ex.Message; + return RedirectToAction("Details", "Books", new { id = form.BookID }); + } + + if (job is null) + { + TempData["OnboardingStoryIntelligenceError"] = "Review the manuscript scan before starting Story Intelligence."; + return RedirectToAction("Details", "Books", new { id = form.BookID }); + } + var progress = await storyIntelligence.GetProgressAsync(job.BatchID); if (progress is not null) { @@ -163,23 +233,7 @@ public sealed class OnboardingController(IOnboardingService onboarding, IOnboard return RedirectToAction("Details", "Books", new { id = bookId }); } - try - { - var job = await storyIntelligence.StartBookAsync(bookId); - if (job is null) - { - TempData["ArchiveError"] = "Story Intelligence could not find that book."; - return RedirectToAction("Details", "Books", new { id = bookId }); - } - - TempData["ArchiveMessage"] = "Story Intelligence analysis has started. You can keep working while PlotDirector analyses the book in the background."; - return RedirectToAction("Details", "Books", new { id = bookId }); - } - catch (InvalidOperationException ex) - { - TempData["ArchiveError"] = ex.Message; - return RedirectToAction("Details", "Books", new { id = bookId }); - } + return RedirectToAction(nameof(StoryIntelligenceBookPlotContext), new { bookId }); } [HttpPost("story-intelligence/skip")] @@ -578,6 +632,25 @@ public sealed class OnboardingController(IOnboardingService onboarding, IOnboard : RedirectToAction(nameof(StoryIntelligenceNarrative), new { batchId = form.BatchID }); } + [HttpPost("story-intelligence/plot-synthesis/retry")] + public async Task RetryStoryIntelligencePlotSynthesis(Guid batchId) + { + var model = await storyIntelligence.GetProgressAsync(batchId); + if (model is null) + { + return NotFound(); + } + + if (!currentUser.UserId.HasValue) + { + return Forbid(); + } + + await wholeBookPlotIntelligence.RetryAsync(model.BookID, currentUser.UserId.Value); + TempData["OnboardingStoryIntelligenceMessage"] = "Plot Lines analysis has been queued again."; + return RedirectToAction(nameof(StoryIntelligenceReviewCentre), new { batchId }); + } + [HttpGet("story-intelligence/narrative/complete")] public IActionResult StoryIntelligenceNarrativeComplete(Guid batchId) { diff --git a/PlotLine/Data/Repositories.cs b/PlotLine/Data/Repositories.cs index 58db14e..53f98d9 100644 --- a/PlotLine/Data/Repositories.cs +++ b/PlotLine/Data/Repositories.cs @@ -3260,6 +3260,13 @@ public sealed class StoryIntelligenceNarrativeCandidateRepository(ISqlConnection candidate.Significance, candidate.ExistingPlotLineID, candidate.ExistingPlotThreadID, + candidate.WholeBookPlotSynthesisRunID, + candidate.AuthorPlotSeedID, + candidate.ExistingPlotLineMatchConfidence, + candidate.ExistingPlotLineMatchReason, + candidate.SeedMatchConfidence, + candidate.SeedMatchReason, + candidate.Source, candidate.RelatedPlotLineID, candidate.ParentCandidateKey, candidate.ParentNarrativeCandidateID, @@ -3293,13 +3300,20 @@ public sealed class StoryIntelligenceNarrativeCandidateRepository(ISqlConnection commandType: CommandType.StoredProcedure); var candidates = (await result.ReadAsync()).ToList(); var evidence = (await result.ReadAsync()).ToList(); + var events = result.IsConsumed + ? [] + : (await result.ReadAsync()).ToList(); var evidenceByCandidate = evidence .GroupBy(item => item.NarrativeCandidateID) .ToDictionary(group => group.Key, group => (IReadOnlyList)group.ToList()); + var eventsByCandidate = events + .GroupBy(item => item.NarrativeCandidateID) + .ToDictionary(group => group.Key, group => (IReadOnlyList)group.ToList()); foreach (var candidate in candidates) { candidate.Evidence = evidenceByCandidate.TryGetValue(candidate.NarrativeCandidateID, out var items) ? items : []; + candidate.Events = eventsByCandidate.TryGetValue(candidate.NarrativeCandidateID, out var candidateEvents) ? candidateEvents : []; } return candidates; diff --git a/PlotLine/Data/WholeBookPlotIntelligenceRepository.cs b/PlotLine/Data/WholeBookPlotIntelligenceRepository.cs new file mode 100644 index 0000000..ce272c1 --- /dev/null +++ b/PlotLine/Data/WholeBookPlotIntelligenceRepository.cs @@ -0,0 +1,365 @@ +using System.Data; +using System.Text.Json; +using Dapper; +using PlotLine.Models; + +namespace PlotLine.Data; + +public interface IWholeBookPlotIntelligenceRepository +{ + Task SaveAuthorSeedsAsync(AuthorPlotSeedSaveRequest request); + Task> ListAuthorSeedsAsync(int bookId); + Task QueueAsync(WholeBookPlotSynthesisQueueRequest request); + Task RetryAsync(int bookId, int userId); + Task ClaimNextAsync(int leaseMinutes); + Task GetCurrentAsync(int bookId); + Task CompleteAsync(WholeBookPlotSynthesisCompletionRequest request, WholeBookPlotSynthesisResult result); + Task FailAsync(int runId, string errorMessage, string? errorDetail, long? durationMs); + Task BuildContextAsync(int projectId, int bookId); +} + +public sealed class WholeBookPlotIntelligenceRepository(ISqlConnectionFactory connectionFactory) : IWholeBookPlotIntelligenceRepository +{ + private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web); + + public async Task SaveAuthorSeedsAsync(AuthorPlotSeedSaveRequest request) + { + using var connection = connectionFactory.CreateConnection(); + await connection.ExecuteAsync( + "dbo.AuthorPlotSeed_SaveForBook", + new + { + request.ProjectID, + request.BookID, + SeedsJson = JsonSerializer.Serialize( + request.Descriptions + .Where(description => !string.IsNullOrWhiteSpace(description)) + .Select(description => new { description = description.Trim() }) + .ToList(), + JsonOptions) + }, + commandType: CommandType.StoredProcedure); + } + + public async Task> ListAuthorSeedsAsync(int bookId) + { + using var connection = connectionFactory.CreateConnection(); + var rows = await connection.QueryAsync( + "dbo.AuthorPlotSeed_ListByBook", + new { BookID = bookId }, + commandType: CommandType.StoredProcedure); + return rows.ToList(); + } + + public async Task QueueAsync(WholeBookPlotSynthesisQueueRequest request) + { + using var connection = connectionFactory.CreateConnection(); + return await connection.QuerySingleAsync( + "dbo.WholeBookPlotSynthesisRun_Queue", + new + { + request.ProjectID, + request.BookID, + request.UserID, + request.SourceStoryIntelligenceRunID, + request.CharacterResolutionVersion + }, + commandType: CommandType.StoredProcedure); + } + + public async Task RetryAsync(int bookId, int userId) + { + using var connection = connectionFactory.CreateConnection(); + return await connection.QuerySingleOrDefaultAsync( + "dbo.WholeBookPlotSynthesisRun_Retry", + new { BookID = bookId, UserID = userId }, + commandType: CommandType.StoredProcedure); + } + + public async Task ClaimNextAsync(int leaseMinutes) + { + using var connection = connectionFactory.CreateConnection(); + return await connection.QuerySingleOrDefaultAsync( + "dbo.WholeBookPlotSynthesisRun_ClaimNext", + new { LeaseMinutes = leaseMinutes }, + commandType: CommandType.StoredProcedure); + } + + public async Task GetCurrentAsync(int bookId) + { + using var connection = connectionFactory.CreateConnection(); + return await connection.QuerySingleOrDefaultAsync( + "dbo.WholeBookPlotSynthesisRun_GetCurrentByBook", + new { BookID = bookId }, + commandType: CommandType.StoredProcedure); + } + + public async Task CompleteAsync(WholeBookPlotSynthesisCompletionRequest request, WholeBookPlotSynthesisResult result) + { + using var connection = connectionFactory.CreateConnection(); + connection.Open(); + using var transaction = connection.BeginTransaction(); + try + { + await connection.ExecuteAsync( + "dbo.WholeBookPlotSynthesisRun_Complete", + new + { + request.WholeBookPlotSynthesisRunID, + request.Model, + request.OpenAIResponseID, + request.InputTokens, + request.OutputTokens, + request.CachedInputTokens, + request.TotalTokens, + request.DurationMs, + request.ResultJson + }, + transaction, + commandType: CommandType.StoredProcedure); + + foreach (var plotLine in result.PlotLines) + { + var candidateId = await connection.QuerySingleAsync( + "dbo.WholeBookPlotSynthesisCandidate_Save", + new + { + request.WholeBookPlotSynthesisRunID, + CandidateType = StoryIntelligenceNarrativeCandidateTypes.PlotLine, + CandidateKey = CandidateKey(StoryIntelligenceNarrativeCandidateTypes.PlotLine, plotLine.Key, plotLine.ProposedTitle), + plotLine.ProposedTitle, + ProposedDescription = plotLine.Description, + Category = "Plot Line", + InferredState = plotLine.BookStatus, + ChangeType = plotLine.BookStatus, + plotLine.Significance, + plotLine.ExistingPlotLineID, + ExistingPlotThreadID = (int?)null, + RelatedPlotLineID = (int?)null, + ParentCandidateKey = (string?)null, + ParentNarrativeCandidateID = (int?)null, + plotLine.AuthorPlotSeedID, + plotLine.ExistingPlotLineMatchConfidence, + plotLine.ExistingPlotLineMatchReason, + plotLine.SeedMatchConfidence, + plotLine.SeedMatchReason, + plotLine.Confidence + }, + transaction, + commandType: CommandType.StoredProcedure); + + var sortOrder = 0; + foreach (var thread in plotLine.Threads) + { + var threadCandidateId = await connection.QuerySingleAsync( + "dbo.WholeBookPlotSynthesisCandidate_Save", + new + { + request.WholeBookPlotSynthesisRunID, + CandidateType = StoryIntelligenceNarrativeCandidateTypes.Thread, + CandidateKey = CandidateKey(StoryIntelligenceNarrativeCandidateTypes.Thread, thread.Key, thread.ProposedTitle), + thread.ProposedTitle, + ProposedDescription = thread.Description, + Category = "Thread", + InferredState = thread.Status, + ChangeType = thread.Status, + Significance = plotLine.Significance, + ExistingPlotLineID = plotLine.ExistingPlotLineID, + thread.ExistingPlotThreadID, + RelatedPlotLineID = plotLine.ExistingPlotLineID, + ParentCandidateKey = CandidateKey(StoryIntelligenceNarrativeCandidateTypes.PlotLine, plotLine.Key, plotLine.ProposedTitle), + ParentNarrativeCandidateID = candidateId, + plotLine.AuthorPlotSeedID, + ExistingPlotLineMatchConfidence = thread.ExistingThreadMatchConfidence, + ExistingPlotLineMatchReason = thread.MatchReason, + SeedMatchConfidence = plotLine.SeedMatchConfidence, + SeedMatchReason = plotLine.SeedMatchReason, + thread.Confidence + }, + transaction, + commandType: CommandType.StoredProcedure); + + foreach (var item in thread.Events.Where(item => item.SceneID > 0)) + { + await connection.ExecuteAsync( + "dbo.WholeBookPlotSynthesisCandidateEvent_Save", + new + { + NarrativeCandidateID = threadCandidateId, + SynthesisRunID = request.WholeBookPlotSynthesisRunID, + EventKey = string.IsNullOrWhiteSpace(item.Key) + ? $"{thread.Key}:{item.SceneID}:{item.EventType}:{sortOrder}" + : item.Key, + item.SceneID, + item.EventType, + EventTitle = item.Title, + EventDescription = item.Description, + item.Evidence, + item.Confidence, + SortOrder = sortOrder++ + }, + transaction, + commandType: CommandType.StoredProcedure); + + await connection.ExecuteAsync( + "dbo.StoryIntelligenceNarrativeCandidateEvidence_Save", + new + { + NarrativeCandidateID = threadCandidateId, + item.SceneID, + ContributionType = item.EventType, + Evidence = item.Evidence, + SortOrder = sortOrder + }, + transaction, + commandType: CommandType.StoredProcedure); + } + } + } + + transaction.Commit(); + } + catch + { + transaction.Rollback(); + throw; + } + } + + public async Task FailAsync(int runId, string errorMessage, string? errorDetail, long? durationMs) + { + using var connection = connectionFactory.CreateConnection(); + await connection.ExecuteAsync( + "dbo.WholeBookPlotSynthesisRun_Fail", + new + { + WholeBookPlotSynthesisRunID = runId, + ErrorMessage = errorMessage, + ErrorDetail = errorDetail, + DurationMs = durationMs + }, + commandType: CommandType.StoredProcedure); + } + + public async Task BuildContextAsync(int projectId, int bookId) + { + using var connection = connectionFactory.CreateConnection(); + using var result = await connection.QueryMultipleAsync( + "dbo.WholeBookPlotSynthesis_ContextGet", + new { ProjectID = projectId, BookID = bookId }, + commandType: CommandType.StoredProcedure); + + var book = await result.ReadSingleAsync(); + var sourceRows = (await result.ReadAsync()).ToList(); + var characters = (await result.ReadAsync()).ToList(); + var aliases = (await result.ReadAsync()).ToList(); + var plotLines = (await result.ReadAsync()).ToList(); + var threads = (await result.ReadAsync()).ToList(); + var seeds = (await result.ReadAsync()).ToList(); + var audit = await result.ReadSingleAsync(); + + var aliasesByCharacter = aliases + .GroupBy(alias => alias.CharacterID) + .ToDictionary(group => group.Key, group => (IReadOnlyList)group.Select(alias => alias.Alias).ToList()); + characters = characters + .Select(character => new WholeBookPlotSynthesisCharacter + { + CharacterID = character.CharacterID, + CanonicalName = character.CanonicalName, + Aliases = aliasesByCharacter.GetValueOrDefault(character.CharacterID) ?? [] + }) + .ToList(); + + var threadsByPlotLine = threads + .GroupBy(thread => thread.PlotLineID) + .ToDictionary( + group => group.Key, + group => (IReadOnlyList)group.Select(thread => new WholeBookPlotSynthesisExistingThread + { + PlotThreadID = thread.PlotThreadID, + Title = thread.Title, + Summary = thread.Summary, + Status = thread.Status + }).ToList()); + plotLines = plotLines + .Select(plotLine => new WholeBookPlotSynthesisExistingPlotLine + { + PlotLineID = plotLine.PlotLineID, + Name = plotLine.Name, + Description = plotLine.Description, + Importance = plotLine.Importance, + BookID = plotLine.BookID, + Threads = threadsByPlotLine.GetValueOrDefault(plotLine.PlotLineID) ?? [] + }) + .ToList(); + + var chapters = sourceRows + .GroupBy(row => new { row.ChapterID, row.ChapterNumber, row.ChapterTitle }) + .Select(group => new WholeBookPlotSynthesisChapter + { + ChapterID = group.Key.ChapterID, + ChapterNumber = group.Key.ChapterNumber, + ChapterTitle = group.Key.ChapterTitle, + Scenes = group.Select(row => new WholeBookPlotSynthesisScene + { + SceneID = row.SceneID, + SceneNumber = row.SceneNumber, + SceneTitle = row.SceneTitle, + StructuralSummary = row.StructuralSummary, + StructuralSetting = row.StructuralSetting, + PovCharacterName = row.PovCharacterName, + SourceText = row.SourceText + }).ToList() + }) + .ToList(); + + return new WholeBookPlotSynthesisContext + { + ProjectID = projectId, + BookID = bookId, + BookTitle = book.BookTitle, + Chapters = chapters, + Characters = characters, + ExistingPlotLines = plotLines, + AuthorSeeds = seeds, + SourceAudit = audit + }; + } + + private static string CandidateKey(string type, string key, string title) + => $"{type}:{(string.IsNullOrWhiteSpace(key) ? title : key).Trim().ToLowerInvariant()}"; + + private sealed class ContextBookRow + { + public string BookTitle { get; init; } = string.Empty; + } + + private sealed class ContextSceneRow + { + public int ChapterID { get; init; } + public decimal ChapterNumber { get; init; } + public string ChapterTitle { get; init; } = string.Empty; + public int SceneID { get; init; } + public decimal SceneNumber { get; init; } + public string SceneTitle { get; init; } = string.Empty; + public string? StructuralSummary { get; init; } + public string? StructuralSetting { get; init; } + public string? PovCharacterName { get; init; } + public string SourceText { get; init; } = string.Empty; + } + + private sealed class ContextCharacterAliasRow + { + public int CharacterID { get; init; } + public string Alias { get; init; } = string.Empty; + } + + private sealed class WholeBookPlotSynthesisExistingThreadRow + { + public int PlotThreadID { get; init; } + public int PlotLineID { get; init; } + public string Title { get; init; } = string.Empty; + public string? Summary { get; init; } + public string? Status { get; init; } + } +} diff --git a/PlotLine/Models/CoreModels.cs b/PlotLine/Models/CoreModels.cs index d4bb1a5..add3a54 100644 --- a/PlotLine/Models/CoreModels.cs +++ b/PlotLine/Models/CoreModels.cs @@ -768,6 +768,10 @@ public sealed class ThreadEvent public IReadOnlyList TargetPlotLineIDs { get; set; } = []; public string EventTitle { get; set; } = string.Empty; public string? EventDescription { get; set; } + public string Source { get; set; } = "Author"; + public int? SourceReferenceID { get; set; } + public string? SourceKey { get; set; } + public decimal? Confidence { get; set; } public DateTime CreatedDate { get; set; } public DateTime UpdatedDate { get; set; } } diff --git a/PlotLine/Models/StoryIntelligenceModels.cs b/PlotLine/Models/StoryIntelligenceModels.cs index 4fd7631..4c0e425 100644 --- a/PlotLine/Models/StoryIntelligenceModels.cs +++ b/PlotLine/Models/StoryIntelligenceModels.cs @@ -101,6 +101,7 @@ public sealed class StoryIntelligenceOptions public string Model { get; init; } = string.Empty; public string ChapterStructureModel { get; init; } = string.Empty; public string SceneIntelligenceModel { get; init; } = string.Empty; + public string WholeBookPlotSynthesisModel { get; init; } = string.Empty; public string ImageGenerationModel { get; init; } = "gpt-image-1.5"; public string ImageGenerationSize { get; init; } = "1024x1024"; public string ImageGenerationQuality { get; init; } = "medium"; @@ -123,6 +124,7 @@ public sealed class StoryIntelligenceOptions public string EffectiveModel => FirstConfigured(Model); public string EffectiveChapterStructureModel => FirstConfigured(ChapterStructureModel, Model); public string EffectiveSceneIntelligenceModel => FirstConfigured(SceneIntelligenceModel, Model); + public string EffectiveWholeBookPlotSynthesisModel => FirstConfigured(WholeBookPlotSynthesisModel, Model); public static string BuildModelSummary(string? chapterStructureModel, string? sceneIntelligenceModel) => $"Chapter={FirstConfigured(chapterStructureModel)}; Scene={FirstConfigured(sceneIntelligenceModel)}"; diff --git a/PlotLine/Models/StoryIntelligenceNarrativeModels.cs b/PlotLine/Models/StoryIntelligenceNarrativeModels.cs index 770453e..0dccd6e 100644 --- a/PlotLine/Models/StoryIntelligenceNarrativeModels.cs +++ b/PlotLine/Models/StoryIntelligenceNarrativeModels.cs @@ -39,6 +39,13 @@ public sealed class StoryIntelligenceNarrativeCandidate public string? ExistingPlotLineName { get; set; } public int? ExistingPlotThreadID { get; set; } public string? ExistingPlotThreadTitle { get; set; } + public int? WholeBookPlotSynthesisRunID { get; set; } + public int? AuthorPlotSeedID { get; set; } + public decimal? ExistingPlotLineMatchConfidence { get; set; } + public string? ExistingPlotLineMatchReason { get; set; } + public decimal? SeedMatchConfidence { get; set; } + public string? SeedMatchReason { get; set; } + public string? Source { get; set; } public int? RelatedPlotLineID { get; set; } public string? RelatedPlotLineName { get; set; } public string? ParentCandidateKey { get; set; } @@ -57,6 +64,7 @@ public sealed class StoryIntelligenceNarrativeCandidate public DateTime CreatedUtc { get; set; } public DateTime UpdatedUtc { get; set; } public IReadOnlyList Evidence { get; set; } = []; + public IReadOnlyList Events { get; set; } = []; } public sealed class StoryIntelligenceNarrativeCandidateEvidence @@ -88,8 +96,38 @@ public sealed class StoryIntelligenceNarrativeCandidateSeed public string? Significance { get; init; } public int? ExistingPlotLineID { get; init; } public int? ExistingPlotThreadID { get; init; } + public int? WholeBookPlotSynthesisRunID { get; init; } + public int? AuthorPlotSeedID { get; init; } + public decimal? ExistingPlotLineMatchConfidence { get; init; } + public string? ExistingPlotLineMatchReason { get; init; } + public decimal? SeedMatchConfidence { get; init; } + public string? SeedMatchReason { get; init; } + public string? Source { get; init; } public int? RelatedPlotLineID { get; init; } public string? ParentCandidateKey { get; init; } public int? ParentNarrativeCandidateID { get; init; } public decimal? Confidence { get; init; } } + +public sealed class StoryIntelligenceNarrativeCandidateEvent +{ + public int NarrativeCandidateEventID { get; set; } + public int NarrativeCandidateID { get; set; } + public int? WholeBookPlotSynthesisRunID { get; set; } + public string EventKey { get; set; } = string.Empty; + public int SceneID { get; set; } + public decimal SceneNumber { get; set; } + public string SceneTitle { get; set; } = string.Empty; + public decimal ChapterNumber { get; set; } + public string ChapterTitle { get; set; } = string.Empty; + public string EventType { get; set; } = string.Empty; + public string EventTitle { get; set; } = string.Empty; + public string? EventDescription { get; set; } + public string? Evidence { get; set; } + public decimal? Confidence { get; set; } + public int SortOrder { get; set; } + public bool IsIgnored { get; set; } + public int? AcceptedThreadEventID { get; set; } + public DateTime CreatedUtc { get; set; } + public DateTime UpdatedUtc { get; set; } +} diff --git a/PlotLine/Models/WholeBookPlotIntelligenceModels.cs b/PlotLine/Models/WholeBookPlotIntelligenceModels.cs new file mode 100644 index 0000000..ba832bf --- /dev/null +++ b/PlotLine/Models/WholeBookPlotIntelligenceModels.cs @@ -0,0 +1,211 @@ +using System.Text.Json; + +namespace PlotLine.Models; + +public static class WholeBookPlotSynthesisStatuses +{ + public const string Queued = "Queued"; + public const string Running = "Running"; + public const string Completed = "Completed"; + public const string Failed = "Failed"; + public const string Superseded = "Superseded"; +} + +public static class WholeBookPlotSynthesisSources +{ + public const string Author = "Author"; + public const string WholeBook = "WholeBookPlotSynthesis"; + public const string SceneSignals = "SceneSignals"; +} + +public sealed class AuthorPlotSeed +{ + public int AuthorPlotSeedID { get; set; } + public int ProjectID { get; set; } + public int BookID { get; set; } + public string Description { get; set; } = string.Empty; + public string Status { get; set; } = "Active"; + public int? MatchedPlotLineID { get; set; } + public string Source { get; set; } = WholeBookPlotSynthesisSources.Author; + public DateTime CreatedUtc { get; set; } + public DateTime UpdatedUtc { get; set; } +} + +public sealed class AuthorPlotSeedSaveRequest +{ + public int ProjectID { get; init; } + public int BookID { get; init; } + public IReadOnlyList Descriptions { get; init; } = []; +} + +public sealed class WholeBookPlotSynthesisRun +{ + public int WholeBookPlotSynthesisRunID { get; set; } + public int ProjectID { get; set; } + public int BookID { get; set; } + public int UserID { get; set; } + public int? SourceStoryIntelligenceRunID { get; set; } + public string CharacterResolutionVersion { get; set; } = string.Empty; + public string Status { get; set; } = WholeBookPlotSynthesisStatuses.Queued; + public DateTime QueuedUtc { get; set; } + public DateTime? StartedUtc { get; set; } + public DateTime? CompletedUtc { get; set; } + public DateTime? FailedUtc { get; set; } + public int AttemptCount { get; set; } + public string? Model { get; set; } + public string? OpenAIResponseID { get; set; } + public int? InputTokens { get; set; } + public int? OutputTokens { get; set; } + public int? CachedInputTokens { get; set; } + public int? TotalTokens { get; set; } + public long? DurationMs { get; set; } + public string? ErrorMessage { get; set; } + public string? ErrorDetail { get; set; } + public string? ResultJson { get; set; } + public DateTime UpdatedUtc { get; set; } +} + +public sealed class WholeBookPlotSynthesisQueueRequest +{ + public int ProjectID { get; init; } + public int BookID { get; init; } + public int UserID { get; init; } + public int? SourceStoryIntelligenceRunID { get; init; } + public string CharacterResolutionVersion { get; init; } = string.Empty; +} + +public sealed class WholeBookPlotSynthesisCompletionRequest +{ + public int WholeBookPlotSynthesisRunID { get; init; } + public string Model { get; init; } = string.Empty; + public string? OpenAIResponseID { get; init; } + public int? InputTokens { get; init; } + public int? OutputTokens { get; init; } + public int? CachedInputTokens { get; init; } + public int? TotalTokens { get; init; } + public long? DurationMs { get; init; } + public string ResultJson { get; init; } = string.Empty; +} + +public sealed class WholeBookPlotSynthesisContext +{ + public int ProjectID { get; init; } + public int BookID { get; init; } + public string BookTitle { get; init; } = string.Empty; + public IReadOnlyList Chapters { get; init; } = []; + public IReadOnlyList Characters { get; init; } = []; + public IReadOnlyList ExistingPlotLines { get; init; } = []; + public IReadOnlyList AuthorSeeds { get; init; } = []; + public WholeBookPlotSynthesisSourceAudit SourceAudit { get; init; } = new(); +} + +public sealed class WholeBookPlotSynthesisSourceAudit +{ + public int SceneCount { get; init; } + public int SourceRowCount { get; init; } + public int MissingSourceCount { get; init; } + public int DuplicateSourceSceneCount { get; init; } + public bool CanReconstruct => SceneCount > 0 && SceneCount == SourceRowCount && MissingSourceCount == 0 && DuplicateSourceSceneCount == 0; +} + +public sealed class WholeBookPlotSynthesisChapter +{ + public int ChapterID { get; init; } + public decimal ChapterNumber { get; init; } + public string ChapterTitle { get; init; } = string.Empty; + public IReadOnlyList Scenes { get; init; } = []; +} + +public sealed class WholeBookPlotSynthesisScene +{ + public int SceneID { get; init; } + public decimal SceneNumber { get; init; } + public string SceneTitle { get; init; } = string.Empty; + public string? StructuralSummary { get; init; } + public string? StructuralSetting { get; init; } + public string? PovCharacterName { get; init; } + public string SourceText { get; init; } = string.Empty; +} + +public sealed class WholeBookPlotSynthesisCharacter +{ + public int CharacterID { get; init; } + public string CanonicalName { get; init; } = string.Empty; + public IReadOnlyList Aliases { get; init; } = []; +} + +public sealed class WholeBookPlotSynthesisExistingPlotLine +{ + public int PlotLineID { get; init; } + public string Name { get; init; } = string.Empty; + public string? Description { get; init; } + public string? Importance { get; init; } + public int? BookID { get; init; } + public IReadOnlyList Threads { get; init; } = []; +} + +public sealed class WholeBookPlotSynthesisExistingThread +{ + public int PlotThreadID { get; init; } + public string Title { get; init; } = string.Empty; + public string? Summary { get; init; } + public string? Status { get; init; } +} + +public sealed class WholeBookPlotSynthesisResult +{ + public WholeBookPlotSynthesisMetadata Metadata { get; init; } = new(); + public IReadOnlyList PlotLines { get; init; } = []; + + public static WholeBookPlotSynthesisResult FromJson(string json) + => JsonSerializer.Deserialize( + json, + new JsonSerializerOptions { PropertyNameCaseInsensitive = true }) ?? new(); +} + +public sealed class WholeBookPlotSynthesisMetadata +{ + public string BookStatus { get; init; } = string.Empty; + public string Summary { get; init; } = string.Empty; +} + +public sealed class WholeBookPlotSynthesisPlotLineResult +{ + public string Key { get; init; } = string.Empty; + public string ProposedTitle { get; init; } = string.Empty; + public string Description { get; init; } = string.Empty; + public string Significance { get; init; } = string.Empty; + public string BookStatus { get; init; } = string.Empty; + public int? ExistingPlotLineID { get; init; } + public decimal? ExistingPlotLineMatchConfidence { get; init; } + public string? ExistingPlotLineMatchReason { get; init; } + public int? AuthorPlotSeedID { get; init; } + public decimal? SeedMatchConfidence { get; init; } + public string? SeedMatchReason { get; init; } + public decimal? Confidence { get; init; } + public IReadOnlyList Threads { get; init; } = []; +} + +public sealed class WholeBookPlotSynthesisThreadResult +{ + public string Key { get; init; } = string.Empty; + public string ProposedTitle { get; init; } = string.Empty; + public string Description { get; init; } = string.Empty; + public string Status { get; init; } = string.Empty; + public int? ExistingPlotThreadID { get; init; } + public decimal? ExistingThreadMatchConfidence { get; init; } + public string? MatchReason { get; init; } + public decimal? Confidence { get; init; } + public IReadOnlyList Events { get; init; } = []; +} + +public sealed class WholeBookPlotSynthesisEventResult +{ + public string Key { get; init; } = string.Empty; + public int SceneID { get; init; } + public string EventType { get; init; } = string.Empty; + public string Title { get; init; } = string.Empty; + public string Description { get; init; } = string.Empty; + public string Evidence { get; init; } = string.Empty; + public decimal? Confidence { get; init; } +} diff --git a/PlotLine/Program.cs b/PlotLine/Program.cs index 61297f3..4ae9170 100644 --- a/PlotLine/Program.cs +++ b/PlotLine/Program.cs @@ -150,6 +150,7 @@ public class Program builder.Services.AddScoped(); builder.Services.AddScoped(); builder.Services.AddScoped(); + builder.Services.AddScoped(); builder.Services.AddScoped(); builder.Services.AddScoped(); builder.Services.AddScoped(); @@ -230,6 +231,7 @@ public class Program builder.Services.AddScoped(); builder.Services.AddScoped(); builder.Services.AddScoped(); + builder.Services.AddScoped(); builder.Services.AddScoped(); builder.Services.AddScoped(); builder.Services.AddScoped(); diff --git a/PlotLine/Services/OnboardingStoryIntelligenceService.cs b/PlotLine/Services/OnboardingStoryIntelligenceService.cs index 9198193..e99388c 100644 --- a/PlotLine/Services/OnboardingStoryIntelligenceService.cs +++ b/PlotLine/Services/OnboardingStoryIntelligenceService.cs @@ -43,6 +43,7 @@ public sealed class OnboardingStoryIntelligenceService( IStoryIntelligenceRelationshipImportService relationshipImport, IStoryIntelligenceKnowledgeImportService knowledgeImport, IStoryIntelligenceNarrativeReviewService narrativeReview, + IWholeBookPlotIntelligenceService wholeBookPlotIntelligence, IStoryIntelligenceSourceRepository sources, IStoryIntelligencePipelineStateService pipelineState, IStoryMemoryRepository storyMemory, @@ -644,7 +645,12 @@ public sealed class OnboardingStoryIntelligenceService( var relationshipReview = await relationshipImport.BuildReviewAsync(batch); var knowledgeReview = await knowledgeImport.BuildReviewAsync(batch); var narrative = await narrativeReview.GetReviewAsync(batch); + var plotSynthesisRun = await wholeBookPlotIntelligence.GetCurrentAsync(batch.BookID); var pipeline = await pipelineState.GetForBookAsync(batch.BookID, batch.UserID); + var narrativePendingCount = narrative.Candidates.Count; + var narrativeThreadCount = narrative.Candidates.Count(candidate => + string.Equals(candidate.CandidateType, StoryIntelligenceNarrativeCandidateTypes.Thread, StringComparison.OrdinalIgnoreCase)); + var narrativeEventCount = narrative.Candidates.Sum(candidate => candidate.Events.Count); return new StoryIntelligenceProgressViewModel { BatchID = batch.BatchID, @@ -661,6 +667,23 @@ public sealed class OnboardingStoryIntelligenceService( RelationshipReview = relationshipReview, KnowledgeReview = knowledgeReview, NarrativeReview = narrative, + PlotSynthesis = plotSynthesisRun is null + ? new WholeBookPlotSynthesisStatusViewModel + { + Status = string.Empty, + CandidateCount = narrativePendingCount, + ThreadCount = narrativeThreadCount, + EventCount = narrativeEventCount + } + : new WholeBookPlotSynthesisStatusViewModel + { + WholeBookPlotSynthesisRunID = plotSynthesisRun.WholeBookPlotSynthesisRunID, + Status = plotSynthesisRun.Status, + CandidateCount = narrativePendingCount, + ThreadCount = narrativeThreadCount, + EventCount = narrativeEventCount, + ErrorMessage = plotSynthesisRun.ErrorMessage + }, PipelineDashboard = new StoryIntelligencePipelineDashboardViewModel { ChaptersAnalysed = chapters.Count(chapter => chapter.IsRunComplete), diff --git a/PlotLine/Services/PersistedStoryIntelligenceWorker.cs b/PlotLine/Services/PersistedStoryIntelligenceWorker.cs index 72f3cc6..4d5d704 100644 --- a/PlotLine/Services/PersistedStoryIntelligenceWorker.cs +++ b/PlotLine/Services/PersistedStoryIntelligenceWorker.cs @@ -51,11 +51,23 @@ public sealed class PersistedStoryIntelligenceWorker( if (!processedRun) { await RecoverReadySceneAnalysisAsync(stoppingToken); + processedRun = await ProcessWholeBookPlotSynthesisAsync(stoppingToken); + } + + if (!processedRun) + { await Task.Delay(pollDelay, stoppingToken); } } } + private async Task ProcessWholeBookPlotSynthesisAsync(CancellationToken stoppingToken) + { + using var scope = scopeFactory.CreateScope(); + var synthesis = scope.ServiceProvider.GetRequiredService(); + return await synthesis.ProcessNextAsync(stoppingToken); + } + private async Task RecoverReadySceneAnalysisAsync(CancellationToken stoppingToken) { using var scope = scopeFactory.CreateScope(); diff --git a/PlotLine/Services/ProjectAccessFilter.cs b/PlotLine/Services/ProjectAccessFilter.cs index 5549584..f8ec3c0 100644 --- a/PlotLine/Services/ProjectAccessFilter.cs +++ b/PlotLine/Services/ProjectAccessFilter.cs @@ -90,6 +90,7 @@ public sealed class ProjectAccessFilter(IProjectAccessService access, ICurrentUs ["Dynamics"] = new(StringComparer.OrdinalIgnoreCase) { ["projectId"] = "Project", ["bookId"] = "Book", ["sceneId"] = "Scene" }, ["Exports"] = new(StringComparer.OrdinalIgnoreCase) { ["projectId"] = "Project", ["sceneId"] = "Scene" }, ["Scenes"] = new(StringComparer.OrdinalIgnoreCase) { ["chapterId"] = "Chapter", ["sceneId"] = "Scene", ["projectId"] = "Project", ["bookId"] = "Book" }, + ["Onboarding"] = new(StringComparer.OrdinalIgnoreCase) { ["bookId"] = "Book" }, ["Acceptance"] = new(StringComparer.OrdinalIgnoreCase) { ["projectId"] = "Project" }, ["StoryBible"] = new(StringComparer.OrdinalIgnoreCase) { ["projectId"] = "Project" } }; diff --git a/PlotLine/Services/StoryIntelligenceCharacterImportService.cs b/PlotLine/Services/StoryIntelligenceCharacterImportService.cs index c20d093..61dae74 100644 --- a/PlotLine/Services/StoryIntelligenceCharacterImportService.cs +++ b/PlotLine/Services/StoryIntelligenceCharacterImportService.cs @@ -19,6 +19,7 @@ public sealed class StoryIntelligenceCharacterImportService( IStoryIntelligencePipelineStateService pipelineState, INameIntelligenceService nameIntelligence, ICoreImportCharacterMappingService coreImportCharacterMapping, + IWholeBookPlotIntelligenceService wholeBookPlotIntelligence, ICurrentUserService currentUser, ILogger logger) : IStoryIntelligenceCharacterImportService { @@ -372,6 +373,7 @@ public sealed class StoryIntelligenceCharacterImportService( }; var remap = await coreImportCharacterMapping.MapBookAsync(batch.ProjectID, batch.BookID); await pipelineState.RecordCharacterImportAsync(batch.ProjectID, batch.BookID); + await wholeBookPlotIntelligence.QueueAfterCharacterResolutionAsync(batch.ProjectID, batch.BookID, batch.UserID); logger.LogInformation( "Imported Story Intelligence characters for batch {BatchID}. Created={Created} LinkedExisting={LinkedExisting} Aliased={Aliased} Ignored={Ignored} LinkedAppearances={LinkedAppearances} PovLinks={PovLinks} RemappedScenes={RemappedScenes} RemappedLinks={RemappedLinks}", diff --git a/PlotLine/Services/StoryIntelligenceNarrativeReviewService.cs b/PlotLine/Services/StoryIntelligenceNarrativeReviewService.cs index bc66c1a..6cf198c 100644 --- a/PlotLine/Services/StoryIntelligenceNarrativeReviewService.cs +++ b/PlotLine/Services/StoryIntelligenceNarrativeReviewService.cs @@ -33,8 +33,9 @@ public sealed class StoryIntelligenceNarrativeReviewService( public async Task GetReviewAsync(OnboardingStoryIntelligenceBatch batch) { - await EnsureCandidatesAsync(batch); - var rows = await candidates.ListByBookAsync(batch.BookID); + var rows = (await candidates.ListByBookAsync(batch.BookID)) + .Where(IsWholeBookCandidate) + .ToList(); var pending = rows .Where(candidate => string.Equals(candidate.Status, StoryIntelligenceNarrativeCandidateStatuses.Pending, StringComparison.OrdinalIgnoreCase)) .ToList(); @@ -72,8 +73,9 @@ public sealed class StoryIntelligenceNarrativeReviewService( public async Task ImportAsync(OnboardingStoryIntelligenceBatch batch, StoryIntelligenceNarrativeImportForm form, int userId) { - await EnsureCandidatesAsync(batch); - var rows = (await candidates.ListByBookAsync(batch.BookID)).ToDictionary(candidate => candidate.NarrativeCandidateID); + var rows = (await candidates.ListByBookAsync(batch.BookID)) + .Where(IsWholeBookCandidate) + .ToDictionary(candidate => candidate.NarrativeCandidateID); var lookups = await plots.GetLookupsAsync(); var plotLineTypeId = lookups.PlotLineTypes.FirstOrDefault(type => string.Equals(type.TypeName, "Main Plot", StringComparison.OrdinalIgnoreCase))?.PlotLineTypeID ?? lookups.PlotLineTypes.FirstOrDefault()?.PlotLineTypeID @@ -87,9 +89,12 @@ public sealed class StoryIntelligenceNarrativeReviewService( var threadStatusId = lookups.ThreadStatuses.FirstOrDefault(status => string.Equals(status.StatusName, "Open", StringComparison.OrdinalIgnoreCase))?.ThreadStatusID ?? lookups.ThreadStatuses.FirstOrDefault()?.ThreadStatusID ?? 0; - var threadEventTypeId = lookups.ThreadEventTypes.FirstOrDefault(type => string.Equals(type.TypeName, "Raised", StringComparison.OrdinalIgnoreCase))?.ThreadEventTypeID - ?? lookups.ThreadEventTypes.FirstOrDefault()?.ThreadEventTypeID - ?? 0; + var threadEventTypeByName = lookups.ThreadEventTypes + .GroupBy(type => type.TypeName, StringComparer.OrdinalIgnoreCase) + .ToDictionary(group => group.Key, group => group.First().ThreadEventTypeID, StringComparer.OrdinalIgnoreCase); + var threadEventTypeId = threadEventTypeByName.TryGetValue("Mentioned", out var mentionedTypeId) + ? mentionedTypeId + : lookups.ThreadEventTypes.FirstOrDefault()?.ThreadEventTypeID ?? 0; var plotEventTypeId = lookups.PlotEventTypes.FirstOrDefault(type => string.Equals(type.TypeName, "Progress", StringComparison.OrdinalIgnoreCase))?.PlotEventTypeID ?? lookups.PlotEventTypes.FirstOrDefault()?.PlotEventTypeID ?? 0; @@ -221,7 +226,9 @@ public sealed class StoryIntelligenceNarrativeReviewService( continue; } - var firstSceneId = candidate.Evidence.OrderBy(evidence => evidence.SortOrder).FirstOrDefault()?.SceneID; + var effectiveEvents = EffectiveEvents(candidate, rows, choices).ToList(); + var firstSceneId = effectiveEvents.OrderBy(item => item.SortOrder).FirstOrDefault()?.SceneID + ?? candidate.Evidence.OrderBy(evidence => evidence.SortOrder).FirstOrDefault()?.SceneID; var threadId = await SavePlotThreadAsync(connection, transaction, new PlotThread { PlotLineID = relatedPlotLineId.Value, @@ -233,17 +240,47 @@ public sealed class StoryIntelligenceNarrativeReviewService( IntroducedSceneID = firstSceneId }); - foreach (var evidence in EffectiveEvidence(candidate, rows, choices).Where(evidence => evidence.SceneID > 0)) + if (effectiveEvents.Count > 0) { - await SaveThreadEventAsync(connection, transaction, new ThreadEvent + foreach (var threadEvent in effectiveEvents.Where(item => item.SceneID > 0)) { - PlotThreadID = threadId, - SceneID = evidence.SceneID, - EventTypeID = threadEventTypeId, - PlotEventTypeID = plotEventTypeId, - EventTitle = string.IsNullOrWhiteSpace(evidence.ContributionType) ? candidate.ProposedTitle : evidence.ContributionType, - EventDescription = evidence.Evidence - }); + await SaveThreadEventAsync(connection, transaction, new ThreadEvent + { + PlotThreadID = threadId, + SceneID = threadEvent.SceneID, + EventTypeID = threadEventTypeByName.GetValueOrDefault( + string.IsNullOrWhiteSpace(threadEvent.EventType) ? "Mentioned" : threadEvent.EventType, + threadEventTypeId), + PlotEventTypeID = plotEventTypeId, + EventTitle = CleanTitle(threadEvent.EventTitle, candidate.ProposedTitle), + EventDescription = CleanNullable(threadEvent.EventDescription) ?? threadEvent.Evidence, + Source = WholeBookPlotSynthesisSources.WholeBook, + SourceReferenceID = threadEvent.NarrativeCandidateEventID, + SourceKey = $"{threadEvent.WholeBookPlotSynthesisRunID}:{threadEvent.NarrativeCandidateID}:{threadEvent.EventKey}", + Confidence = threadEvent.Confidence + }); + } + } + else + { + foreach (var evidence in EffectiveEvidence(candidate, rows, choices).Where(evidence => evidence.SceneID > 0)) + { + await SaveThreadEventAsync(connection, transaction, new ThreadEvent + { + PlotThreadID = threadId, + SceneID = evidence.SceneID, + EventTypeID = threadEventTypeByName.GetValueOrDefault( + string.IsNullOrWhiteSpace(evidence.ContributionType) ? "Mentioned" : evidence.ContributionType, + threadEventTypeId), + PlotEventTypeID = plotEventTypeId, + EventTitle = string.IsNullOrWhiteSpace(evidence.ContributionType) ? candidate.ProposedTitle : evidence.ContributionType, + EventDescription = evidence.Evidence, + Source = WholeBookPlotSynthesisSources.WholeBook, + SourceReferenceID = candidate.NarrativeCandidateID, + SourceKey = $"{candidate.WholeBookPlotSynthesisRunID}:{candidate.NarrativeCandidateID}:{evidence.SceneID}:{evidence.SortOrder}", + Confidence = candidate.Confidence + }); + } } await MarkReviewedAsync(connection, transaction, candidate.NarrativeCandidateID, StoryIntelligenceNarrativeCandidateStatuses.Accepted, action, userId, null, threadId); @@ -503,7 +540,11 @@ public sealed class StoryIntelligenceNarrativeReviewService( threadEvent.PlotEventTypeID, threadEvent.TargetPlotLineID, threadEvent.EventTitle, - threadEvent.EventDescription + threadEvent.EventDescription, + threadEvent.Source, + threadEvent.SourceReferenceID, + threadEvent.SourceKey, + threadEvent.Confidence }, transaction, commandType: CommandType.StoredProcedure); @@ -573,9 +614,24 @@ public sealed class StoryIntelligenceNarrativeReviewService( SceneLabel = $"Chapter {evidence.ChapterNumber:0.##}, Scene {evidence.SceneNumber:0.##}: {evidence.SceneTitle}", ContributionType = evidence.ContributionType ?? string.Empty, Evidence = evidence.Evidence ?? string.Empty + }).ToList(), + Events = candidate.Events.Select(item => new StoryIntelligenceNarrativeEventViewModel + { + NarrativeCandidateEventID = item.NarrativeCandidateEventID, + SceneID = item.SceneID, + SceneLabel = $"Chapter {item.ChapterNumber:0.##}, Scene {item.SceneNumber:0.##}: {item.SceneTitle}", + EventType = item.EventType, + EventTitle = item.EventTitle, + EventDescription = item.EventDescription ?? string.Empty, + Evidence = item.Evidence ?? string.Empty, + Confidence = item.Confidence.HasValue ? $"{item.Confidence.Value:P0}" : "Unknown" }).ToList() }; + private static bool IsWholeBookCandidate(StoryIntelligenceNarrativeCandidate candidate) + => string.Equals(candidate.Source, WholeBookPlotSynthesisSources.WholeBook, StringComparison.OrdinalIgnoreCase) + || candidate.WholeBookPlotSynthesisRunID.HasValue; + private static Scene? ResolveImportedScene( StoryIntelligenceSavedSceneResult sceneResult, IReadOnlyDictionary importedById, @@ -709,6 +765,81 @@ public sealed class StoryIntelligenceNarrativeReviewService( .ToList(); } + private static IReadOnlyList EffectiveEvents( + StoryIntelligenceNarrativeCandidate candidate, + IReadOnlyDictionary candidates, + IReadOnlyDictionary choices) + { + var rows = new List(); + AddEvents(candidate, rows, choices); + foreach (var choice in choices.Values) + { + if (choice.MergeTargetNarrativeCandidateID != candidate.NarrativeCandidateID + || !string.Equals(choice.Action, StoryIntelligenceNarrativeReviewActions.MergeCandidate, StringComparison.OrdinalIgnoreCase) + || !candidates.TryGetValue(choice.NarrativeCandidateID, out var merged) + || !string.Equals(merged.CandidateType, candidate.CandidateType, StringComparison.OrdinalIgnoreCase)) + { + continue; + } + + AddEvents(merged, rows, choices); + } + + return rows + .Where(item => item.SceneID > 0) + .GroupBy(item => new { item.NarrativeCandidateEventID, item.SceneID, item.EventKey }) + .Select(group => group.First()) + .OrderBy(item => item.SortOrder) + .ThenBy(item => item.SceneID) + .ToList(); + } + + private static void AddEvents( + StoryIntelligenceNarrativeCandidate candidate, + List rows, + IReadOnlyDictionary choices) + { + var eventChoices = choices.TryGetValue(candidate.NarrativeCandidateID, out var choice) + ? choice.Events.ToDictionary(item => item.NarrativeCandidateEventID) + : new Dictionary(); + + foreach (var item in candidate.Events) + { + if (eventChoices.TryGetValue(item.NarrativeCandidateEventID, out var eventChoice)) + { + if (!eventChoice.Include) + { + continue; + } + + rows.Add(new StoryIntelligenceNarrativeCandidateEvent + { + NarrativeCandidateEventID = item.NarrativeCandidateEventID, + NarrativeCandidateID = item.NarrativeCandidateID, + WholeBookPlotSynthesisRunID = item.WholeBookPlotSynthesisRunID, + EventKey = item.EventKey, + SceneID = item.SceneID, + SceneNumber = item.SceneNumber, + SceneTitle = item.SceneTitle, + ChapterNumber = item.ChapterNumber, + ChapterTitle = item.ChapterTitle, + EventType = CleanTitle(eventChoice.EventType, item.EventType), + EventTitle = CleanTitle(eventChoice.EventTitle, item.EventTitle), + EventDescription = CleanNullable(eventChoice.EventDescription) ?? item.EventDescription, + Evidence = item.Evidence, + Confidence = item.Confidence, + SortOrder = item.SortOrder + }); + continue; + } + + if (!item.IsIgnored) + { + rows.Add(item); + } + } + } + private static string Key(string? value) => StoryIntelligenceNarrativeSignalConsolidator.Key(value); } diff --git a/PlotLine/Services/StoryIntelligenceOpenAIInfrastructure.cs b/PlotLine/Services/StoryIntelligenceOpenAIInfrastructure.cs index 66a0f1f..f01004d 100644 --- a/PlotLine/Services/StoryIntelligenceOpenAIInfrastructure.cs +++ b/PlotLine/Services/StoryIntelligenceOpenAIInfrastructure.cs @@ -158,6 +158,7 @@ public sealed class StoryIntelligenceClient( Model = settings.EffectiveModel, ChapterStructureModel = settings.EffectiveChapterStructureModel, SceneIntelligenceModel = settings.EffectiveSceneIntelligenceModel, + WholeBookPlotSynthesisModel = settings.EffectiveWholeBookPlotSynthesisModel, MaxOutputTokens = settings.MaxOutputTokens, TimeoutSeconds = settings.TimeoutSeconds, ClientConstructed = true, @@ -831,6 +832,7 @@ public sealed class StoryIntelligenceClientConfigurationStatus public string Model { get; init; } = string.Empty; public string ChapterStructureModel { get; init; } = string.Empty; public string SceneIntelligenceModel { get; init; } = string.Empty; + public string WholeBookPlotSynthesisModel { get; init; } = string.Empty; public int MaxOutputTokens { get; init; } public int TimeoutSeconds { get; init; } public bool ClientConstructed { get; init; } diff --git a/PlotLine/Services/WholeBookPlotIntelligenceService.cs b/PlotLine/Services/WholeBookPlotIntelligenceService.cs new file mode 100644 index 0000000..c06e7b7 --- /dev/null +++ b/PlotLine/Services/WholeBookPlotIntelligenceService.cs @@ -0,0 +1,414 @@ +using System.Diagnostics; +using System.Text; +using System.Text.Json; +using System.Text.Json.Serialization; +using Microsoft.Extensions.Options; +using PlotLine.Data; +using PlotLine.Models; +using PlotLine.ViewModels; + +namespace PlotLine.Services; + +public interface IWholeBookPlotIntelligenceService +{ + Task GetAuthorPlotContextAsync(int? bookId = null); + Task SaveAuthorPlotContextAsync(int projectId, int bookId, IReadOnlyList descriptions); + Task QueueAfterCharacterResolutionAsync(int projectId, int bookId, int userId); + Task RetryAsync(int bookId, int userId); + Task GetCurrentAsync(int bookId); + Task ProcessNextAsync(CancellationToken cancellationToken); + string BuildPromptForTest(WholeBookPlotSynthesisContext context); + StoryIntelligenceResponseContract BuildResponseContractForTest(); +} + +public sealed class WholeBookPlotIntelligenceService( + IWholeBookPlotIntelligenceRepository repository, + IStoryIntelligencePipelineRepository pipelines, + IBookRepository books, + IStoryIntelligenceClient client, + IOptions options, + IStoryIntelligenceProgressNotifier notifier, + ICurrentUserService currentUser, + ILogger logger) : IWholeBookPlotIntelligenceService +{ + private const string PromptVersion = "Whole-Book-Plot-Synthesis-V1"; + private const int MaxPromptCharacters = 900_000; + private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web) + { + WriteIndented = true + }; + private readonly StoryIntelligenceOptions settings = options.Value; + + public async Task GetAuthorPlotContextAsync(int? bookId = null) + { + var userId = currentUser.UserId; + if (!userId.HasValue) + { + return null; + } + + var book = bookId.HasValue ? await books.GetAsync(bookId.Value) : null; + if (book is null && bookId.HasValue) + { + return null; + } + + var descriptions = book is null + ? new List() + : (await repository.ListAuthorSeedsAsync(book.BookID)).Select(seed => seed.Description).ToList(); + + while (descriptions.Count < 3) + { + descriptions.Add(string.Empty); + } + + return new StoryIntelligenceAuthorPlotContextViewModel + { + ProjectID = book?.ProjectID ?? 0, + BookID = book?.BookID ?? 0, + BookTitle = book?.BookDisplayTitle ?? string.Empty, + Descriptions = descriptions + }; + } + + public Task SaveAuthorPlotContextAsync(int projectId, int bookId, IReadOnlyList descriptions) + => repository.SaveAuthorSeedsAsync(new AuthorPlotSeedSaveRequest + { + ProjectID = projectId, + BookID = bookId, + Descriptions = descriptions + }); + + public async Task QueueAfterCharacterResolutionAsync(int projectId, int bookId, int userId) + { + var pipeline = await pipelines.GetByBookForUserAsync(bookId, userId); + var latestRunId = pipeline?.LastRunID ?? (await pipelines.ListCommittedRunsByBookAsync(bookId, userId)) + .OrderByDescending(run => run.StoryIntelligenceRunID) + .FirstOrDefault() + ?.StoryIntelligenceRunID; + return await repository.QueueAsync(new WholeBookPlotSynthesisQueueRequest + { + ProjectID = projectId, + BookID = bookId, + UserID = userId, + SourceStoryIntelligenceRunID = latestRunId, + CharacterResolutionVersion = $"CharacterImport:{bookId}" + }); + } + + public Task RetryAsync(int bookId, int userId) + => repository.RetryAsync(bookId, userId); + + public Task GetCurrentAsync(int bookId) + => repository.GetCurrentAsync(bookId); + + public async Task ProcessNextAsync(CancellationToken cancellationToken) + { + var run = await repository.ClaimNextAsync(Math.Max(15, settings.ClaimLeaseMinutes)); + if (run is null) + { + return false; + } + + var stopwatch = Stopwatch.StartNew(); + try + { + var context = await repository.BuildContextAsync(run.ProjectID, run.BookID); + if (!context.SourceAudit.CanReconstruct) + { + throw new InvalidOperationException( + $"Scene manuscript source audit failed. Scenes={context.SourceAudit.SceneCount}, sources={context.SourceAudit.SourceRowCount}, missing={context.SourceAudit.MissingSourceCount}, duplicateSourceScenes={context.SourceAudit.DuplicateSourceSceneCount}."); + } + + var prompt = BuildPrompt(context); + if (prompt.Length > MaxPromptCharacters) + { + prompt = BuildPrompt(context, includeOptionalContext: false); + } + + if (prompt.Length > MaxPromptCharacters) + { + throw new InvalidOperationException("Whole-book Plot Intelligence prompt would exceed the configured safe context limit. No manuscript text was silently truncated."); + } + + var result = await client.ExecutePromptAsync( + prompt, + PromptVersion, + cancellationToken, + settings.EffectiveWholeBookPlotSynthesisModel, + responseContract: WholeBookPlotSynthesisStructuredOutputSchema.Contract); + var outputJson = ExtractOutputText(result.RawResponseText); + var parsed = WholeBookPlotSynthesisResult.FromJson(outputJson); + stopwatch.Stop(); + + await repository.CompleteAsync(new WholeBookPlotSynthesisCompletionRequest + { + WholeBookPlotSynthesisRunID = run.WholeBookPlotSynthesisRunID, + Model = result.Model, + OpenAIResponseID = ExtractResponseId(result.RawResponseText), + InputTokens = result.InputTokens, + OutputTokens = result.OutputTokens, + CachedInputTokens = ExtractCachedTokens(result.RawResponseText), + TotalTokens = (result.InputTokens ?? 0) + (result.OutputTokens ?? 0) > 0 + ? (result.InputTokens ?? 0) + (result.OutputTokens ?? 0) + : null, + DurationMs = stopwatch.ElapsedMilliseconds, + ResultJson = outputJson + }, parsed); + await notifier.PublishGlobalAsync(run.UserID); + logger.LogInformation( + "Whole-book Plot Intelligence completed. RunID={RunID} ProjectID={ProjectID} BookID={BookID} PlotLines={PlotLineCount} DurationMs={DurationMs}", + run.WholeBookPlotSynthesisRunID, + run.ProjectID, + run.BookID, + parsed.PlotLines.Count, + stopwatch.ElapsedMilliseconds); + return true; + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + stopwatch.Stop(); + await repository.FailAsync(run.WholeBookPlotSynthesisRunID, ex.Message, ex.ToString(), stopwatch.ElapsedMilliseconds); + await notifier.PublishGlobalAsync(run.UserID); + logger.LogError(ex, "Whole-book Plot Intelligence failed. RunID={RunID} BookID={BookID}", run.WholeBookPlotSynthesisRunID, run.BookID); + return true; + } + } + + public string BuildPromptForTest(WholeBookPlotSynthesisContext context) + => BuildPrompt(context); + + public StoryIntelligenceResponseContract BuildResponseContractForTest() + => WholeBookPlotSynthesisStructuredOutputSchema.Contract; + + private static string BuildPrompt(WholeBookPlotSynthesisContext context, bool includeOptionalContext = true) + { + var builder = new StringBuilder(); + builder.AppendLine("You are PlotDirector's Whole-Book Plot Intelligence engine."); + builder.AppendLine("Read the complete manuscript and identify sustained plot lines, subordinate threads, and every meaningful scene touch."); + builder.AppendLine("Return only data that conforms to the strict schema. Reference only SceneIDs, PlotLineIDs, PlotThreadIDs, and AuthorPlotSeedIDs supplied in this request."); + builder.AppendLine("A major Plot Line is a sustained narrative arc, mystery, conflict, goal, hidden truth, character struggle, investigation, relationship arc, or continuing question."); + builder.AppendLine("Do not promote isolated incidents, simple presence, locations, or recurring props without narrative development into major Plot Lines."); + builder.AppendLine("A Thread is a meaningful subordinate strand within a Plot Line. Do not fragment every scene development into a separate Thread."); + builder.AppendLine("Create an event for a scene when the scene meaningfully mentions, reinforces, advances, complicates, foreshadows, questions, misdirects, reveals, pays off, contradicts, or resolves the thread."); + builder.AppendLine("Do not attach a scene merely because associated characters are present."); + builder.AppendLine("Use Mentioned only as the fallback for genuine reinforcement when no more informative event type applies."); + builder.AppendLine(); + builder.AppendLine("[BOOK]"); + builder.AppendLine($"BookID: {context.BookID}"); + builder.AppendLine($"Title: {context.BookTitle}"); + builder.AppendLine(); + + if (includeOptionalContext) + { + builder.AppendLine("[CANONICAL CHARACTERS]"); + foreach (var character in context.Characters) + { + builder.AppendLine($"CharacterID: {character.CharacterID}; CanonicalName: {character.CanonicalName}; Aliases: {string.Join(", ", character.Aliases)}"); + } + builder.AppendLine(); + + builder.AppendLine("[EXISTING PROJECT PLOT LINES]"); + foreach (var plotLine in context.ExistingPlotLines) + { + builder.AppendLine($"PlotLineID: {plotLine.PlotLineID}; Name: {plotLine.Name}; Importance: {plotLine.Importance}; BookID: {plotLine.BookID?.ToString() ?? "series"}"); + if (!string.IsNullOrWhiteSpace(plotLine.Description)) + { + builder.AppendLine($"Description: {plotLine.Description}"); + } + + foreach (var thread in plotLine.Threads) + { + builder.AppendLine($" ThreadID: {thread.PlotThreadID}; Title: {thread.Title}; Status: {thread.Status}"); + } + } + builder.AppendLine(); + + builder.AppendLine("[AUTHOR PLOT SEEDS]"); + foreach (var seed in context.AuthorSeeds) + { + builder.AppendLine($"AuthorPlotSeedID: {seed.AuthorPlotSeedID}; Description: {seed.Description}"); + } + builder.AppendLine(); + } + + builder.AppendLine("[MANUSCRIPT]"); + foreach (var chapter in context.Chapters) + { + builder.AppendLine(); + builder.AppendLine($"[CHAPTER {chapter.ChapterNumber:0.##}]"); + builder.AppendLine($"ChapterID: {chapter.ChapterID}"); + builder.AppendLine($"Title: {chapter.ChapterTitle}"); + foreach (var scene in chapter.Scenes) + { + builder.AppendLine(); + builder.AppendLine("[SCENE]"); + builder.AppendLine($"SceneID: {scene.SceneID}"); + builder.AppendLine($"SceneNumber: {scene.SceneNumber:0.##}"); + builder.AppendLine($"Title: {scene.SceneTitle}"); + builder.AppendLine($"POV: {scene.PovCharacterName}"); + builder.AppendLine($"Setting: {scene.StructuralSetting}"); + builder.AppendLine(""); + builder.AppendLine(scene.SourceText); + builder.AppendLine(""); + } + } + + return builder.ToString(); + } + + private static string ExtractOutputText(string rawResponseText) + { + var response = JsonSerializer.Deserialize(rawResponseText, JsonOptions) + ?? throw new JsonException("OpenAI response envelope was empty."); + return response.Output? + .SelectMany(item => item.Content ?? []) + .FirstOrDefault(content => string.Equals(content.Type, "output_text", StringComparison.OrdinalIgnoreCase) + && !string.IsNullOrWhiteSpace(content.Text)) + ?.Text + ?.Trim() + ?? throw new JsonException("OpenAI response did not contain output_text content."); + } + + private static string? ExtractResponseId(string rawResponseText) + { + try + { + return JsonSerializer.Deserialize(rawResponseText, JsonOptions)?.Id; + } + catch (JsonException) + { + return null; + } + } + + private static int? ExtractCachedTokens(string rawResponseText) + { + try + { + return JsonSerializer.Deserialize(rawResponseText, JsonOptions)?.Usage?.InputTokenDetails?.CachedTokens; + } + catch (JsonException) + { + return null; + } + } + + private sealed class WholeBookOpenAIResponseEnvelope + { + public string? Id { get; init; } + public List? Output { get; init; } + public WholeBookOpenAIResponseUsage? Usage { get; init; } + } + + private sealed class WholeBookOpenAIResponseOutputItem + { + public List? Content { get; init; } + } + + private sealed class WholeBookOpenAIResponseContentItem + { + public string? Type { get; init; } + public string? Text { get; init; } + } + + private sealed class WholeBookOpenAIResponseUsage + { + [JsonPropertyName("input_tokens")] + public int? InputTokens { get; init; } + + [JsonPropertyName("output_tokens")] + public int? OutputTokens { get; init; } + + [JsonPropertyName("input_tokens_details")] + public WholeBookOpenAIInputTokenDetails? InputTokenDetails { get; init; } + } + + private sealed class WholeBookOpenAIInputTokenDetails + { + [JsonPropertyName("cached_tokens")] + public int? CachedTokens { get; init; } + } +} + +public static class WholeBookPlotSynthesisStructuredOutputSchema +{ + public static readonly StoryIntelligenceResponseContract Contract = new() + { + Name = "whole_book_plot_synthesis", + Strict = true, + Schema = new Dictionary + { + ["type"] = "object", + ["additionalProperties"] = false, + ["required"] = new[] { "metadata", "plotLines" }, + ["properties"] = new Dictionary + { + ["metadata"] = ObjectSchema(new Dictionary + { + ["bookStatus"] = StringSchema(), + ["summary"] = StringSchema() + }, "bookStatus", "summary"), + ["plotLines"] = ArraySchema(ObjectSchema(new Dictionary + { + ["key"] = StringSchema(), + ["proposedTitle"] = StringSchema(), + ["description"] = StringSchema(), + ["significance"] = StringSchema(), + ["bookStatus"] = StringSchema(), + ["existingPlotLineID"] = NullableIntegerSchema(), + ["existingPlotLineMatchConfidence"] = NullableNumberSchema(), + ["existingPlotLineMatchReason"] = NullableStringSchema(), + ["authorPlotSeedID"] = NullableIntegerSchema(), + ["seedMatchConfidence"] = NullableNumberSchema(), + ["seedMatchReason"] = NullableStringSchema(), + ["confidence"] = NullableNumberSchema(), + ["threads"] = ArraySchema(ObjectSchema(new Dictionary + { + ["key"] = StringSchema(), + ["proposedTitle"] = StringSchema(), + ["description"] = StringSchema(), + ["status"] = StringSchema(), + ["existingPlotThreadID"] = NullableIntegerSchema(), + ["existingThreadMatchConfidence"] = NullableNumberSchema(), + ["matchReason"] = NullableStringSchema(), + ["confidence"] = NullableNumberSchema(), + ["events"] = ArraySchema(ObjectSchema(new Dictionary + { + ["key"] = StringSchema(), + ["sceneID"] = IntegerSchema(), + ["eventType"] = EnumSchema("Introduced", "Mentioned", "Developed", "Complicated", "Clue Planted", "Question Raised", "Question Answered", "False Lead", "Revealed", "Payoff", "Resolved", "Contradicted", "Retconned", "Note"), + ["title"] = StringSchema(), + ["description"] = StringSchema(), + ["evidence"] = StringSchema(), + ["confidence"] = NullableNumberSchema() + }, "key", "sceneID", "eventType", "title", "description", "evidence", "confidence")) + }, "key", "proposedTitle", "description", "status", "existingPlotThreadID", "existingThreadMatchConfidence", "matchReason", "confidence", "events")) + }, "key", "proposedTitle", "description", "significance", "bookStatus", "existingPlotLineID", "existingPlotLineMatchConfidence", "existingPlotLineMatchReason", "authorPlotSeedID", "seedMatchConfidence", "seedMatchReason", "confidence", "threads")) + } + } + }; + + private static Dictionary ObjectSchema(Dictionary properties, params string[] required) + => new() + { + ["type"] = "object", + ["additionalProperties"] = false, + ["required"] = required, + ["properties"] = properties + }; + + private static Dictionary ArraySchema(object items) + => new() + { + ["type"] = "array", + ["items"] = items + }; + + private static Dictionary StringSchema() => new() { ["type"] = "string" }; + private static Dictionary IntegerSchema() => new() { ["type"] = "integer" }; + private static Dictionary NullableIntegerSchema() => new() { ["type"] = new[] { "integer", "null" } }; + private static Dictionary NullableNumberSchema() => new() { ["type"] = new[] { "number", "null" }, ["minimum"] = 0, ["maximum"] = 1 }; + private static Dictionary NullableStringSchema() => new() { ["type"] = new[] { "string", "null" } }; + private static Dictionary EnumSchema(params string[] values) => new() { ["type"] = "string", ["enum"] = values }; +} diff --git a/PlotLine/Sql/174_Phase27_WholeBookPlotIntelligence.sql b/PlotLine/Sql/174_Phase27_WholeBookPlotIntelligence.sql new file mode 100644 index 0000000..b444666 --- /dev/null +++ b/PlotLine/Sql/174_Phase27_WholeBookPlotIntelligence.sql @@ -0,0 +1,817 @@ +SET ANSI_NULLS ON; +GO +SET QUOTED_IDENTIFIER ON; +GO + +IF OBJECT_ID(N'dbo.AuthorPlotSeeds', N'U') IS NULL +BEGIN + CREATE TABLE dbo.AuthorPlotSeeds + ( + AuthorPlotSeedID int IDENTITY(1,1) NOT NULL CONSTRAINT PK_AuthorPlotSeeds PRIMARY KEY, + ProjectID int NOT NULL, + BookID int NOT NULL, + Description nvarchar(max) NOT NULL, + Status nvarchar(30) NOT NULL CONSTRAINT DF_AuthorPlotSeeds_Status DEFAULT N'Active', + MatchedPlotLineID int NULL, + Source nvarchar(40) NOT NULL CONSTRAINT DF_AuthorPlotSeeds_Source DEFAULT N'Author', + CreatedUtc datetime2 NOT NULL CONSTRAINT DF_AuthorPlotSeeds_CreatedUtc DEFAULT SYSUTCDATETIME(), + UpdatedUtc datetime2 NOT NULL CONSTRAINT DF_AuthorPlotSeeds_UpdatedUtc DEFAULT SYSUTCDATETIME(), + CONSTRAINT FK_AuthorPlotSeeds_Projects FOREIGN KEY (ProjectID) REFERENCES dbo.Projects(ProjectID), + CONSTRAINT FK_AuthorPlotSeeds_Books FOREIGN KEY (BookID) REFERENCES dbo.Books(BookID) ON DELETE CASCADE, + CONSTRAINT FK_AuthorPlotSeeds_MatchedPlotLines FOREIGN KEY (MatchedPlotLineID) REFERENCES dbo.PlotLines(PlotLineID) ON DELETE SET NULL + ); +END; +GO + +IF EXISTS (SELECT 1 FROM sys.foreign_keys WHERE name = N'FK_AuthorPlotSeeds_Books' AND parent_object_id = OBJECT_ID(N'dbo.AuthorPlotSeeds')) +BEGIN + ALTER TABLE dbo.AuthorPlotSeeds DROP CONSTRAINT FK_AuthorPlotSeeds_Books; + ALTER TABLE dbo.AuthorPlotSeeds WITH CHECK + ADD CONSTRAINT FK_AuthorPlotSeeds_Books FOREIGN KEY (BookID) REFERENCES dbo.Books(BookID) ON DELETE CASCADE; +END; +GO + +IF EXISTS (SELECT 1 FROM sys.foreign_keys WHERE name = N'FK_AuthorPlotSeeds_MatchedPlotLines' AND parent_object_id = OBJECT_ID(N'dbo.AuthorPlotSeeds')) +BEGIN + ALTER TABLE dbo.AuthorPlotSeeds DROP CONSTRAINT FK_AuthorPlotSeeds_MatchedPlotLines; + ALTER TABLE dbo.AuthorPlotSeeds WITH CHECK + ADD CONSTRAINT FK_AuthorPlotSeeds_MatchedPlotLines FOREIGN KEY (MatchedPlotLineID) REFERENCES dbo.PlotLines(PlotLineID) ON DELETE SET NULL; +END; +GO + +IF NOT EXISTS (SELECT 1 FROM sys.indexes WHERE name = N'IX_AuthorPlotSeeds_Book_Status' AND object_id = OBJECT_ID(N'dbo.AuthorPlotSeeds')) + CREATE INDEX IX_AuthorPlotSeeds_Book_Status ON dbo.AuthorPlotSeeds(BookID, Status); +GO + +IF OBJECT_ID(N'dbo.WholeBookPlotSynthesisRuns', N'U') IS NULL +BEGIN + CREATE TABLE dbo.WholeBookPlotSynthesisRuns + ( + WholeBookPlotSynthesisRunID int IDENTITY(1,1) NOT NULL CONSTRAINT PK_WholeBookPlotSynthesisRuns PRIMARY KEY, + ProjectID int NOT NULL, + BookID int NOT NULL, + UserID int NOT NULL, + SourceStoryIntelligenceRunID int NULL, + SourceStoryIntelligenceRunKey AS ISNULL(SourceStoryIntelligenceRunID, 0) PERSISTED, + CharacterResolutionVersion nvarchar(120) NOT NULL, + Status nvarchar(30) NOT NULL CONSTRAINT DF_WholeBookPlotSynthesisRuns_Status DEFAULT N'Queued', + QueuedUtc datetime2 NOT NULL CONSTRAINT DF_WholeBookPlotSynthesisRuns_QueuedUtc DEFAULT SYSUTCDATETIME(), + StartedUtc datetime2 NULL, + CompletedUtc datetime2 NULL, + FailedUtc datetime2 NULL, + AttemptCount int NOT NULL CONSTRAINT DF_WholeBookPlotSynthesisRuns_AttemptCount DEFAULT 0, + Model nvarchar(100) NULL, + OpenAIResponseID nvarchar(120) NULL, + InputTokens int NULL, + OutputTokens int NULL, + CachedInputTokens int NULL, + TotalTokens int NULL, + DurationMs bigint NULL, + ErrorMessage nvarchar(max) NULL, + ErrorDetail nvarchar(max) NULL, + ResultJson nvarchar(max) NULL, + IsCurrent bit NOT NULL CONSTRAINT DF_WholeBookPlotSynthesisRuns_IsCurrent DEFAULT 1, + UpdatedUtc datetime2 NOT NULL CONSTRAINT DF_WholeBookPlotSynthesisRuns_UpdatedUtc DEFAULT SYSUTCDATETIME(), + CONSTRAINT FK_WholeBookPlotSynthesisRuns_Projects FOREIGN KEY (ProjectID) REFERENCES dbo.Projects(ProjectID), + CONSTRAINT FK_WholeBookPlotSynthesisRuns_Books FOREIGN KEY (BookID) REFERENCES dbo.Books(BookID) ON DELETE CASCADE, + CONSTRAINT FK_WholeBookPlotSynthesisRuns_AppUser FOREIGN KEY (UserID) REFERENCES dbo.AppUser(UserID), + CONSTRAINT FK_WholeBookPlotSynthesisRuns_SourceRun FOREIGN KEY (SourceStoryIntelligenceRunID) REFERENCES dbo.StoryIntelligenceRuns(StoryIntelligenceRunID) ON DELETE SET NULL + ); +END; +GO + +IF COL_LENGTH(N'dbo.WholeBookPlotSynthesisRuns', N'SourceStoryIntelligenceRunKey') IS NULL + ALTER TABLE dbo.WholeBookPlotSynthesisRuns ADD SourceStoryIntelligenceRunKey AS ISNULL(SourceStoryIntelligenceRunID, 0) PERSISTED; +GO + +IF NOT EXISTS (SELECT 1 FROM sys.indexes WHERE name = N'IX_WholeBookPlotSynthesisRuns_Queue' AND object_id = OBJECT_ID(N'dbo.WholeBookPlotSynthesisRuns')) + CREATE INDEX IX_WholeBookPlotSynthesisRuns_Queue ON dbo.WholeBookPlotSynthesisRuns(Status, QueuedUtc) INCLUDE (BookID, UserID); +GO + +IF NOT EXISTS (SELECT 1 FROM sys.indexes WHERE name = N'UX_WholeBookPlotSynthesisRuns_CurrentVersion' AND object_id = OBJECT_ID(N'dbo.WholeBookPlotSynthesisRuns')) + CREATE UNIQUE INDEX UX_WholeBookPlotSynthesisRuns_CurrentVersion + ON dbo.WholeBookPlotSynthesisRuns(BookID, SourceStoryIntelligenceRunKey, CharacterResolutionVersion) + WHERE IsCurrent = 1 AND Status IN (N'Queued', N'Running', N'Completed'); +GO + +IF COL_LENGTH(N'dbo.StoryIntelligenceNarrativeCandidates', N'WholeBookPlotSynthesisRunID') IS NULL + ALTER TABLE dbo.StoryIntelligenceNarrativeCandidates ADD WholeBookPlotSynthesisRunID int NULL; +GO +IF COL_LENGTH(N'dbo.StoryIntelligenceNarrativeCandidates', N'AuthorPlotSeedID') IS NULL + ALTER TABLE dbo.StoryIntelligenceNarrativeCandidates ADD AuthorPlotSeedID int NULL; +GO +IF COL_LENGTH(N'dbo.StoryIntelligenceNarrativeCandidates', N'ExistingPlotLineMatchConfidence') IS NULL + ALTER TABLE dbo.StoryIntelligenceNarrativeCandidates ADD ExistingPlotLineMatchConfidence decimal(5,4) NULL; +GO +IF COL_LENGTH(N'dbo.StoryIntelligenceNarrativeCandidates', N'ExistingPlotLineMatchReason') IS NULL + ALTER TABLE dbo.StoryIntelligenceNarrativeCandidates ADD ExistingPlotLineMatchReason nvarchar(max) NULL; +GO +IF COL_LENGTH(N'dbo.StoryIntelligenceNarrativeCandidates', N'SeedMatchConfidence') IS NULL + ALTER TABLE dbo.StoryIntelligenceNarrativeCandidates ADD SeedMatchConfidence decimal(5,4) NULL; +GO +IF COL_LENGTH(N'dbo.StoryIntelligenceNarrativeCandidates', N'SeedMatchReason') IS NULL + ALTER TABLE dbo.StoryIntelligenceNarrativeCandidates ADD SeedMatchReason nvarchar(max) NULL; +GO +IF COL_LENGTH(N'dbo.StoryIntelligenceNarrativeCandidates', N'Source') IS NULL + ALTER TABLE dbo.StoryIntelligenceNarrativeCandidates ADD Source nvarchar(60) NULL; +GO + +IF OBJECT_ID(N'dbo.StoryIntelligenceNarrativeCandidateEvents', N'U') IS NULL +BEGIN + CREATE TABLE dbo.StoryIntelligenceNarrativeCandidateEvents + ( + NarrativeCandidateEventID int IDENTITY(1,1) NOT NULL CONSTRAINT PK_StoryIntelligenceNarrativeCandidateEvents PRIMARY KEY, + NarrativeCandidateID int NOT NULL, + WholeBookPlotSynthesisRunID int NULL, + EventKey nvarchar(260) NOT NULL, + SceneID int NOT NULL, + EventType nvarchar(100) NOT NULL, + EventTitle nvarchar(200) NOT NULL, + EventDescription nvarchar(max) NULL, + Evidence nvarchar(max) NULL, + Confidence decimal(5,4) NULL, + SortOrder int NOT NULL CONSTRAINT DF_StoryIntelligenceNarrativeCandidateEvents_SortOrder DEFAULT 0, + IsIgnored bit NOT NULL CONSTRAINT DF_StoryIntelligenceNarrativeCandidateEvents_IsIgnored DEFAULT 0, + AcceptedThreadEventID int NULL, + CreatedUtc datetime2 NOT NULL CONSTRAINT DF_StoryIntelligenceNarrativeCandidateEvents_CreatedUtc DEFAULT SYSUTCDATETIME(), + UpdatedUtc datetime2 NOT NULL CONSTRAINT DF_StoryIntelligenceNarrativeCandidateEvents_UpdatedUtc DEFAULT SYSUTCDATETIME(), + CONSTRAINT FK_StoryIntelligenceNarrativeCandidateEvents_Candidates FOREIGN KEY (NarrativeCandidateID) REFERENCES dbo.StoryIntelligenceNarrativeCandidates(NarrativeCandidateID) ON DELETE CASCADE, + CONSTRAINT FK_StoryIntelligenceNarrativeCandidateEvents_SynthesisRun FOREIGN KEY (WholeBookPlotSynthesisRunID) REFERENCES dbo.WholeBookPlotSynthesisRuns(WholeBookPlotSynthesisRunID), + CONSTRAINT FK_StoryIntelligenceNarrativeCandidateEvents_Scenes FOREIGN KEY (SceneID) REFERENCES dbo.Scenes(SceneID), + CONSTRAINT FK_StoryIntelligenceNarrativeCandidateEvents_AcceptedThreadEvents FOREIGN KEY (AcceptedThreadEventID) REFERENCES dbo.ThreadEvents(ThreadEventID) + ); +END; +GO + +IF NOT EXISTS (SELECT 1 FROM sys.indexes WHERE name = N'UX_StoryIntelligenceNarrativeCandidateEvents_Key' AND object_id = OBJECT_ID(N'dbo.StoryIntelligenceNarrativeCandidateEvents')) + CREATE UNIQUE INDEX UX_StoryIntelligenceNarrativeCandidateEvents_Key ON dbo.StoryIntelligenceNarrativeCandidateEvents(NarrativeCandidateID, EventKey); +GO + +IF COL_LENGTH(N'dbo.ThreadEvents', N'Source') IS NULL + ALTER TABLE dbo.ThreadEvents ADD Source nvarchar(60) NOT NULL CONSTRAINT DF_ThreadEvents_Source DEFAULT N'Author'; +GO +IF COL_LENGTH(N'dbo.ThreadEvents', N'SourceReferenceID') IS NULL + ALTER TABLE dbo.ThreadEvents ADD SourceReferenceID int NULL; +GO +IF COL_LENGTH(N'dbo.ThreadEvents', N'SourceKey') IS NULL + ALTER TABLE dbo.ThreadEvents ADD SourceKey nvarchar(260) NULL; +GO +IF COL_LENGTH(N'dbo.ThreadEvents', N'Confidence') IS NULL + ALTER TABLE dbo.ThreadEvents ADD Confidence decimal(5,4) NULL; +GO + +IF NOT EXISTS (SELECT 1 FROM sys.indexes WHERE name = N'UX_ThreadEvents_SourceKey' AND object_id = OBJECT_ID(N'dbo.ThreadEvents')) + CREATE UNIQUE INDEX UX_ThreadEvents_SourceKey ON dbo.ThreadEvents(Source, SourceKey) WHERE SourceKey IS NOT NULL; +GO + +CREATE OR ALTER PROCEDURE dbo.AuthorPlotSeed_SaveForBook + @ProjectID int, + @BookID int, + @SeedsJson nvarchar(max) +AS +BEGIN + SET NOCOUNT ON; + SET XACT_ABORT ON; + + DECLARE @Seeds TABLE (Description nvarchar(max) NOT NULL, SortOrder int IDENTITY(1,1)); + + INSERT @Seeds (Description) + SELECT DISTINCT LEFT(LTRIM(RTRIM(JSON_VALUE(value, '$.description'))), 4000) + FROM OPENJSON(@SeedsJson) + WHERE NULLIF(LTRIM(RTRIM(JSON_VALUE(value, '$.description'))), N'') IS NOT NULL; + + UPDATE dbo.AuthorPlotSeeds + SET Status = N'Archived', + UpdatedUtc = SYSUTCDATETIME() + WHERE ProjectID = @ProjectID + AND BookID = @BookID + AND Source = N'Author' + AND Status = N'Active' + AND NOT EXISTS + ( + SELECT 1 FROM @Seeds seed WHERE seed.Description = AuthorPlotSeeds.Description + ); + + INSERT dbo.AuthorPlotSeeds (ProjectID, BookID, Description, Status, Source) + SELECT @ProjectID, @BookID, seed.Description, N'Active', N'Author' + FROM @Seeds seed + WHERE NOT EXISTS + ( + SELECT 1 + FROM dbo.AuthorPlotSeeds existing + WHERE existing.ProjectID = @ProjectID + AND existing.BookID = @BookID + AND existing.Source = N'Author' + AND existing.Description = seed.Description + AND existing.Status = N'Active' + ); +END; +GO + +CREATE OR ALTER PROCEDURE dbo.AuthorPlotSeed_ListByBook + @BookID int +AS +BEGIN + SET NOCOUNT ON; + SELECT AuthorPlotSeedID, ProjectID, BookID, Description, Status, MatchedPlotLineID, Source, CreatedUtc, UpdatedUtc + FROM dbo.AuthorPlotSeeds + WHERE BookID = @BookID AND Status = N'Active' + ORDER BY AuthorPlotSeedID; +END; +GO + +CREATE OR ALTER PROCEDURE dbo.WholeBookPlotSynthesisRun_Queue + @ProjectID int, + @BookID int, + @UserID int, + @SourceStoryIntelligenceRunID int = NULL, + @CharacterResolutionVersion nvarchar(120) +AS +BEGIN + SET NOCOUNT ON; + SET XACT_ABORT ON; + + DECLARE @ExistingID int; + SELECT TOP (1) @ExistingID = WholeBookPlotSynthesisRunID + FROM dbo.WholeBookPlotSynthesisRuns + WHERE BookID = @BookID + AND ISNULL(SourceStoryIntelligenceRunID, 0) = ISNULL(@SourceStoryIntelligenceRunID, 0) + AND CharacterResolutionVersion = @CharacterResolutionVersion + AND IsCurrent = 1 + AND Status IN (N'Queued', N'Running', N'Completed') + ORDER BY WholeBookPlotSynthesisRunID DESC; + + IF @ExistingID IS NULL + BEGIN + INSERT dbo.WholeBookPlotSynthesisRuns (ProjectID, BookID, UserID, SourceStoryIntelligenceRunID, CharacterResolutionVersion, Status) + VALUES (@ProjectID, @BookID, @UserID, @SourceStoryIntelligenceRunID, @CharacterResolutionVersion, N'Queued'); + SET @ExistingID = CAST(SCOPE_IDENTITY() AS int); + END; + + SELECT * FROM dbo.WholeBookPlotSynthesisRuns WHERE WholeBookPlotSynthesisRunID = @ExistingID; +END; +GO + +CREATE OR ALTER PROCEDURE dbo.WholeBookPlotSynthesisRun_Retry + @BookID int, + @UserID int +AS +BEGIN + SET NOCOUNT ON; + SET XACT_ABORT ON; + + DECLARE @PreviousID int, @ProjectID int, @SourceRunID int, @Version nvarchar(120); + SELECT TOP (1) + @PreviousID = WholeBookPlotSynthesisRunID, + @ProjectID = ProjectID, + @SourceRunID = SourceStoryIntelligenceRunID, + @Version = CharacterResolutionVersion + FROM dbo.WholeBookPlotSynthesisRuns + WHERE BookID = @BookID AND UserID = @UserID AND IsCurrent = 1 AND Status = N'Failed' + ORDER BY WholeBookPlotSynthesisRunID DESC; + + IF @PreviousID IS NULL + BEGIN + SELECT TOP (0) * FROM dbo.WholeBookPlotSynthesisRuns; + RETURN; + END; + + UPDATE dbo.WholeBookPlotSynthesisRuns + SET IsCurrent = 0, + Status = N'Superseded', + UpdatedUtc = SYSUTCDATETIME() + WHERE WholeBookPlotSynthesisRunID = @PreviousID; + + INSERT dbo.WholeBookPlotSynthesisRuns (ProjectID, BookID, UserID, SourceStoryIntelligenceRunID, CharacterResolutionVersion, Status) + VALUES (@ProjectID, @BookID, @UserID, @SourceRunID, @Version, N'Queued'); + + SELECT * FROM dbo.WholeBookPlotSynthesisRuns WHERE WholeBookPlotSynthesisRunID = CAST(SCOPE_IDENTITY() AS int); +END; +GO + +CREATE OR ALTER PROCEDURE dbo.WholeBookPlotSynthesisRun_ClaimNext + @LeaseMinutes int = 90 +AS +BEGIN + SET NOCOUNT ON; + SET XACT_ABORT ON; + + DECLARE @RunID int; + + SELECT TOP (1) @RunID = WholeBookPlotSynthesisRunID + FROM dbo.WholeBookPlotSynthesisRuns WITH (UPDLOCK, READPAST) + WHERE IsCurrent = 1 + AND ( + Status = N'Queued' + OR (Status = N'Running' AND StartedUtc < DATEADD(minute, -ABS(@LeaseMinutes), SYSUTCDATETIME())) + ) + ORDER BY QueuedUtc, WholeBookPlotSynthesisRunID; + + IF @RunID IS NULL + BEGIN + SELECT TOP (0) * FROM dbo.WholeBookPlotSynthesisRuns; + RETURN; + END; + + UPDATE dbo.WholeBookPlotSynthesisRuns + SET Status = N'Running', + StartedUtc = SYSUTCDATETIME(), + AttemptCount = AttemptCount + 1, + UpdatedUtc = SYSUTCDATETIME() + WHERE WholeBookPlotSynthesisRunID = @RunID; + + SELECT * FROM dbo.WholeBookPlotSynthesisRuns WHERE WholeBookPlotSynthesisRunID = @RunID; +END; +GO + +CREATE OR ALTER PROCEDURE dbo.WholeBookPlotSynthesisRun_GetCurrentByBook + @BookID int +AS +BEGIN + SET NOCOUNT ON; + SELECT TOP (1) * + FROM dbo.WholeBookPlotSynthesisRuns + WHERE BookID = @BookID AND IsCurrent = 1 + ORDER BY WholeBookPlotSynthesisRunID DESC; +END; +GO + +CREATE OR ALTER PROCEDURE dbo.WholeBookPlotSynthesisRun_Complete + @WholeBookPlotSynthesisRunID int, + @Model nvarchar(100), + @OpenAIResponseID nvarchar(120) = NULL, + @InputTokens int = NULL, + @OutputTokens int = NULL, + @CachedInputTokens int = NULL, + @TotalTokens int = NULL, + @DurationMs bigint = NULL, + @ResultJson nvarchar(max) +AS +BEGIN + SET NOCOUNT ON; + UPDATE dbo.WholeBookPlotSynthesisRuns + SET Status = N'Completed', + CompletedUtc = SYSUTCDATETIME(), + Model = @Model, + OpenAIResponseID = @OpenAIResponseID, + InputTokens = @InputTokens, + OutputTokens = @OutputTokens, + CachedInputTokens = @CachedInputTokens, + TotalTokens = @TotalTokens, + DurationMs = @DurationMs, + ResultJson = @ResultJson, + ErrorMessage = NULL, + ErrorDetail = NULL, + UpdatedUtc = SYSUTCDATETIME() + WHERE WholeBookPlotSynthesisRunID = @WholeBookPlotSynthesisRunID; +END; +GO + +CREATE OR ALTER PROCEDURE dbo.WholeBookPlotSynthesisRun_Fail + @WholeBookPlotSynthesisRunID int, + @ErrorMessage nvarchar(max), + @ErrorDetail nvarchar(max) = NULL, + @DurationMs bigint = NULL +AS +BEGIN + SET NOCOUNT ON; + UPDATE dbo.WholeBookPlotSynthesisRuns + SET Status = N'Failed', + FailedUtc = SYSUTCDATETIME(), + ErrorMessage = @ErrorMessage, + ErrorDetail = @ErrorDetail, + DurationMs = @DurationMs, + UpdatedUtc = SYSUTCDATETIME() + WHERE WholeBookPlotSynthesisRunID = @WholeBookPlotSynthesisRunID; +END; +GO + +CREATE OR ALTER PROCEDURE dbo.WholeBookPlotSynthesis_ContextGet + @ProjectID int, + @BookID int +AS +BEGIN + SET NOCOUNT ON; + + SELECT BookTitle = b.BookTitle + FROM dbo.Books b + WHERE b.BookID = @BookID AND b.ProjectID = @ProjectID AND b.IsArchived = 0; + + SELECT + c.ChapterID, + c.ChapterNumber, + c.ChapterTitle, + s.SceneID, + s.SceneNumber, + s.SceneTitle, + s.StructuralSummary, + s.StructuralSetting, + pov.CharacterName AS PovCharacterName, + sms.SourceText + FROM dbo.Chapters c + INNER JOIN dbo.Scenes s ON s.ChapterID = c.ChapterID AND s.IsArchived = 0 + INNER JOIN dbo.SceneManuscriptSources sms ON sms.SceneID = s.SceneID + LEFT JOIN dbo.Characters pov ON pov.CharacterID = COALESCE(s.POVCharacterID, c.POVCharacterID) + WHERE c.BookID = @BookID AND c.IsArchived = 0 + ORDER BY c.SortOrder, c.ChapterNumber, s.SortOrder, s.SceneNumber, s.SceneID; + + SELECT CharacterID, CanonicalName = CharacterName + FROM dbo.Characters + WHERE ProjectID = @ProjectID AND IsArchived = 0 + ORDER BY CharacterName; + + SELECT alias.CharacterID, alias.Alias + FROM dbo.CharacterAliases alias + INNER JOIN dbo.Characters character ON character.CharacterID = alias.CharacterID + WHERE character.ProjectID = @ProjectID + AND character.IsArchived = 0 + ORDER BY alias.CharacterID, alias.Alias; + + SELECT + pl.PlotLineID, + Name = pl.PlotLineName, + pl.Description, + Importance = pi.ImportanceName, + pl.BookID + FROM dbo.PlotLines pl + LEFT JOIN dbo.PlotImportance pi ON pi.PlotImportanceID = pl.PlotImportanceID + WHERE pl.ProjectID = @ProjectID AND pl.IsArchived = 0 + ORDER BY pl.SortOrder, pl.PlotLineName; + + SELECT + pt.PlotThreadID, + pt.PlotLineID, + Title = pt.ThreadTitle, + pt.Summary, + Status = ts.StatusName + FROM dbo.PlotThreads pt + INNER JOIN dbo.PlotLines pl ON pl.PlotLineID = pt.PlotLineID + LEFT JOIN dbo.ThreadStatuses ts ON ts.ThreadStatusID = pt.ThreadStatusID + WHERE pl.ProjectID = @ProjectID + AND pl.IsArchived = 0 + AND pt.IsArchived = 0 + ORDER BY pt.PlotLineID, pt.Importance DESC, pt.ThreadTitle; + + EXEC dbo.AuthorPlotSeed_ListByBook @BookID; + + SELECT + SceneCount = ( + SELECT COUNT(1) + FROM dbo.Scenes s + INNER JOIN dbo.Chapters c ON c.ChapterID = s.ChapterID + WHERE c.BookID = @BookID AND c.IsArchived = 0 AND s.IsArchived = 0 + ), + SourceRowCount = ( + SELECT COUNT(1) + FROM dbo.SceneManuscriptSources sms + INNER JOIN dbo.Scenes s ON s.SceneID = sms.SceneID + INNER JOIN dbo.Chapters c ON c.ChapterID = s.ChapterID + WHERE c.BookID = @BookID AND c.IsArchived = 0 AND s.IsArchived = 0 + ), + MissingSourceCount = ( + SELECT COUNT(1) + FROM dbo.Scenes s + INNER JOIN dbo.Chapters c ON c.ChapterID = s.ChapterID + LEFT JOIN dbo.SceneManuscriptSources sms ON sms.SceneID = s.SceneID + WHERE c.BookID = @BookID AND c.IsArchived = 0 AND s.IsArchived = 0 AND sms.SceneID IS NULL + ), + DuplicateSourceSceneCount = 0; +END; +GO + +CREATE OR ALTER PROCEDURE dbo.WholeBookPlotSynthesisCandidate_Save + @WholeBookPlotSynthesisRunID int, + @CandidateType nvarchar(30), + @CandidateKey nvarchar(220), + @ProposedTitle nvarchar(200), + @ProposedDescription nvarchar(max) = NULL, + @Category nvarchar(100) = NULL, + @InferredState nvarchar(100) = NULL, + @ChangeType nvarchar(40) = NULL, + @Significance nvarchar(20) = NULL, + @ExistingPlotLineID int = NULL, + @ExistingPlotThreadID int = NULL, + @RelatedPlotLineID int = NULL, + @ParentCandidateKey nvarchar(220) = NULL, + @ParentNarrativeCandidateID int = NULL, + @AuthorPlotSeedID int = NULL, + @ExistingPlotLineMatchConfidence decimal(5,4) = NULL, + @ExistingPlotLineMatchReason nvarchar(max) = NULL, + @SeedMatchConfidence decimal(5,4) = NULL, + @SeedMatchReason nvarchar(max) = NULL, + @Confidence decimal(5,4) = NULL +AS +BEGIN + SET NOCOUNT ON; + + DECLARE @ProjectID int, @BookID int, @NarrativeCandidateID int; + SELECT @ProjectID = ProjectID, @BookID = BookID + FROM dbo.WholeBookPlotSynthesisRuns + WHERE WholeBookPlotSynthesisRunID = @WholeBookPlotSynthesisRunID; + + SELECT @NarrativeCandidateID = NarrativeCandidateID + FROM dbo.StoryIntelligenceNarrativeCandidates + WHERE BookID = @BookID AND CandidateType = @CandidateType AND CandidateKey = @CandidateKey; + + IF @NarrativeCandidateID IS NULL + BEGIN + INSERT dbo.StoryIntelligenceNarrativeCandidates + ( + ProjectID, BookID, CandidateType, CandidateKey, ProposedTitle, ProposedDescription, Category, + InferredState, ChangeType, Significance, ExistingPlotLineID, ExistingPlotThreadID, RelatedPlotLineID, + ParentCandidateKey, ParentNarrativeCandidateID, WholeBookPlotSynthesisRunID, AuthorPlotSeedID, + ExistingPlotLineMatchConfidence, ExistingPlotLineMatchReason, SeedMatchConfidence, SeedMatchReason, + Confidence, Source + ) + VALUES + ( + @ProjectID, @BookID, @CandidateType, @CandidateKey, @ProposedTitle, @ProposedDescription, @Category, + @InferredState, @ChangeType, @Significance, @ExistingPlotLineID, @ExistingPlotThreadID, @RelatedPlotLineID, + @ParentCandidateKey, @ParentNarrativeCandidateID, @WholeBookPlotSynthesisRunID, @AuthorPlotSeedID, + @ExistingPlotLineMatchConfidence, @ExistingPlotLineMatchReason, @SeedMatchConfidence, @SeedMatchReason, + @Confidence, N'WholeBookPlotSynthesis' + ); + SET @NarrativeCandidateID = CAST(SCOPE_IDENTITY() AS int); + END + ELSE + BEGIN + UPDATE dbo.StoryIntelligenceNarrativeCandidates + SET ProposedTitle = @ProposedTitle, + ProposedDescription = @ProposedDescription, + Category = @Category, + InferredState = @InferredState, + ChangeType = @ChangeType, + Significance = @Significance, + ExistingPlotLineID = @ExistingPlotLineID, + ExistingPlotThreadID = @ExistingPlotThreadID, + RelatedPlotLineID = @RelatedPlotLineID, + ParentCandidateKey = @ParentCandidateKey, + ParentNarrativeCandidateID = @ParentNarrativeCandidateID, + WholeBookPlotSynthesisRunID = @WholeBookPlotSynthesisRunID, + AuthorPlotSeedID = @AuthorPlotSeedID, + ExistingPlotLineMatchConfidence = @ExistingPlotLineMatchConfidence, + ExistingPlotLineMatchReason = @ExistingPlotLineMatchReason, + SeedMatchConfidence = @SeedMatchConfidence, + SeedMatchReason = @SeedMatchReason, + Confidence = @Confidence, + Source = N'WholeBookPlotSynthesis', + UpdatedUtc = SYSUTCDATETIME() + WHERE NarrativeCandidateID = @NarrativeCandidateID + AND Status = N'Pending'; + END; + + SELECT @NarrativeCandidateID; +END; +GO + +CREATE OR ALTER PROCEDURE dbo.WholeBookPlotSynthesisCandidateEvent_Save + @NarrativeCandidateID int, + @SynthesisRunID int, + @EventKey nvarchar(260), + @SceneID int, + @EventType nvarchar(100), + @EventTitle nvarchar(200), + @EventDescription nvarchar(max) = NULL, + @Evidence nvarchar(max) = NULL, + @Confidence decimal(5,4) = NULL, + @SortOrder int = 0 +AS +BEGIN + SET NOCOUNT ON; + + MERGE dbo.StoryIntelligenceNarrativeCandidateEvents AS target + USING (SELECT @NarrativeCandidateID AS NarrativeCandidateID, @EventKey AS EventKey) AS source + ON target.NarrativeCandidateID = source.NarrativeCandidateID AND target.EventKey = source.EventKey + WHEN MATCHED THEN + UPDATE SET WholeBookPlotSynthesisRunID = @SynthesisRunID, + SceneID = @SceneID, + EventType = @EventType, + EventTitle = @EventTitle, + EventDescription = @EventDescription, + Evidence = @Evidence, + Confidence = @Confidence, + SortOrder = @SortOrder, + UpdatedUtc = SYSUTCDATETIME() + WHEN NOT MATCHED THEN + INSERT (NarrativeCandidateID, WholeBookPlotSynthesisRunID, EventKey, SceneID, EventType, EventTitle, EventDescription, Evidence, Confidence, SortOrder) + VALUES (@NarrativeCandidateID, @SynthesisRunID, @EventKey, @SceneID, @EventType, @EventTitle, @EventDescription, @Evidence, @Confidence, @SortOrder); +END; +GO + +CREATE OR ALTER PROCEDURE dbo.StoryIntelligenceNarrativeCandidate_Save + @ProjectID int, + @BookID int, + @CandidateType nvarchar(30), + @CandidateKey nvarchar(220), + @ProposedTitle nvarchar(200), + @ProposedDescription nvarchar(max) = NULL, + @Category nvarchar(100) = NULL, + @InferredState nvarchar(100) = NULL, + @ChangeType nvarchar(40) = NULL, + @Significance nvarchar(20) = NULL, + @ExistingPlotLineID int = NULL, + @ExistingPlotThreadID int = NULL, + @WholeBookPlotSynthesisRunID int = NULL, + @AuthorPlotSeedID int = NULL, + @ExistingPlotLineMatchConfidence decimal(5,4) = NULL, + @ExistingPlotLineMatchReason nvarchar(max) = NULL, + @SeedMatchConfidence decimal(5,4) = NULL, + @SeedMatchReason nvarchar(max) = NULL, + @Source nvarchar(60) = NULL, + @RelatedPlotLineID int = NULL, + @ParentCandidateKey nvarchar(220) = NULL, + @ParentNarrativeCandidateID int = NULL, + @Confidence decimal(5,4) = NULL +AS +BEGIN + SET NOCOUNT ON; + + DECLARE @NarrativeCandidateID int; + + SELECT @NarrativeCandidateID = NarrativeCandidateID + FROM dbo.StoryIntelligenceNarrativeCandidates + WHERE BookID = @BookID AND CandidateType = @CandidateType AND CandidateKey = @CandidateKey; + + IF @ParentNarrativeCandidateID IS NULL AND @ParentCandidateKey IS NOT NULL + BEGIN + SELECT @ParentNarrativeCandidateID = NarrativeCandidateID + FROM dbo.StoryIntelligenceNarrativeCandidates + WHERE BookID = @BookID + AND CandidateType = N'PlotLine' + AND CandidateKey = @ParentCandidateKey; + END; + + IF @NarrativeCandidateID IS NULL + BEGIN + INSERT dbo.StoryIntelligenceNarrativeCandidates + (ProjectID, BookID, CandidateType, CandidateKey, ProposedTitle, ProposedDescription, Category, InferredState, + ChangeType, Significance, ExistingPlotLineID, ExistingPlotThreadID, RelatedPlotLineID, ParentCandidateKey, + ParentNarrativeCandidateID, WholeBookPlotSynthesisRunID, AuthorPlotSeedID, ExistingPlotLineMatchConfidence, + ExistingPlotLineMatchReason, SeedMatchConfidence, SeedMatchReason, Confidence, Source) + VALUES + (@ProjectID, @BookID, @CandidateType, @CandidateKey, @ProposedTitle, @ProposedDescription, @Category, @InferredState, + @ChangeType, @Significance, @ExistingPlotLineID, @ExistingPlotThreadID, @RelatedPlotLineID, @ParentCandidateKey, + @ParentNarrativeCandidateID, @WholeBookPlotSynthesisRunID, @AuthorPlotSeedID, @ExistingPlotLineMatchConfidence, + @ExistingPlotLineMatchReason, @SeedMatchConfidence, @SeedMatchReason, @Confidence, COALESCE(@Source, N'SceneSignals')); + SET @NarrativeCandidateID = CAST(SCOPE_IDENTITY() AS int); + END + ELSE + BEGIN + UPDATE dbo.StoryIntelligenceNarrativeCandidates + SET ProposedTitle = @ProposedTitle, + ProposedDescription = @ProposedDescription, + Category = @Category, + InferredState = @InferredState, + ChangeType = @ChangeType, + Significance = @Significance, + ExistingPlotLineID = @ExistingPlotLineID, + ExistingPlotThreadID = @ExistingPlotThreadID, + RelatedPlotLineID = @RelatedPlotLineID, + ParentCandidateKey = @ParentCandidateKey, + ParentNarrativeCandidateID = @ParentNarrativeCandidateID, + WholeBookPlotSynthesisRunID = @WholeBookPlotSynthesisRunID, + AuthorPlotSeedID = @AuthorPlotSeedID, + ExistingPlotLineMatchConfidence = @ExistingPlotLineMatchConfidence, + ExistingPlotLineMatchReason = @ExistingPlotLineMatchReason, + SeedMatchConfidence = @SeedMatchConfidence, + SeedMatchReason = @SeedMatchReason, + Confidence = @Confidence, + Source = COALESCE(@Source, Source, N'SceneSignals'), + UpdatedUtc = SYSUTCDATETIME() + WHERE NarrativeCandidateID = @NarrativeCandidateID + AND Status = N'Pending'; + END; + + SELECT @NarrativeCandidateID; +END; +GO + +CREATE OR ALTER PROCEDURE dbo.StoryIntelligenceNarrativeCandidate_ListByBook + @BookID int +AS +BEGIN + SET NOCOUNT ON; + + SELECT nc.NarrativeCandidateID, nc.ProjectID, nc.BookID, nc.CandidateType, nc.CandidateKey, + nc.ProposedTitle, nc.ProposedDescription, nc.Category, nc.InferredState, nc.ChangeType, nc.Significance, + nc.ExistingPlotLineID, epl.PlotLineName AS ExistingPlotLineName, + nc.ExistingPlotThreadID, ept.ThreadTitle AS ExistingPlotThreadTitle, + nc.WholeBookPlotSynthesisRunID, nc.AuthorPlotSeedID, nc.ExistingPlotLineMatchConfidence, + nc.ExistingPlotLineMatchReason, nc.SeedMatchConfidence, nc.SeedMatchReason, nc.Source, + nc.RelatedPlotLineID, rpl.PlotLineName AS RelatedPlotLineName, + nc.ParentCandidateKey, nc.ParentNarrativeCandidateID, parent.ProposedTitle AS ParentCandidateTitle, + nc.MergeTargetNarrativeCandidateID, + nc.Confidence, nc.Status, nc.ReviewAction, nc.AcceptedPlotLineID, apl.PlotLineName AS AcceptedPlotLineName, + nc.AcceptedPlotThreadID, apt.ThreadTitle AS AcceptedPlotThreadTitle, nc.ReviewedByUserID, nc.ReviewedUtc, + nc.CreatedUtc, nc.UpdatedUtc + FROM dbo.StoryIntelligenceNarrativeCandidates nc + LEFT JOIN dbo.PlotLines epl ON epl.PlotLineID = nc.ExistingPlotLineID + LEFT JOIN dbo.PlotThreads ept ON ept.PlotThreadID = nc.ExistingPlotThreadID + LEFT JOIN dbo.PlotLines rpl ON rpl.PlotLineID = nc.RelatedPlotLineID + LEFT JOIN dbo.StoryIntelligenceNarrativeCandidates parent ON parent.NarrativeCandidateID = nc.ParentNarrativeCandidateID + LEFT JOIN dbo.PlotLines apl ON apl.PlotLineID = nc.AcceptedPlotLineID + LEFT JOIN dbo.PlotThreads apt ON apt.PlotThreadID = nc.AcceptedPlotThreadID + WHERE nc.BookID = @BookID + ORDER BY CASE nc.CandidateType WHEN N'PlotLine' THEN 0 ELSE 1 END, nc.ProposedTitle; + + SELECT e.NarrativeCandidateEvidenceID, e.NarrativeCandidateID, e.SceneID, + s.SceneNumber, s.SceneTitle, c.ChapterNumber, c.ChapterTitle, + e.ContributionType, e.Evidence, e.SortOrder, e.CreatedUtc + FROM dbo.StoryIntelligenceNarrativeCandidateEvidence e + INNER JOIN dbo.StoryIntelligenceNarrativeCandidates nc ON nc.NarrativeCandidateID = e.NarrativeCandidateID + INNER JOIN dbo.Scenes s ON s.SceneID = e.SceneID + INNER JOIN dbo.Chapters c ON c.ChapterID = s.ChapterID + WHERE nc.BookID = @BookID + ORDER BY e.NarrativeCandidateID, c.SortOrder, s.SortOrder, e.SortOrder; + + SELECT ev.NarrativeCandidateEventID, ev.NarrativeCandidateID, ev.WholeBookPlotSynthesisRunID, ev.EventKey, + ev.SceneID, s.SceneNumber, s.SceneTitle, c.ChapterNumber, c.ChapterTitle, + ev.EventType, ev.EventTitle, ev.EventDescription, ev.Evidence, ev.Confidence, ev.SortOrder, + ev.IsIgnored, ev.AcceptedThreadEventID, ev.CreatedUtc, ev.UpdatedUtc + FROM dbo.StoryIntelligenceNarrativeCandidateEvents ev + INNER JOIN dbo.StoryIntelligenceNarrativeCandidates nc ON nc.NarrativeCandidateID = ev.NarrativeCandidateID + INNER JOIN dbo.Scenes s ON s.SceneID = ev.SceneID + INNER JOIN dbo.Chapters c ON c.ChapterID = s.ChapterID + WHERE nc.BookID = @BookID + ORDER BY ev.NarrativeCandidateID, c.SortOrder, s.SortOrder, ev.SortOrder; +END; +GO + +CREATE OR ALTER PROCEDURE dbo.ThreadEvent_Save + @ThreadEventID int = NULL, + @PlotThreadID int, + @SceneID int, + @EventTypeID int, + @PlotEventTypeID int = 2, + @TargetPlotLineID int = NULL, + @EventTitle nvarchar(200), + @EventDescription nvarchar(max) = NULL, + @Source nvarchar(60) = N'Author', + @SourceReferenceID int = NULL, + @SourceKey nvarchar(260) = NULL, + @Confidence decimal(5,4) = NULL +AS +BEGIN + SET NOCOUNT ON; + + IF (@ThreadEventID IS NULL OR @ThreadEventID = 0) AND @SourceKey IS NOT NULL + BEGIN + SELECT @ThreadEventID = ThreadEventID + FROM dbo.ThreadEvents + WHERE Source = @Source AND SourceKey = @SourceKey; + END; + + IF @ThreadEventID IS NULL OR @ThreadEventID = 0 + BEGIN + INSERT dbo.ThreadEvents (PlotThreadID, SceneID, EventTypeID, PlotEventTypeID, TargetPlotLineID, EventTitle, EventDescription, Source, SourceReferenceID, SourceKey, Confidence) + VALUES (@PlotThreadID, @SceneID, @EventTypeID, @PlotEventTypeID, @TargetPlotLineID, @EventTitle, @EventDescription, @Source, @SourceReferenceID, @SourceKey, @Confidence); + SET @ThreadEventID = CAST(SCOPE_IDENTITY() AS int); + END + ELSE + BEGIN + UPDATE dbo.ThreadEvents + SET PlotThreadID = @PlotThreadID, + SceneID = @SceneID, + EventTypeID = @EventTypeID, + PlotEventTypeID = @PlotEventTypeID, + TargetPlotLineID = @TargetPlotLineID, + EventTitle = @EventTitle, + EventDescription = @EventDescription, + Source = @Source, + SourceReferenceID = @SourceReferenceID, + SourceKey = @SourceKey, + Confidence = @Confidence, + UpdatedDate = SYSUTCDATETIME() + WHERE ThreadEventID = @ThreadEventID + AND (Source <> N'Author' OR @Source = N'Author'); + END; + + DECLARE @EventTypeName nvarchar(100); + SELECT @EventTypeName = TypeName FROM dbo.ThreadEventTypes WHERE ThreadEventTypeID = @EventTypeID; + + IF @EventTypeName = N'Introduced' + BEGIN + UPDATE pt + SET IntroducedSceneID = COALESCE(IntroducedSceneID, @SceneID), + ThreadStatusID = CASE WHEN ts.StatusName = N'Planned' THEN introduced.ThreadStatusID ELSE pt.ThreadStatusID END, + UpdatedDate = SYSUTCDATETIME() + FROM dbo.PlotThreads pt + INNER JOIN dbo.ThreadStatuses ts ON ts.ThreadStatusID = pt.ThreadStatusID + CROSS APPLY (SELECT ThreadStatusID FROM dbo.ThreadStatuses WHERE StatusName = N'Introduced') introduced + WHERE pt.PlotThreadID = @PlotThreadID; + END; + + IF @EventTypeName IN (N'Resolved', N'Payoff') + BEGIN + UPDATE pt + SET ActualResolutionSceneID = COALESCE(ActualResolutionSceneID, @SceneID), + ThreadStatusID = CASE WHEN @EventTypeName = N'Resolved' THEN resolved.ThreadStatusID ELSE pt.ThreadStatusID END, + UpdatedDate = SYSUTCDATETIME() + FROM dbo.PlotThreads pt + CROSS APPLY (SELECT ThreadStatusID FROM dbo.ThreadStatuses WHERE StatusName = N'Resolved') resolved + WHERE pt.PlotThreadID = @PlotThreadID; + END; + + SELECT @ThreadEventID; +END; +GO diff --git a/PlotLine/ViewModels/OnboardingViewModels.cs b/PlotLine/ViewModels/OnboardingViewModels.cs index d4fc904..7440de3 100644 --- a/PlotLine/ViewModels/OnboardingViewModels.cs +++ b/PlotLine/ViewModels/OnboardingViewModels.cs @@ -143,6 +143,21 @@ public sealed class StoryIntelligenceOverviewViewModel public string AppearancePreference { get; init; } = StoryMemoryAppearancePreferences.Balanced; } +public sealed class StoryIntelligenceAuthorPlotContextViewModel +{ + public int ProjectID { get; init; } + public int BookID { get; init; } + public string BookTitle { get; init; } = string.Empty; + public IReadOnlyList Descriptions { get; init; } = ["", "", ""]; +} + +public sealed class StoryIntelligenceAuthorPlotContextForm +{ + public int ProjectID { get; set; } + public int BookID { get; set; } + public List PlotSeeds { get; set; } = []; +} + public sealed class StoryIntelligenceProgressViewModel { public StoryIntelligenceJobProgress Job { get; init; } = new(); @@ -160,6 +175,7 @@ public sealed class StoryIntelligenceProgressViewModel public StoryIntelligenceRelationshipReviewViewModel RelationshipReview { get; init; } = new(); public StoryIntelligenceKnowledgeReviewViewModel KnowledgeReview { get; init; } = new(); public StoryIntelligenceNarrativeReviewViewModel NarrativeReview { get; init; } = new(); + public WholeBookPlotSynthesisStatusViewModel PlotSynthesis { get; init; } = new(); public StoryIntelligencePipelineDashboardViewModel PipelineDashboard { get; init; } = new(); public int ChapterCount => Chapters.Count; public int CompletedChapterCount => Chapters.Count(chapter => chapter.IsRunComplete); @@ -200,6 +216,20 @@ public sealed class StoryIntelligenceProgressViewModel + AnalysisIssueCount; } +public sealed class WholeBookPlotSynthesisStatusViewModel +{ + public int? WholeBookPlotSynthesisRunID { get; init; } + public string Status { get; init; } = string.Empty; + public int CandidateCount { get; init; } + public int ThreadCount { get; init; } + public int EventCount { get; init; } + public string? ErrorMessage { get; init; } + public bool IsQueued => string.Equals(Status, WholeBookPlotSynthesisStatuses.Queued, StringComparison.OrdinalIgnoreCase); + public bool IsRunning => string.Equals(Status, WholeBookPlotSynthesisStatuses.Running, StringComparison.OrdinalIgnoreCase); + public bool IsCompleted => string.Equals(Status, WholeBookPlotSynthesisStatuses.Completed, StringComparison.OrdinalIgnoreCase); + public bool IsFailed => string.Equals(Status, WholeBookPlotSynthesisStatuses.Failed, StringComparison.OrdinalIgnoreCase); +} + public sealed class StoryIntelligencePipelineDashboardViewModel { public int ChaptersAnalysed { get; init; } @@ -249,6 +279,10 @@ public sealed class StoryIntelligenceNarrativeReviewCandidateViewModel public string Significance { get; init; } = string.Empty; public string Confidence { get; init; } = "Unknown"; public int? ExistingPlotLineID { get; init; } + public int? AuthorPlotSeedID { get; init; } + public string Source { get; init; } = string.Empty; + public string? ExistingPlotLineMatchReason { get; init; } + public string? SeedMatchReason { get; init; } public int? RelatedPlotLineID { get; init; } public string? RelatedPlotLineName { get; init; } public int? ParentNarrativeCandidateID { get; init; } @@ -256,6 +290,19 @@ public sealed class StoryIntelligenceNarrativeReviewCandidateViewModel public IReadOnlyList ChildThreads { get; init; } = []; public string DefaultAction { get; init; } = StoryIntelligenceNarrativeReviewActions.AcceptNew; public IReadOnlyList Evidence { get; init; } = []; + public IReadOnlyList Events { get; init; } = []; +} + +public sealed class StoryIntelligenceNarrativeEventViewModel +{ + public int NarrativeCandidateEventID { get; init; } + public int SceneID { get; init; } + public string SceneLabel { get; init; } = string.Empty; + public string EventType { get; init; } = string.Empty; + public string EventTitle { get; init; } = string.Empty; + public string EventDescription { get; init; } = string.Empty; + public string Evidence { get; init; } = string.Empty; + public string Confidence { get; init; } = "Unknown"; } public sealed class StoryIntelligenceNarrativeEvidenceViewModel @@ -283,6 +330,16 @@ public sealed class StoryIntelligenceNarrativeImportChoiceForm public int? RelatedPlotLineID { get; set; } public int? ParentNarrativeCandidateID { get; set; } public int? MergeTargetNarrativeCandidateID { get; set; } + public List Events { get; set; } = []; +} + +public sealed class StoryIntelligenceNarrativeEventChoiceForm +{ + public int NarrativeCandidateEventID { get; set; } + public bool Include { get; set; } = true; + public string EventType { get; set; } = string.Empty; + public string EventTitle { get; set; } = string.Empty; + public string? EventDescription { get; set; } } public sealed class StoryIntelligenceNarrativeImportResultViewModel diff --git a/PlotLine/Views/Books/Details.cshtml b/PlotLine/Views/Books/Details.cshtml index aefbd00..34c20a2 100644 --- a/PlotLine/Views/Books/Details.cshtml +++ b/PlotLine/Views/Books/Details.cshtml @@ -75,9 +75,11 @@ @if (pipeline is null) { -
- -
+ @action.ActionLabel } else { diff --git a/PlotLine/Views/Onboarding/BuildComplete.cshtml b/PlotLine/Views/Onboarding/BuildComplete.cshtml index a99f655..768e02a 100644 --- a/PlotLine/Views/Onboarding/BuildComplete.cshtml +++ b/PlotLine/Views/Onboarding/BuildComplete.cshtml @@ -111,9 +111,7 @@
@if (coreComplete) { -
- -
+ Analyse my book Not now } else diff --git a/PlotLine/Views/Onboarding/StoryIntelligenceAuthorPlotContext.cshtml b/PlotLine/Views/Onboarding/StoryIntelligenceAuthorPlotContext.cshtml new file mode 100644 index 0000000..70569fb --- /dev/null +++ b/PlotLine/Views/Onboarding/StoryIntelligenceAuthorPlotContext.cshtml @@ -0,0 +1,69 @@ +@model StoryIntelligenceAuthorPlotContextViewModel +@{ + ViewData["Title"] = "Help PlotDirector understand your story"; + var descriptions = Model.Descriptions.Count >= 3 + ? Model.Descriptions.ToList() + : Model.Descriptions.Concat(Enumerable.Repeat(string.Empty, 3 - Model.Descriptions.Count)).ToList(); +} + +
+
+
+

Story Intelligence

+

Help PlotDirector understand your story

+

Plot lines are the major stories, mysteries, conflicts and character arcs that develop through your book or series.

+

If you already know some important plot lines, you can tell PlotDirector about them here. A sentence or two is enough. PlotDirector will still analyse the manuscript and try to discover other plot lines for you.

+
+ +
+ + + +
+ @for (var i = 0; i < descriptions.Count; i++) + { +
+ + +
+ } +
+ + + +
+ + Not now +
+
+
+
+ +@section Scripts { + +} diff --git a/PlotLine/Views/Onboarding/StoryIntelligenceNarrative.cshtml b/PlotLine/Views/Onboarding/StoryIntelligenceNarrative.cshtml index e26dd86..5849979 100644 --- a/PlotLine/Views/Onboarding/StoryIntelligenceNarrative.cshtml +++ b/PlotLine/Views/Onboarding/StoryIntelligenceNarrative.cshtml @@ -7,6 +7,11 @@ ? Model.NarrativeReview.Candidates.ToList() : Model.NarrativeReview.Candidates.Take(previewLimit).ToList(); var hiddenCandidateCount = Math.Max(0, Model.NarrativeReview.Candidates.Count - narrativeCandidates.Count); + var eventTypes = new[] + { + "Introduced", "Mentioned", "Developed", "Complicated", "Clue Planted", "Question Raised", "Question Answered", + "False Lead", "Revealed", "Payoff", "Resolved", "Contradicted", "Retconned", "Note" + }; }
@@ -137,15 +142,59 @@
} -
- Supporting scenes -
    - @foreach (var evidence in candidate.Evidence) + @if (candidate.Events.Count > 0) + { +
    + Proposed timeline events + @for (var e = 0; e < candidate.Events.Count; e++) { -
  • @evidence.SceneLabel: @(string.IsNullOrWhiteSpace(evidence.Evidence) ? evidence.ContributionType : evidence.Evidence)
    • + var item = candidate.Events[e]; +
    + + + +
    +
    + + +
    +
    + + +
    +
    + + +
    +
    + @if (!string.IsNullOrWhiteSpace(item.Evidence)) + { +

    @item.Evidence

    + } +
    } -
-
+ + } + else + { +
+ Supporting scenes +
    + @foreach (var evidence in candidate.Evidence) + { +
  • @evidence.SceneLabel: @(string.IsNullOrWhiteSpace(evidence.Evidence) ? evidence.ContributionType : evidence.Evidence)
    • + } +
+
+ } } diff --git a/PlotLine/Views/Onboarding/StoryIntelligenceReviewCentre.cshtml b/PlotLine/Views/Onboarding/StoryIntelligenceReviewCentre.cshtml index 4253c5b..89d5830 100644 --- a/PlotLine/Views/Onboarding/StoryIntelligenceReviewCentre.cshtml +++ b/PlotLine/Views/Onboarding/StoryIntelligenceReviewCentre.cshtml @@ -60,14 +60,7 @@ ModuleAction("Locations", Model.LocationNeedsAttentionCount, Model.PipelineDashboard.LocationStageComplete), "StoryIntelligenceLocations") - @ReviewModule( - "Plot Lines & Threads", - Model.NarrativeNeedsAttentionCount, - Model.PipelineDashboard.NarrativeStageComplete, - !Model.CharacterGateSatisfied, - "Recurring arcs and threads discovered from scene evidence.", - ModuleAction("Plot Lines & Threads", Model.NarrativeNeedsAttentionCount, Model.PipelineDashboard.NarrativeStageComplete), - "StoryIntelligenceNarrative") + @ReviewPlotSynthesis() @ReviewModule( "Relationships", @@ -104,6 +97,77 @@ private string ModuleAction(string title, int count, bool complete) => count > 0 && !complete ? $"Review {title}" : $"View {title}"; + private Microsoft.AspNetCore.Html.IHtmlContent ReviewPlotSynthesis() + { + var blocked = !Model.CharacterGateSatisfied; + var status = "Preparing analysis"; + var statusClass = "story-review-centre-tile__status story-review-centre-tile__status--locked"; + var description = "Whole-book plot synthesis starts after Characters are resolved."; + var actionHtml = "Locked"; + + if (!blocked) + { + statusClass = "story-review-centre-tile__status story-review-centre-tile__status--ready"; + if (Model.PlotSynthesis.IsQueued) + { + status = "Preparing"; + description = "PlotDirector is preparing the whole-book plot synthesis."; + actionHtml = "Preparing"; + } + else if (Model.PlotSynthesis.IsRunning) + { + status = "Analysing"; + description = "PlotDirector is reading the full book to find sustained Plot Lines, Threads, and scene events."; + actionHtml = "Analysing"; + } + else if (Model.PlotSynthesis.IsFailed) + { + status = "Failed"; + statusClass = "story-review-centre-tile__status story-review-centre-tile__status--warning"; + description = string.IsNullOrWhiteSpace(Model.PlotSynthesis.ErrorMessage) + ? "Whole-book plot synthesis failed and can be retried." + : Model.PlotSynthesis.ErrorMessage; + actionHtml = $$""" +
+ + +
+ """; + } + else if (Model.PlotSynthesis.IsCompleted && Model.NarrativeNeedsAttentionCount > 0) + { + status = $"{Model.NarrativeNeedsAttentionCount:N0} need attention"; + statusClass = "story-review-centre-tile__status story-review-centre-tile__status--warning"; + description = $"{Model.PlotSynthesis.ThreadCount:N0} thread candidates and {Model.PlotSynthesis.EventCount:N0} proposed scene events are ready."; + var href = Url.Action("StoryIntelligenceNarrative", "Onboarding", new { batchId = Model.BatchID }) ?? "#"; + actionHtml = $"Review Plot Lines & Threads"; + } + else if (Model.PipelineDashboard.NarrativeStageComplete) + { + status = "Complete"; + description = "Plot Lines and Threads review has been saved."; + var href = Url.Action("StoryIntelligenceNarrative", "Onboarding", new { batchId = Model.BatchID }) ?? "#"; + actionHtml = $"View Plot Lines & Threads"; + } + else + { + description = "Plot synthesis will start automatically once character resolution is saved."; + actionHtml = "Preparing"; + } + } + + return new Microsoft.AspNetCore.Html.HtmlString($$""" +
+
+

Plot Lines & Threads

+

{{System.Net.WebUtility.HtmlEncode(description)}}

+
+ {{System.Net.WebUtility.HtmlEncode(status)}} + {{actionHtml}} +
+ """); + } + private string StatusText(string title, int count, bool complete, bool blocked) { if (blocked) diff --git a/PlotLine/appsettings.json b/PlotLine/appsettings.json index 16ccf8e..1fa6780 100644 --- a/PlotLine/appsettings.json +++ b/PlotLine/appsettings.json @@ -27,6 +27,7 @@ "StoryIntelligence": { "ChapterStructureModel": "gpt-5-mini", "SceneIntelligenceModel": "gpt-5-mini", + "WholeBookPlotSynthesisModel": "gpt-5", "SceneIntelligenceMaxOutputTokens": 10000, "MaxGlobalConcurrentAiRequests": 4, "MaxConcurrentAiRequestsPerBook": 2,