From b6aa53ee12d1463a107ef5977c4141b80df09e29 Mon Sep 17 00:00:00 2001 From: Nick Beckley Date: Wed, 26 Aug 2026 21:12:06 +0000 Subject: [PATCH] Refactor Story Intelligence to use canonical scenes --- PlotLine.Tests/Program.cs | 55 +- PlotLine/Data/OnboardingBuildRepository.cs | 11 + .../Data/StoryIntelligenceResultRepository.cs | 2 +- .../StoryIntelligencePersistenceModels.cs | 1 + .../PersistedStoryIntelligenceRunner.cs | 490 ++++++++++++------ .../StoryIntelligenceAssetImportService.cs | 9 +- ...StoryIntelligenceCharacterImportService.cs | 9 +- .../StoryIntelligenceImportCommitService.cs | 2 + ...StoryIntelligenceKnowledgeImportService.cs | 9 +- .../StoryIntelligenceLocationImportService.cs | 9 +- ...StoryIntelligenceNarrativeReviewService.cs | 9 +- ...ryIntelligenceRelationshipImportService.cs | 9 +- ..._CanonicalSceneSourceStoryIntelligence.sql | 41 ++ .../StoryIntelligenceProgress.cshtml | 6 +- .../wwwroot/js/story-intelligence-progress.js | 6 +- 15 files changed, 502 insertions(+), 166 deletions(-) create mode 100644 PlotLine/Sql/161_Phase25F_CanonicalSceneSourceStoryIntelligence.sql diff --git a/PlotLine.Tests/Program.cs b/PlotLine.Tests/Program.cs index a735f82..e356cc3 100644 --- a/PlotLine.Tests/Program.cs +++ b/PlotLine.Tests/Program.cs @@ -83,6 +83,8 @@ var tests = new (string Name, Action Test)[] ("Core import remaps characters after review", CoreImportRemapsCharactersAfterReview), ("Scan review separates known matches from unresolved names", ScanReviewSeparatesKnownMatchesFromUnresolvedNames), ("Story Intelligence uses persisted scene source after restart", StoryIntelligenceUsesPersistedSceneSourceAfterRestart), + ("Story Intelligence canonical mode bypasses duplicate boundary analysis", StoryIntelligenceCanonicalModeBypassesDuplicateBoundaryAnalysis), + ("Story Intelligence downstream imports prefer canonical SceneIDs", StoryIntelligenceDownstreamImportsPreferCanonicalSceneIds), ("Word sync requires save acknowledgement", WordSyncRequiresSaveAcknowledgement), ("Word sync retry resumes existing structure", WordSyncRetryResumesExistingStructure), ("Story Intelligence enriches existing structural scenes", StoryIntelligenceEnrichesExistingStructuralScenes), @@ -1981,7 +1983,56 @@ static void StoryIntelligenceUsesPersistedSceneSourceAfterRestart() Assert(onboardingStory.Contains("durableSource = await sources.GetChapterSourceAsync", StringComparison.Ordinal), "Onboarding Story Intelligence should read durable chapter text from scene sources."); Assert(onboardingStory.Contains("CoreImportComplete", StringComparison.Ordinal), "Optional onboarding Story Intelligence should require core import completion."); Assert(existingQueue.Contains("sources.GetChapterSourceAsync", StringComparison.Ordinal), "Later Story Intelligence queueing should start from persisted chapter/scene source, not onboarding DTOs."); - Assert(runner.Contains("structuralSummary = boundary.StructuralSummary", StringComparison.Ordinal), "Scene Intelligence context should receive structural summaries from the boundary pass."); + Assert(runner.Contains("FirstNonBlank(source.StructuralSummary, scene.StructuralSummary)", StringComparison.Ordinal), "Scene Intelligence context should receive structural summaries from canonical scene sources."); +} + +static void StoryIntelligenceCanonicalModeBypassesDuplicateBoundaryAnalysis() +{ + var root = Path.Combine(AppContext.BaseDirectory, "../../../../PlotLine"); + var runner = File.ReadAllText(Path.Combine(root, "Services/PersistedStoryIntelligenceRunner.cs")); + var repository = File.ReadAllText(Path.Combine(root, "Data/OnboardingBuildRepository.cs")); + var progressView = File.ReadAllText(Path.Combine(root, "Views/Onboarding/StoryIntelligenceProgress.cshtml")); + var progressJs = File.ReadAllText(Path.Combine(root, "wwwroot/js/story-intelligence-progress.js")); + var migration = File.ReadAllText(Path.Combine(root, "Sql/161_Phase25F_CanonicalSceneSourceStoryIntelligence.sql")); + + Assert(runner.Contains("\"CanonicalSceneSource\"", StringComparison.Ordinal), "Runner should have an explicit canonical scene source analysis mode."); + Assert(runner.Contains("BuildCanonicalSceneBlock", StringComparison.Ordinal), "Canonical mode should build scene prompts from existing scenes."); + Assert(runner.Contains("ProcessLegacyChapterAsync", StringComparison.Ordinal), "Legacy Chapter Structure analysis should remain isolated in its fallback path."); + Assert(runner.Contains("builds.ListSceneManuscriptSourcesByChapterAsync", StringComparison.Ordinal), "Canonical mode should load scene source rows directly."); + Assert(runner.Contains("throw new StoryIntelligenceRunFailureException(\n StoryIntelligenceFailureStages.DocumentRead", StringComparison.Ordinal), "Partial canonical source coverage should fail as a data integrity issue."); + Assert(runner.Contains("SceneID = block.SceneID", StringComparison.Ordinal), "Saved scene results should retain canonical SceneIDs."); + Assert(!runner.Contains("existingScenes.FirstOrDefault(scene => Convert.ToInt32(scene.SceneNumber) == block.TemporarySceneNumber)", StringComparison.Ordinal), "Runner should not remap canonical scene results by temporary scene number."); + Assert(repository.Contains("SceneManuscriptSource_ListByChapter", StringComparison.Ordinal), "Canonical source lookup should use a stored procedure."); + Assert(migration.Contains("CREATE OR ALTER PROCEDURE dbo.SceneManuscriptSource_ListByChapter", StringComparison.Ordinal), "Forward migration should provide the canonical scene source procedure."); + Assert(progressView.Contains("Loading scenes...", StringComparison.Ordinal), "Canonical progress should load scenes rather than claim to find them."); + Assert(progressJs.Contains("Loading scenes...", StringComparison.Ordinal), "Live canonical progress should load scenes rather than claim to find them."); +} + +static void StoryIntelligenceDownstreamImportsPreferCanonicalSceneIds() +{ + var root = Path.Combine(AppContext.BaseDirectory, "../../../../PlotLine"); + var commit = File.ReadAllText(Path.Combine(root, "Services/StoryIntelligenceImportCommitService.cs")); + var repository = File.ReadAllText(Path.Combine(root, "Data/StoryIntelligenceResultRepository.cs")); + var model = File.ReadAllText(Path.Combine(root, "Models/StoryIntelligencePersistenceModels.cs")); + var importServices = new[] + { + "StoryIntelligenceCharacterImportService.cs", + "StoryIntelligenceRelationshipImportService.cs", + "StoryIntelligenceLocationImportService.cs", + "StoryIntelligenceKnowledgeImportService.cs", + "StoryIntelligenceAssetImportService.cs", + "StoryIntelligenceNarrativeReviewService.cs" + }; + + Assert(model.Contains("public int? SceneID { get; init; }", StringComparison.Ordinal), "Import preview items should carry canonical SceneID."); + Assert(commit.Contains("SceneID = sceneResult.SceneID", StringComparison.Ordinal), "Import preview should preserve the saved scene result ID."); + Assert(repository.Contains("SceneID = item.SceneID ??", StringComparison.Ordinal), "Commit should update canonical scenes by SceneID before falling back to order."); + foreach (var file in importServices) + { + var source = File.ReadAllText(Path.Combine(root, "Services", file)); + Assert(source.Contains("var importedById = importedScenes.ToDictionary(scene => scene.SceneID, scene => scene);", StringComparison.Ordinal), $"{file} should index imported scenes by canonical SceneID."); + Assert(source.Contains("sceneResult.SceneID.HasValue", StringComparison.Ordinal), $"{file} should resolve scene evidence by canonical SceneID before range/number fallbacks."); + } } static void WordSyncRequiresSaveAcknowledgement() @@ -2047,7 +2098,7 @@ static void StoryIntelligenceEnrichesExistingStructuralScenes() var commit = File.ReadAllText(Path.Combine(root, "Services/StoryIntelligenceImportCommitService.cs")); Assert(runner.Contains("ListChapterScenesAsync", StringComparison.Ordinal), "Story Intelligence scene results should be linked to existing SceneIDs."); - Assert(repository.Contains("SceneID = existingScenes.Count > index", StringComparison.Ordinal), "Story Intelligence commit should update existing structural scenes when present."); + Assert(repository.Contains("SceneID = item.SceneID ?? (existingScenes.Count > index", StringComparison.Ordinal), "Story Intelligence commit should update existing structural scenes when present."); Assert(repository.Contains("Enriched existing structural scenes", StringComparison.Ordinal), "Commit audit should distinguish enrichment from scene creation."); Assert(commit.Contains("readiness.ActiveSceneCount != importScenes.Count", StringComparison.Ordinal), "Review readiness should allow matching structural scenes while blocking mismatches."); } diff --git a/PlotLine/Data/OnboardingBuildRepository.cs b/PlotLine/Data/OnboardingBuildRepository.cs index 9a5eb74..b7c86c9 100644 --- a/PlotLine/Data/OnboardingBuildRepository.cs +++ b/PlotLine/Data/OnboardingBuildRepository.cs @@ -16,6 +16,7 @@ public interface IOnboardingBuildRepository Task SaveStructuralChapterSourceAsync(StructuralChapterSourceSaveRequest request); Task SaveSceneManuscriptSourceAsync(SceneManuscriptSourceSaveRequest request); Task SaveSceneStructuralSummaryAsync(int sceneId, string structuralSummary); + Task> ListSceneManuscriptSourcesByChapterAsync(int chapterId); Task> ListSceneManuscriptSourcesByBookAsync(int bookId); } @@ -182,6 +183,16 @@ public sealed class OnboardingBuildRepository(ISqlConnectionFactory connectionFa commandType: CommandType.StoredProcedure); return rows.ToList(); } + + public async Task> ListSceneManuscriptSourcesByChapterAsync(int chapterId) + { + using var connection = connectionFactory.CreateConnection(); + var rows = await connection.QueryAsync( + "dbo.SceneManuscriptSource_ListByChapter", + new { ChapterID = chapterId }, + commandType: CommandType.StoredProcedure); + return rows.ToList(); + } } public sealed class OnboardingManuscriptBuildRequest diff --git a/PlotLine/Data/StoryIntelligenceResultRepository.cs b/PlotLine/Data/StoryIntelligenceResultRepository.cs index 142304e..ef6a0c7 100644 --- a/PlotLine/Data/StoryIntelligenceResultRepository.cs +++ b/PlotLine/Data/StoryIntelligenceResultRepository.cs @@ -642,7 +642,7 @@ public sealed class StoryIntelligenceResultRepository(ISqlConnectionFactory conn "dbo.Scene_Save", new { - SceneID = existingScenes.Count > index ? existingScenes[index].SceneID : (int?)null, + SceneID = item.SceneID ?? (existingScenes.Count > index ? existingScenes[index].SceneID : (int?)null), ChapterID = request.ChapterID, SceneNumber = Convert.ToDecimal(item.TemporarySceneNumber), SceneTitle = sceneTitle, diff --git a/PlotLine/Models/StoryIntelligencePersistenceModels.cs b/PlotLine/Models/StoryIntelligencePersistenceModels.cs index 870f5cd..75d2262 100644 --- a/PlotLine/Models/StoryIntelligencePersistenceModels.cs +++ b/PlotLine/Models/StoryIntelligencePersistenceModels.cs @@ -490,6 +490,7 @@ public sealed class StoryIntelligenceImportCommit public sealed class StoryIntelligenceSceneImportItem { public int SceneResultID { get; init; } + public int? SceneID { get; init; } public int TemporarySceneNumber { get; init; } public int? StartParagraph { get; init; } public int? EndParagraph { get; init; } diff --git a/PlotLine/Services/PersistedStoryIntelligenceRunner.cs b/PlotLine/Services/PersistedStoryIntelligenceRunner.cs index 15aa48f..7762607 100644 --- a/PlotLine/Services/PersistedStoryIntelligenceRunner.cs +++ b/PlotLine/Services/PersistedStoryIntelligenceRunner.cs @@ -14,6 +14,7 @@ public interface IPersistedStoryIntelligenceRunner public sealed class PersistedStoryIntelligenceRunner( IStoryIntelligenceResultRepository repository, IStoryIntelligencePipelineRepository pipelines, + IOnboardingBuildRepository builds, ISceneMetricTypeRepository metricTypes, IPlotRepository plots, IStoryPromptRepository prompts, @@ -77,138 +78,9 @@ public sealed class PersistedStoryIntelligenceRunner( "Story Intelligence has started reading this chapter.", stopwatch.ElapsedMilliseconds); - if (string.IsNullOrWhiteSpace(run.SourceText)) - { - throw new StoryIntelligenceRunFailureException( - StoryIntelligenceFailureStages.DocumentRead, - "Story Intelligence run has no source text."); - } - await entitlement.EnsureCanExecuteAsync(run.UserID); await ThrowIfCancellationRequestedAsync(run.StoryIntelligenceRunID, stopwatch.ElapsedMilliseconds, cancellationToken); - var paragraphs = StoryIntelligenceParagraphs.Split(run.SourceText); - if (paragraphs.Count == 0) - { - throw new StoryIntelligenceRunFailureException( - StoryIntelligenceFailureStages.DocumentRead, - "Story Intelligence run source text does not contain readable paragraphs."); - } - - var chapterContextJson = BuildChapterContextJson(run); - var chapterTemplate = await prompts.LoadPromptAsync(ChapterPromptFile, cancellationToken); - var numberedChapterText = StoryIntelligenceParagraphs.Number(paragraphs); - var chapterPrompt = BuildChapterPrompt(chapterTemplate, chapterContextJson, numberedChapterText); - currentFailureStage = StoryIntelligenceFailureStages.ChapterStructure; - - await repository.UpdateProgressAsync( - run.StoryIntelligenceRunID, - currentStage: StoryIntelligenceFailureStages.ChapterStructure, - currentMessage: "Running Chapter Structure analysis.", - totalDurationMs: stopwatch.ElapsedMilliseconds); - await PublishAsync( - run, - "Chapter started", - StoryIntelligenceRunStatuses.Running, - StoryIntelligenceFailureStages.ChapterStructure, - "PlotDirector is finding the scene boundaries in this chapter.", - stopwatch.ElapsedMilliseconds); - - var chapterClientResult = await client.ExecutePromptAsync( - chapterPrompt, - chapterPromptVersion, - cancellationToken, - stageModels.ChapterStructureModel); - totals.Add(chapterClientResult, pricing); - AiJsonParseResult chapterAttempt; - try - { - chapterAttempt = await ParseAiJsonWithRetryAsync( - chapterPrompt, - chapterPromptVersion, - stageModels.ChapterStructureModel, - chapterClientResult, - totals, - "Chapter Structure", - cancellationToken); - } - catch (AiJsonParseException ex) - { - await repository.SaveChapterResultAsync( - run.StoryIntelligenceRunID, - new StoryIntelligenceChapterResultSaveRequest - { - ProjectID = run.ProjectID, - BookID = run.BookID, - ChapterID = run.ChapterID, - ChapterNumber = ChapterNumber(run), - SourceLabel = SourceLabel(run), - PromptVersion = chapterPromptVersion, - Model = stageModels.ChapterStructureModel, - RawResponseJson = ex.RawResponseJson, - OutputTextJson = ex.RawAssistantOutput, - ParsedJson = SerialiseFailedAiJsonParse(ex), - ValidationErrorsCount = 1, - ValidationWarningsCount = ex.RetryAttempted ? 1 : 0, - InputTokens = ex.InputTokens, - OutputTokens = ex.OutputTokens, - TotalTokens = SumTokens(ex.InputTokens, ex.OutputTokens), - DurationMs = Convert.ToInt64(ex.Duration.TotalMilliseconds) - }); - - throw new StoryIntelligenceRunFailureException( - StoryIntelligenceFailureStages.Validation, - "Chapter Structure JSON could not be parsed after one retry.", - ex.ToString()); - } - - var parsedChapter = chapterAttempt.Parsed; - var chapterNormalisation = ChapterStructureBoundaryNormaliser.Normalise(parsedChapter, paragraphs.Count); - var chapterForProcessing = chapterNormalisation.ChapterStructure; - var chapterValidation = chapterValidator.Validate(chapterForProcessing, paragraphs.Count); - foreach (var warning in chapterAttempt.Warnings) - { - chapterValidation.Warnings.Add(new PlotLine.Models.StoryIntelligence.ValidationIssue - { - Severity = "Warning", - Path = "$", - Message = warning, - SuggestedFix = "Review the raw and repaired Chapter Structure JSON." - }); - } - - ChapterStructureBoundaryNormaliser.AddIssuesTo(chapterValidation, chapterNormalisation); - hasWarnings = chapterValidation.Warnings.Count > 0; - chapterResultId = await repository.SaveChapterResultAsync( - run.StoryIntelligenceRunID, - new StoryIntelligenceChapterResultSaveRequest - { - ProjectID = run.ProjectID, - BookID = run.BookID, - ChapterID = run.ChapterID, - ChapterNumber = ChapterNumber(run), - SourceLabel = SourceLabel(run), - PromptVersion = chapterPromptVersion, - Model = chapterAttempt.Model, - RawResponseJson = chapterAttempt.RawResponseJson, - OutputTextJson = chapterAttempt.RawAssistantOutput, - ParsedJson = SerialiseChapterParseAudit(chapterForProcessing, chapterNormalisation, chapterAttempt), - ValidationErrorsCount = chapterValidation.Errors.Count, - ValidationWarningsCount = chapterValidation.Warnings.Count, - InputTokens = chapterAttempt.InputTokens, - OutputTokens = chapterAttempt.OutputTokens, - TotalTokens = SumTokens(chapterAttempt.InputTokens, chapterAttempt.OutputTokens), - DurationMs = Convert.ToInt64(chapterAttempt.Duration.TotalMilliseconds) - }); - - if (!chapterValidation.IsValid || chapterForProcessing?.SceneBoundaries is null) - { - throw new StoryIntelligenceRunFailureException( - StoryIntelligenceFailureStages.ChapterStructure, - "Chapter Structure validation failed.", - BuildValidationDetail(chapterValidation)); - } - var configuredMetrics = run.ProjectID.HasValue ? (await metricTypes.ListForManagementAsync(run.ProjectID.Value)) .Where(metric => metric.IsActive && metric.IsEnabledForProject) @@ -223,16 +95,34 @@ public sealed class PersistedStoryIntelligenceRunner( .ThenBy(plotLine => plotLine.PlotLineName) .ToList() : []; - var sceneBlocks = chapterForProcessing.SceneBoundaries - .Select(boundary => BuildSceneBlock(run, paragraphs, boundary, configuredMetrics, existingPlotLines)) - .ToList(); + var mode = await SelectSceneAnalysisModeAsync( + run, + configuredMetrics, + existingPlotLines, + chapterPromptVersion, + stageModels, + totals, + stopwatch.ElapsedMilliseconds, + cancellationToken); + var sceneBlocks = mode.SceneBlocks; + chapterResultId = mode.ChapterResultId; + hasWarnings = mode.HasWarnings; failedScenes += sceneBlocks.Count(block => !block.SplitValid); currentFailureStage = StoryIntelligenceFailureStages.SceneSplit; + logger.LogInformation( + "Story Intelligence mode: {Mode}. RunID={RunID} ProjectID={ProjectID} BookID={BookID} ChapterID={ChapterID} SceneCount={SceneCount}", + mode.Mode, + run.StoryIntelligenceRunID, + run.ProjectID, + run.BookID, + run.ChapterID, + sceneBlocks.Count); + await repository.UpdateProgressAsync( run.StoryIntelligenceRunID, currentStage: StoryIntelligenceFailureStages.SceneSplit, - currentMessage: $"Detected {sceneBlocks.Count:N0} suggested scene(s).", + currentMessage: mode.PreparationMessage, totalDetectedScenes: sceneBlocks.Count, failedScenes: failedScenes, totalInputTokens: totals.InputTokens, @@ -243,19 +133,16 @@ public sealed class PersistedStoryIntelligenceRunner( estimatedCostGBP: totals.EstimatedCostGBP); await PublishAsync( run, - "Scene boundaries detected", + mode.PreparedEventType, StoryIntelligenceRunStatuses.Running, StoryIntelligenceFailureStages.SceneSplit, - $"Detected {sceneBlocks.Count:N0} suggested scene(s).", + mode.PreparationMessage, stopwatch.ElapsedMilliseconds, totalDetectedScenes: sceneBlocks.Count, completedScenes: completedScenes, failedScenes: failedScenes, totalTokens: totals.TotalTokens); - var existingScenes = run.ChapterID.HasValue - ? await repository.ListChapterScenesAsync(run.ChapterID.Value) - : Array.Empty(); var sceneTemplate = await prompts.LoadPromptAsync(ScenePromptFile, cancellationToken); foreach (var block in sceneBlocks) { @@ -268,7 +155,7 @@ public sealed class PersistedStoryIntelligenceRunner( "Scene analysis failed", StoryIntelligenceRunStatuses.Running, StoryIntelligenceFailureStages.SceneIntelligence, - $"Suggested scene {block.TemporarySceneNumber:N0} could not be analysed because its paragraph range was invalid.", + $"{SceneProgressLabel(block)} could not be analysed because its paragraph range was invalid.", stopwatch.ElapsedMilliseconds, totalDetectedScenes: sceneBlocks.Count, completedScenes: completedScenes, @@ -280,7 +167,7 @@ public sealed class PersistedStoryIntelligenceRunner( await repository.UpdateProgressAsync( run.StoryIntelligenceRunID, currentStage: StoryIntelligenceFailureStages.SceneIntelligence, - currentMessage: $"Running Scene Intelligence for suggested scene {block.TemporarySceneNumber:N0}.", + currentMessage: $"Analysing {SceneProgressLabel(block)} of {sceneBlocks.Count:N0}.", completedScenes: completedScenes, failedScenes: failedScenes, totalDurationMs: stopwatch.ElapsedMilliseconds); @@ -289,7 +176,7 @@ public sealed class PersistedStoryIntelligenceRunner( "Scene analysis started", StoryIntelligenceRunStatuses.Running, StoryIntelligenceFailureStages.SceneIntelligence, - $"Reading scene {block.TemporarySceneNumber:N0}.", + $"Analysing {SceneProgressLabel(block)}.", stopwatch.ElapsedMilliseconds, totalDetectedScenes: sceneBlocks.Count, completedScenes: completedScenes, @@ -338,7 +225,7 @@ public sealed class PersistedStoryIntelligenceRunner( ProjectID = run.ProjectID, BookID = run.BookID, ChapterID = run.ChapterID, - SceneID = existingScenes.FirstOrDefault(scene => Convert.ToInt32(scene.SceneNumber) == block.TemporarySceneNumber)?.SceneID, + SceneID = block.SceneID, TemporarySceneNumber = block.TemporarySceneNumber, StartParagraph = block.StartParagraph, EndParagraph = block.EndParagraph, @@ -549,6 +436,238 @@ public sealed class PersistedStoryIntelligenceRunner( } } + private async Task SelectSceneAnalysisModeAsync( + StoryIntelligenceQueuedRun run, + IReadOnlyList configuredMetrics, + IReadOnlyList existingPlotLines, + string chapterPromptVersion, + StoryIntelligenceStageModels stageModels, + TokenTotals totals, + long elapsedMs, + CancellationToken cancellationToken) + { + if (!run.ChapterID.HasValue) + { + return await ProcessLegacyChapterAsync(run, configuredMetrics, existingPlotLines, chapterPromptVersion, stageModels, totals, elapsedMs, cancellationToken); + } + + var canonicalScenes = await repository.ListChapterScenesAsync(run.ChapterID.Value); + if (canonicalScenes.Count == 0) + { + logger.LogInformation( + "Story Intelligence mode: LegacyBoundaryDetection. RunID={RunID} ChapterID={ChapterID} has no active canonical scenes.", + run.StoryIntelligenceRunID, + run.ChapterID); + return await ProcessLegacyChapterAsync(run, configuredMetrics, existingPlotLines, chapterPromptVersion, stageModels, totals, elapsedMs, cancellationToken); + } + + var sourceRows = await builds.ListSceneManuscriptSourcesByChapterAsync(run.ChapterID.Value); + if (sourceRows.Count == 0) + { + logger.LogInformation( + "Story Intelligence mode: LegacyBoundaryDetection. RunID={RunID} ChapterID={ChapterID} has active scenes but no canonical scene source rows.", + run.StoryIntelligenceRunID, + run.ChapterID); + return await ProcessLegacyChapterAsync(run, configuredMetrics, existingPlotLines, chapterPromptVersion, stageModels, totals, elapsedMs, cancellationToken); + } + + var sourcesByScene = sourceRows.ToDictionary(source => source.SceneID); + var missingSources = canonicalScenes + .Where(scene => !sourcesByScene.TryGetValue(scene.SceneID, out var source) + || string.IsNullOrWhiteSpace(source.SourceText)) + .Select(scene => scene.SceneID) + .ToList(); + var extraSources = sourceRows + .Where(source => canonicalScenes.All(scene => scene.SceneID != source.SceneID)) + .Select(source => source.SceneID) + .ToList(); + + if (missingSources.Count > 0 || extraSources.Count > 0) + { + var detail = $"Canonical scene source coverage is incomplete for ChapterID {run.ChapterID.Value}. " + + $"Active scenes: {canonicalScenes.Count:N0}; source rows: {sourceRows.Count:N0}; " + + $"missing/blank source SceneIDs: {string.Join(", ", missingSources)}; extra source SceneIDs: {string.Join(", ", extraSources)}."; + throw new StoryIntelligenceRunFailureException( + StoryIntelligenceFailureStages.DocumentRead, + $"Canonical scene source is missing for ChapterID {run.ChapterID.Value}. Story Intelligence cannot analyse this chapter until source data is repaired.", + detail); + } + + var sceneBlocks = canonicalScenes + .Select((scene, index) => BuildCanonicalSceneBlock(run, scene, sourcesByScene[scene.SceneID], index + 1, configuredMetrics, existingPlotLines)) + .ToList(); + + if (sceneBlocks.Any(block => !block.SplitValid)) + { + var invalidSceneIds = sceneBlocks + .Where(block => !block.SplitValid) + .Select(block => block.SceneID?.ToString() ?? block.TemporarySceneNumber.ToString("N0")) + .ToList(); + throw new StoryIntelligenceRunFailureException( + StoryIntelligenceFailureStages.DocumentRead, + $"Canonical scene source is missing for SceneID {invalidSceneIds.FirstOrDefault() ?? "unknown"}. Story Intelligence cannot analyse this scene until source data is repaired.", + $"Invalid canonical scene source rows: {string.Join(", ", invalidSceneIds)}."); + } + + return new SceneAnalysisModeSelection( + "CanonicalSceneSource", + sceneBlocks, + null, + false, + $"Loaded {sceneBlocks.Count:N0} canonical scene(s) for analysis.", + "Canonical scenes loaded"); + } + + private async Task ProcessLegacyChapterAsync( + StoryIntelligenceQueuedRun run, + IReadOnlyList configuredMetrics, + IReadOnlyList existingPlotLines, + string chapterPromptVersion, + StoryIntelligenceStageModels stageModels, + TokenTotals totals, + long elapsedMs, + CancellationToken cancellationToken) + { + if (string.IsNullOrWhiteSpace(run.SourceText)) + { + throw new StoryIntelligenceRunFailureException( + StoryIntelligenceFailureStages.DocumentRead, + "Story Intelligence run has no source text."); + } + + var paragraphs = StoryIntelligenceParagraphs.Split(run.SourceText); + if (paragraphs.Count == 0) + { + throw new StoryIntelligenceRunFailureException( + StoryIntelligenceFailureStages.DocumentRead, + "Story Intelligence run source text does not contain readable paragraphs."); + } + + var chapterContextJson = BuildChapterContextJson(run); + var chapterTemplate = await prompts.LoadPromptAsync(ChapterPromptFile, cancellationToken); + var numberedChapterText = StoryIntelligenceParagraphs.Number(paragraphs); + var chapterPrompt = BuildChapterPrompt(chapterTemplate, chapterContextJson, numberedChapterText); + + await repository.UpdateProgressAsync( + run.StoryIntelligenceRunID, + currentStage: StoryIntelligenceFailureStages.ChapterStructure, + currentMessage: "Running legacy Chapter Structure analysis.", + totalDurationMs: elapsedMs); + await PublishAsync( + run, + "Legacy chapter structure started", + StoryIntelligenceRunStatuses.Running, + StoryIntelligenceFailureStages.ChapterStructure, + "PlotDirector is using legacy scene-boundary detection for this chapter.", + elapsedMs); + + var chapterClientResult = await client.ExecutePromptAsync( + chapterPrompt, + chapterPromptVersion, + cancellationToken, + stageModels.ChapterStructureModel); + totals.Add(chapterClientResult, pricing); + AiJsonParseResult chapterAttempt; + try + { + chapterAttempt = await ParseAiJsonWithRetryAsync( + chapterPrompt, + chapterPromptVersion, + stageModels.ChapterStructureModel, + chapterClientResult, + totals, + "Chapter Structure", + cancellationToken); + } + catch (AiJsonParseException ex) + { + await repository.SaveChapterResultAsync( + run.StoryIntelligenceRunID, + new StoryIntelligenceChapterResultSaveRequest + { + ProjectID = run.ProjectID, + BookID = run.BookID, + ChapterID = run.ChapterID, + ChapterNumber = ChapterNumber(run), + SourceLabel = SourceLabel(run), + PromptVersion = chapterPromptVersion, + Model = stageModels.ChapterStructureModel, + RawResponseJson = ex.RawResponseJson, + OutputTextJson = ex.RawAssistantOutput, + ParsedJson = SerialiseFailedAiJsonParse(ex), + ValidationErrorsCount = 1, + ValidationWarningsCount = ex.RetryAttempted ? 1 : 0, + InputTokens = ex.InputTokens, + OutputTokens = ex.OutputTokens, + TotalTokens = SumTokens(ex.InputTokens, ex.OutputTokens), + DurationMs = Convert.ToInt64(ex.Duration.TotalMilliseconds) + }); + + throw new StoryIntelligenceRunFailureException( + StoryIntelligenceFailureStages.Validation, + "Chapter Structure JSON could not be parsed after one retry.", + ex.ToString()); + } + + var parsedChapter = chapterAttempt.Parsed; + var chapterNormalisation = ChapterStructureBoundaryNormaliser.Normalise(parsedChapter, paragraphs.Count); + var chapterForProcessing = chapterNormalisation.ChapterStructure; + var chapterValidation = chapterValidator.Validate(chapterForProcessing, paragraphs.Count); + foreach (var warning in chapterAttempt.Warnings) + { + chapterValidation.Warnings.Add(new PlotLine.Models.StoryIntelligence.ValidationIssue + { + Severity = "Warning", + Path = "$", + Message = warning, + SuggestedFix = "Review the raw and repaired Chapter Structure JSON." + }); + } + + ChapterStructureBoundaryNormaliser.AddIssuesTo(chapterValidation, chapterNormalisation); + var chapterResultId = await repository.SaveChapterResultAsync( + run.StoryIntelligenceRunID, + new StoryIntelligenceChapterResultSaveRequest + { + ProjectID = run.ProjectID, + BookID = run.BookID, + ChapterID = run.ChapterID, + ChapterNumber = ChapterNumber(run), + SourceLabel = SourceLabel(run), + PromptVersion = chapterPromptVersion, + Model = chapterAttempt.Model, + RawResponseJson = chapterAttempt.RawResponseJson, + OutputTextJson = chapterAttempt.RawAssistantOutput, + ParsedJson = SerialiseChapterParseAudit(chapterForProcessing, chapterNormalisation, chapterAttempt), + ValidationErrorsCount = chapterValidation.Errors.Count, + ValidationWarningsCount = chapterValidation.Warnings.Count, + InputTokens = chapterAttempt.InputTokens, + OutputTokens = chapterAttempt.OutputTokens, + TotalTokens = SumTokens(chapterAttempt.InputTokens, chapterAttempt.OutputTokens), + DurationMs = Convert.ToInt64(chapterAttempt.Duration.TotalMilliseconds) + }); + + if (!chapterValidation.IsValid || chapterForProcessing?.SceneBoundaries is null) + { + throw new StoryIntelligenceRunFailureException( + StoryIntelligenceFailureStages.ChapterStructure, + "Chapter Structure validation failed.", + BuildValidationDetail(chapterValidation)); + } + + var sceneBlocks = chapterForProcessing.SceneBoundaries + .Select(boundary => BuildLegacySceneBlock(run, paragraphs, boundary, configuredMetrics, existingPlotLines)) + .ToList(); + + return new SceneAnalysisModeSelection( + "LegacyBoundaryDetection", + sceneBlocks, + chapterResultId, + chapterValidation.Warnings.Count > 0, + $"Prepared {sceneBlocks.Count:N0} legacy scene-boundary result(s).", + "Legacy scene boundaries detected"); + } + private async Task ThrowIfCancellationRequestedAsync(int runId, long elapsedMs, CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); @@ -810,7 +929,7 @@ public sealed class PersistedStoryIntelligenceRunner( ProjectID = run.ProjectID, BookID = run.BookID, ChapterID = run.ChapterID, - SceneID = null, + SceneID = block.SceneID, TemporarySceneNumber = block.TemporarySceneNumber, StartParagraph = block.StartParagraph, EndParagraph = block.EndParagraph, @@ -859,7 +978,45 @@ public sealed class PersistedStoryIntelligenceRunner( ? SceneIntelligenceMaxOutputTokens() : settings.MaxOutputTokens; - private static StorySceneTextBlock BuildSceneBlock( + private static StorySceneTextBlock BuildCanonicalSceneBlock( + StoryIntelligenceQueuedRun run, + Scene scene, + SceneManuscriptSourceRecord source, + int ordinal, + IReadOnlyList configuredMetrics, + IReadOnlyList existingPlotLines) + { + var sceneNumber = SceneDisplayNumber(scene.SceneNumber, ordinal); + var start = source.SourceStartParagraph ?? source.WordStartParagraphIndex ?? 0; + var end = source.SourceEndParagraph ?? source.WordEndParagraphIndex ?? 0; + var sourceLabel = $"{SourceLabel(run)}, scene {sceneNumber:N0}"; + var sceneText = (source.SourceText ?? string.Empty).Trim(); + var splitValid = scene.SceneID > 0 + && !string.IsNullOrWhiteSpace(sceneText) + && (!source.SourceStartParagraph.HasValue || !source.SourceEndParagraph.HasValue || source.SourceStartParagraph <= source.SourceEndParagraph) + && (!source.WordStartParagraphIndex.HasValue || !source.WordEndParagraphIndex.HasValue || source.WordStartParagraphIndex <= source.WordEndParagraphIndex); + + return new StorySceneTextBlock( + scene.SceneID, + sceneNumber, + start, + end, + sourceLabel, + sceneText, + splitValid + ? BuildSceneContextJson( + run, + scene.SceneID, + scene.SceneNumber, + FirstNonBlank(source.StructuralSummary, scene.StructuralSummary), + sourceLabel, + configuredMetrics, + existingPlotLines) + : string.Empty, + splitValid); + } + + private static StorySceneTextBlock BuildLegacySceneBlock( StoryIntelligenceQueuedRun run, IReadOnlyList paragraphs, ChapterSceneBoundary boundary, @@ -873,17 +1030,25 @@ public sealed class PersistedStoryIntelligenceRunner( if (sceneNumber <= 0 || start <= 0 || end <= 0 || start > end || end > paragraphs.Count) { - return new StorySceneTextBlock(sceneNumber, start, end, sourceLabel, string.Empty, string.Empty, false); + return new StorySceneTextBlock(null, sceneNumber, start, end, sourceLabel, string.Empty, string.Empty, false); } var sceneText = string.Join(Environment.NewLine + Environment.NewLine, paragraphs.Skip(start - 1).Take(end - start + 1)); return new StorySceneTextBlock( + null, sceneNumber, start, end, sourceLabel, sceneText, - BuildSceneContextJson(run, boundary, configuredMetrics, existingPlotLines), + BuildSceneContextJson( + run, + null, + boundary.SceneNumber, + boundary.StructuralSummary, + sourceLabel, + configuredMetrics, + existingPlotLines), true); } @@ -900,7 +1065,10 @@ public sealed class PersistedStoryIntelligenceRunner( private static string BuildSceneContextJson( StoryIntelligenceQueuedRun run, - ChapterSceneBoundary boundary, + int? sceneId, + decimal? sceneNumber, + string? structuralSummary, + string sourceLabel, IReadOnlyList configuredMetrics, IReadOnlyList existingPlotLines) => JsonSerializer.Serialize(new @@ -908,11 +1076,11 @@ public sealed class PersistedStoryIntelligenceRunner( projectId = run.ProjectID, bookId = run.BookID, chapterId = run.ChapterID, - sceneId = (int?)null, + sceneId, chapterNumber = ChapterNumber(run), - sceneNumber = boundary.SceneNumber, - structuralSummary = boundary.StructuralSummary, - sourceLabel = $"{SourceLabel(run)}, suggested scene {boundary.SceneNumber}", + sceneNumber, + structuralSummary, + sourceLabel, knownCharacters = KnownCharacters(run), configuredMetrics = configuredMetrics.Select(metric => new { @@ -934,6 +1102,17 @@ public sealed class PersistedStoryIntelligenceRunner( }) }, JsonOptions); + private static int SceneDisplayNumber(decimal sceneNumber, int ordinal) + => decimal.Truncate(sceneNumber) == sceneNumber && sceneNumber > 0 && sceneNumber <= int.MaxValue + ? Convert.ToInt32(sceneNumber) + : ordinal; + + private static string? FirstNonBlank(params string?[] values) + => values.FirstOrDefault(value => !string.IsNullOrWhiteSpace(value))?.Trim(); + + private static string SceneProgressLabel(StorySceneTextBlock block) + => $"scene {block.TemporarySceneNumber:N0}"; + private static string MetricContextKey(SceneMetricType metric) => $"metric_{metric.MetricTypeID}"; @@ -1177,6 +1356,7 @@ public sealed class PersistedStoryIntelligenceRunner( } private sealed record StorySceneTextBlock( + int? SceneID, int TemporarySceneNumber, int StartParagraph, int EndParagraph, @@ -1185,6 +1365,14 @@ public sealed class PersistedStoryIntelligenceRunner( string SceneContextJson, bool SplitValid); + private sealed record SceneAnalysisModeSelection( + string Mode, + IReadOnlyList SceneBlocks, + int? ChapterResultId, + bool HasWarnings, + string PreparationMessage, + string PreparedEventType); + private sealed class TokenTotals { public int? InputTokens { get; private set; } diff --git a/PlotLine/Services/StoryIntelligenceAssetImportService.cs b/PlotLine/Services/StoryIntelligenceAssetImportService.cs index 1869b13..3c45355 100644 --- a/PlotLine/Services/StoryIntelligenceAssetImportService.cs +++ b/PlotLine/Services/StoryIntelligenceAssetImportService.cs @@ -320,6 +320,7 @@ public sealed class StoryIntelligenceAssetImportService( var importedScenes = (await scenes.ListByChapterAsync(item.ChapterID)) .Where(scene => scene.ImportRunID == item.RunID) .ToList(); + var importedById = importedScenes.ToDictionary(scene => scene.SceneID, scene => scene); var importedByRange = importedScenes .Where(scene => scene.SourceStartParagraph.HasValue && scene.SourceEndParagraph.HasValue) .ToDictionary(scene => $"{scene.SourceStartParagraph}-{scene.SourceEndParagraph}", scene => scene, StringComparer.OrdinalIgnoreCase); @@ -334,7 +335,7 @@ public sealed class StoryIntelligenceAssetImportService( continue; } - var importedScene = ResolveImportedScene(sceneResult, importedByRange, importedByNumber); + var importedScene = ResolveImportedScene(sceneResult, importedById, importedByRange, importedByNumber); if (importedScene is null) { continue; @@ -492,9 +493,15 @@ public sealed class StoryIntelligenceAssetImportService( private static Scene? ResolveImportedScene( StoryIntelligenceSavedSceneResult sceneResult, + IReadOnlyDictionary byId, IReadOnlyDictionary byRange, IReadOnlyDictionary byNumber) { + if (sceneResult.SceneID.HasValue && byId.TryGetValue(sceneResult.SceneID.Value, out var idMatch)) + { + return idMatch; + } + if (sceneResult.StartParagraph.HasValue && sceneResult.EndParagraph.HasValue && byRange.TryGetValue($"{sceneResult.StartParagraph}-{sceneResult.EndParagraph}", out var rangeMatch)) diff --git a/PlotLine/Services/StoryIntelligenceCharacterImportService.cs b/PlotLine/Services/StoryIntelligenceCharacterImportService.cs index bdac3f1..a1b0e21 100644 --- a/PlotLine/Services/StoryIntelligenceCharacterImportService.cs +++ b/PlotLine/Services/StoryIntelligenceCharacterImportService.cs @@ -394,6 +394,7 @@ public sealed class StoryIntelligenceCharacterImportService( var importedScenes = (await scenes.ListByChapterAsync(item.ChapterID)) .Where(scene => scene.ImportRunID == item.RunID) .ToList(); + var importedById = importedScenes.ToDictionary(scene => scene.SceneID, scene => scene); var importedByRange = importedScenes .Where(scene => scene.SourceStartParagraph.HasValue && scene.SourceEndParagraph.HasValue) .ToDictionary(scene => $"{scene.SourceStartParagraph}-{scene.SourceEndParagraph}", scene => scene, StringComparer.OrdinalIgnoreCase); @@ -408,7 +409,7 @@ public sealed class StoryIntelligenceCharacterImportService( continue; } - var importedScene = ResolveImportedScene(sceneResult, importedByRange, importedByNumber); + var importedScene = ResolveImportedScene(sceneResult, importedById, importedByRange, importedByNumber); if (importedScene is null) { continue; @@ -645,9 +646,15 @@ public sealed class StoryIntelligenceCharacterImportService( private static Scene? ResolveImportedScene( StoryIntelligenceSavedSceneResult sceneResult, + IReadOnlyDictionary byId, IReadOnlyDictionary byRange, IReadOnlyDictionary byNumber) { + if (sceneResult.SceneID.HasValue && byId.TryGetValue(sceneResult.SceneID.Value, out var idMatch)) + { + return idMatch; + } + if (sceneResult.StartParagraph.HasValue && sceneResult.EndParagraph.HasValue && byRange.TryGetValue($"{sceneResult.StartParagraph}-{sceneResult.EndParagraph}", out var rangeMatch)) diff --git a/PlotLine/Services/StoryIntelligenceImportCommitService.cs b/PlotLine/Services/StoryIntelligenceImportCommitService.cs index ac8fdb5..f273215 100644 --- a/PlotLine/Services/StoryIntelligenceImportCommitService.cs +++ b/PlotLine/Services/StoryIntelligenceImportCommitService.cs @@ -258,6 +258,7 @@ public sealed class StoryIntelligenceImportCommitService( return new StoryIntelligenceSceneImportItem { SceneResultID = sceneResult.SceneResultID, + SceneID = sceneResult.SceneID, TemporarySceneNumber = sceneResult.TemporarySceneNumber, StartParagraph = sceneResult.StartParagraph, EndParagraph = sceneResult.EndParagraph, @@ -313,6 +314,7 @@ public sealed class StoryIntelligenceImportCommitService( return new StoryIntelligenceSceneImportItem { SceneResultID = sceneResult.SceneResultID, + SceneID = sceneResult.SceneID, TemporarySceneNumber = sceneResult.TemporarySceneNumber, StartParagraph = sceneResult.StartParagraph, EndParagraph = sceneResult.EndParagraph, diff --git a/PlotLine/Services/StoryIntelligenceKnowledgeImportService.cs b/PlotLine/Services/StoryIntelligenceKnowledgeImportService.cs index bbb2daf..0fe38bf 100644 --- a/PlotLine/Services/StoryIntelligenceKnowledgeImportService.cs +++ b/PlotLine/Services/StoryIntelligenceKnowledgeImportService.cs @@ -237,6 +237,7 @@ public sealed class StoryIntelligenceKnowledgeImportService( var importedScenes = (await scenes.ListByChapterAsync(item.ChapterID)) .Where(scene => scene.ImportRunID == item.RunID) .ToList(); + var importedById = importedScenes.ToDictionary(scene => scene.SceneID, scene => scene); var importedByRange = importedScenes .Where(scene => scene.SourceStartParagraph.HasValue && scene.SourceEndParagraph.HasValue) .ToDictionary(scene => $"{scene.SourceStartParagraph}-{scene.SourceEndParagraph}", scene => scene, StringComparer.OrdinalIgnoreCase); @@ -251,7 +252,7 @@ public sealed class StoryIntelligenceKnowledgeImportService( continue; } - var importedScene = ResolveImportedScene(sceneResult, importedByRange, importedByNumber); + var importedScene = ResolveImportedScene(sceneResult, importedById, importedByRange, importedByNumber); if (importedScene is null) { continue; @@ -418,9 +419,15 @@ public sealed class StoryIntelligenceKnowledgeImportService( private static Scene? ResolveImportedScene( StoryIntelligenceSavedSceneResult sceneResult, + IReadOnlyDictionary byId, IReadOnlyDictionary byRange, IReadOnlyDictionary byNumber) { + if (sceneResult.SceneID.HasValue && byId.TryGetValue(sceneResult.SceneID.Value, out var idMatch)) + { + return idMatch; + } + if (sceneResult.StartParagraph.HasValue && sceneResult.EndParagraph.HasValue && byRange.TryGetValue($"{sceneResult.StartParagraph}-{sceneResult.EndParagraph}", out var rangeMatch)) diff --git a/PlotLine/Services/StoryIntelligenceLocationImportService.cs b/PlotLine/Services/StoryIntelligenceLocationImportService.cs index fe88899..232c731 100644 --- a/PlotLine/Services/StoryIntelligenceLocationImportService.cs +++ b/PlotLine/Services/StoryIntelligenceLocationImportService.cs @@ -276,6 +276,7 @@ public sealed class StoryIntelligenceLocationImportService( var importedScenes = (await scenes.ListByChapterAsync(item.ChapterID)) .Where(scene => scene.ImportRunID == item.RunID) .ToList(); + var importedById = importedScenes.ToDictionary(scene => scene.SceneID, scene => scene); var importedByRange = importedScenes .Where(scene => scene.SourceStartParagraph.HasValue && scene.SourceEndParagraph.HasValue) .ToDictionary(scene => $"{scene.SourceStartParagraph}-{scene.SourceEndParagraph}", scene => scene, StringComparer.OrdinalIgnoreCase); @@ -290,7 +291,7 @@ public sealed class StoryIntelligenceLocationImportService( continue; } - var importedScene = ResolveImportedScene(sceneResult, importedByRange, importedByNumber); + var importedScene = ResolveImportedScene(sceneResult, importedById, importedByRange, importedByNumber); if (importedScene is null) { continue; @@ -434,9 +435,15 @@ public sealed class StoryIntelligenceLocationImportService( private static Scene? ResolveImportedScene( StoryIntelligenceSavedSceneResult sceneResult, + IReadOnlyDictionary byId, IReadOnlyDictionary byRange, IReadOnlyDictionary byNumber) { + if (sceneResult.SceneID.HasValue && byId.TryGetValue(sceneResult.SceneID.Value, out var idMatch)) + { + return idMatch; + } + if (sceneResult.StartParagraph.HasValue && sceneResult.EndParagraph.HasValue && byRange.TryGetValue($"{sceneResult.StartParagraph}-{sceneResult.EndParagraph}", out var rangeMatch)) diff --git a/PlotLine/Services/StoryIntelligenceNarrativeReviewService.cs b/PlotLine/Services/StoryIntelligenceNarrativeReviewService.cs index 678e03b..1c3059d 100644 --- a/PlotLine/Services/StoryIntelligenceNarrativeReviewService.cs +++ b/PlotLine/Services/StoryIntelligenceNarrativeReviewService.cs @@ -206,6 +206,7 @@ public sealed class StoryIntelligenceNarrativeReviewService( var importedScenes = (await scenes.ListByChapterAsync(item.ChapterID)) .Where(scene => scene.ImportRunID == item.RunID) .ToList(); + var importedById = importedScenes.ToDictionary(scene => scene.SceneID, scene => scene); var importedByRange = importedScenes .Where(scene => scene.SourceStartParagraph.HasValue && scene.SourceEndParagraph.HasValue) .ToDictionary(scene => $"{scene.SourceStartParagraph}-{scene.SourceEndParagraph}", scene => scene, StringComparer.OrdinalIgnoreCase); @@ -214,7 +215,7 @@ public sealed class StoryIntelligenceNarrativeReviewService( foreach (var sceneResult in sceneResults) { - var importedScene = ResolveImportedScene(sceneResult, importedByRange, importedByNumber); + var importedScene = ResolveImportedScene(sceneResult, importedById, importedByRange, importedByNumber); var parsed = TryReadScene(sceneResult); if (importedScene is null || parsed is null) { @@ -337,9 +338,15 @@ public sealed class StoryIntelligenceNarrativeReviewService( private static Scene? ResolveImportedScene( StoryIntelligenceSavedSceneResult sceneResult, + IReadOnlyDictionary importedById, IReadOnlyDictionary importedByRange, IReadOnlyDictionary importedByNumber) { + if (sceneResult.SceneID.HasValue && importedById.TryGetValue(sceneResult.SceneID.Value, out var byId)) + { + return byId; + } + if (sceneResult.StartParagraph.HasValue && sceneResult.EndParagraph.HasValue && importedByRange.TryGetValue($"{sceneResult.StartParagraph}-{sceneResult.EndParagraph}", out var byRange)) diff --git a/PlotLine/Services/StoryIntelligenceRelationshipImportService.cs b/PlotLine/Services/StoryIntelligenceRelationshipImportService.cs index c244bef..5a3edd9 100644 --- a/PlotLine/Services/StoryIntelligenceRelationshipImportService.cs +++ b/PlotLine/Services/StoryIntelligenceRelationshipImportService.cs @@ -270,6 +270,7 @@ public sealed class StoryIntelligenceRelationshipImportService( var importedScenes = (await scenes.ListByChapterAsync(item.ChapterID)) .Where(scene => scene.ImportRunID == item.RunID) .ToList(); + var importedById = importedScenes.ToDictionary(scene => scene.SceneID, scene => scene); var importedByRange = importedScenes .Where(scene => scene.SourceStartParagraph.HasValue && scene.SourceEndParagraph.HasValue) .ToDictionary(scene => $"{scene.SourceStartParagraph}-{scene.SourceEndParagraph}", scene => scene, StringComparer.OrdinalIgnoreCase); @@ -284,7 +285,7 @@ public sealed class StoryIntelligenceRelationshipImportService( continue; } - var importedScene = ResolveImportedScene(sceneResult, importedByRange, importedByNumber); + var importedScene = ResolveImportedScene(sceneResult, importedById, importedByRange, importedByNumber); if (importedScene is null) { continue; @@ -415,9 +416,15 @@ public sealed class StoryIntelligenceRelationshipImportService( private static Scene? ResolveImportedScene( StoryIntelligenceSavedSceneResult sceneResult, + IReadOnlyDictionary byId, IReadOnlyDictionary byRange, IReadOnlyDictionary byNumber) { + if (sceneResult.SceneID.HasValue && byId.TryGetValue(sceneResult.SceneID.Value, out var idMatch)) + { + return idMatch; + } + if (sceneResult.StartParagraph.HasValue && sceneResult.EndParagraph.HasValue && byRange.TryGetValue($"{sceneResult.StartParagraph}-{sceneResult.EndParagraph}", out var rangeMatch)) diff --git a/PlotLine/Sql/161_Phase25F_CanonicalSceneSourceStoryIntelligence.sql b/PlotLine/Sql/161_Phase25F_CanonicalSceneSourceStoryIntelligence.sql new file mode 100644 index 0000000..1c67c9b --- /dev/null +++ b/PlotLine/Sql/161_Phase25F_CanonicalSceneSourceStoryIntelligence.sql @@ -0,0 +1,41 @@ +SET ANSI_NULLS ON; +GO +SET QUOTED_IDENTIFIER ON; +GO + +CREATE OR ALTER PROCEDURE dbo.SceneManuscriptSource_ListByChapter + @ChapterID int +AS +BEGIN + SET NOCOUNT ON; + + SELECT source.SceneID, source.ProjectID, source.BookID, source.ChapterID, source.SourceText, + source.SourceWordCount, source.SourceCharacterCount, source.SourceStartParagraph, source.SourceEndParagraph, + source.WordStartParagraphIndex, source.WordEndParagraphIndex, source.SourceKind, scene.StructuralSummary, + source.CreatedUtc, source.UpdatedUtc + FROM dbo.SceneManuscriptSources source + INNER JOIN dbo.Scenes scene ON scene.SceneID = source.SceneID + WHERE source.ChapterID = @ChapterID + AND scene.IsArchived = 0 + ORDER BY scene.SortOrder, scene.SceneNumber, scene.SceneID; +END; +GO + +CREATE OR ALTER PROCEDURE dbo.SceneManuscriptSource_ListByBook + @BookID int +AS +BEGIN + SET NOCOUNT ON; + + SELECT source.SceneID, source.ProjectID, source.BookID, source.ChapterID, source.SourceText, + source.SourceWordCount, source.SourceCharacterCount, source.SourceStartParagraph, source.SourceEndParagraph, + source.WordStartParagraphIndex, source.WordEndParagraphIndex, source.SourceKind, scene.StructuralSummary, + source.CreatedUtc, source.UpdatedUtc + FROM dbo.SceneManuscriptSources source + INNER JOIN dbo.Scenes scene ON scene.SceneID = source.SceneID + INNER JOIN dbo.Chapters chapter ON chapter.ChapterID = scene.ChapterID + WHERE source.BookID = @BookID + AND scene.IsArchived = 0 + ORDER BY chapter.SortOrder, chapter.ChapterNumber, scene.SortOrder, scene.SceneNumber, scene.SceneID; +END; +GO diff --git a/PlotLine/Views/Onboarding/StoryIntelligenceProgress.cshtml b/PlotLine/Views/Onboarding/StoryIntelligenceProgress.cshtml index 3859e00..051135a 100644 --- a/PlotLine/Views/Onboarding/StoryIntelligenceProgress.cshtml +++ b/PlotLine/Views/Onboarding/StoryIntelligenceProgress.cshtml @@ -190,7 +190,7 @@ return stage switch { "Queued" => "Waiting", - StoryIntelligenceFailureStages.ChapterStructure => "Finding scenes", + StoryIntelligenceFailureStages.ChapterStructure => "Legacy boundary detection", StoryIntelligenceFailureStages.SceneIntelligence => "Reading scenes", StoryIntelligenceFailureStages.SceneSplit => "Preparing review", StoryIntelligenceFailureStages.Validation => "Preparing review", @@ -219,7 +219,7 @@ return chapter.CurrentStage switch { "Queued" => "Waiting to read this chapter.", - StoryIntelligenceFailureStages.ChapterStructure => "Finding scenes in this chapter.", + StoryIntelligenceFailureStages.ChapterStructure => "Running legacy scene-boundary detection for this chapter.", StoryIntelligenceFailureStages.SceneSplit => "Preparing the detected scenes.", StoryIntelligenceFailureStages.SceneIntelligence => "Reading the detected scenes.", StoryIntelligenceFailureStages.Validation => "Checking the results.", @@ -343,7 +343,7 @@ { if ((chapter?.TotalDetectedScenes ?? 0) <= 0) { - return "Finding scenes..."; + return "Loading scenes..."; } var completed = Math.Clamp(chapter?.CompletedScenes ?? 0, 0, chapter?.TotalDetectedScenes ?? 0); diff --git a/PlotLine/wwwroot/js/story-intelligence-progress.js b/PlotLine/wwwroot/js/story-intelligence-progress.js index 90ee363..9f56dfe 100644 --- a/PlotLine/wwwroot/js/story-intelligence-progress.js +++ b/PlotLine/wwwroot/js/story-intelligence-progress.js @@ -56,7 +56,7 @@ case "DocumentRead": return "Preparing chapter"; case "ChapterStructure": - return "Finding scenes"; + return "Legacy boundary detection"; case "SceneSplit": return "Preparing scenes"; case "SceneIntelligence": @@ -148,7 +148,7 @@ case "DocumentRead": return "Preparing this chapter."; case "ChapterStructure": - return "Finding scenes in this chapter."; + return "Running legacy scene-boundary detection for this chapter."; case "SceneSplit": return "Preparing the detected scenes."; case "SceneIntelligence": @@ -303,7 +303,7 @@ root.querySelectorAll("[data-story-chapter-progress-text]").forEach((node) => { node.textContent = totalNumber > 0 ? `Scene ${numberText(Math.min(Math.max(currentSceneDisplay, 0), totalNumber))} of ${numberText(totalNumber)}` - : "Finding scenes..."; + : "Loading scenes..."; }); root.querySelectorAll("[data-story-current-message]").forEach((node) => { node.textContent = message;