diff --git a/PlotLine.Tests/Program.cs b/PlotLine.Tests/Program.cs index 4af3462..6c50ef2 100644 --- a/PlotLine.Tests/Program.cs +++ b/PlotLine.Tests/Program.cs @@ -64,6 +64,9 @@ var tests = new (string Name, Action Test)[] ("Story Intelligence progress does not prematurely report safe close", StoryIntelligenceProgressDoesNotPrematurelyReportSafeClose), ("Structural import creates scenes before Word sync", StructuralImportCreatesScenesBeforeWordSync), ("Structural import persists paragraph mapping and source text", StructuralImportPersistsParagraphMappingAndSourceText), + ("Structural import persists canonical scene source text", StructuralImportPersistsCanonicalSceneSourceText), + ("Core import wizard orders scenes before characters", CoreImportWizardOrdersScenesBeforeCharacters), + ("Story Intelligence uses persisted scene source after restart", StoryIntelligenceUsesPersistedSceneSourceAfterRestart), ("Word sync requires save acknowledgement", WordSyncRequiresSaveAcknowledgement), ("Story Intelligence enriches existing structural scenes", StoryIntelligenceEnrichesExistingStructuralScenes), ("Phase 21T repairs durable character understanding", Phase21TRepairsDurableCharacterUnderstanding), @@ -1361,6 +1364,55 @@ static void StructuralImportPersistsParagraphMappingAndSourceText() Assert(onboarding.Contains("SaveStructuralChapterSourcesAsync", StringComparison.Ordinal), "Onboarding import should save source text after canonical ChapterIDs exist."); } +static void StructuralImportPersistsCanonicalSceneSourceText() +{ + var root = Path.Combine(AppContext.BaseDirectory, "../../../../PlotLine"); + var migration = File.ReadAllText(Path.Combine(root, "Sql/150_Phase24B_SceneManuscriptSources.sql")); + var onboarding = File.ReadAllText(Path.Combine(root, "Services/OnboardingService.cs")); + var analyzer = File.ReadAllText(Path.Combine(root, "Services/ManuscriptStructureAnalysisService.cs")); + + Assert(migration.Contains("CREATE TABLE dbo.SceneManuscriptSources", StringComparison.Ordinal), "Scene manuscript source text should be stored in a canonical SceneID table."); + Assert(migration.Contains("SceneID int NOT NULL CONSTRAINT PK_SceneManuscriptSources PRIMARY KEY", StringComparison.Ordinal), "Scene source text should be keyed by SceneID."); + Assert(migration.Contains("IX_SceneManuscriptSources_BookChapter", StringComparison.Ordinal), "Scene source text should be indexed for book/chapter retrieval."); + Assert(onboarding.Contains("SaveSceneManuscriptSourcesAsync", StringComparison.Ordinal), "Structural import should persist scene source text before Word sync."); + Assert(onboarding.IndexOf("await SaveSceneManuscriptSourcesAsync(preview, structuralScenes, result)", StringComparison.Ordinal) + < onboarding.IndexOf("await PublishBuildProgress(userId, preview.PreviewID, \"Updating and saving Word markers", StringComparison.Ordinal), "Scene source text should be persisted before marker sync can complete."); + Assert(analyzer.Contains("string.Join(Environment.NewLine + Environment.NewLine", StringComparison.Ordinal), "Scene source text should preserve paragraph separation."); + Assert(analyzer.Contains("item.IncludeInSourceText", StringComparison.Ordinal), "Scene source text should exclude non-prose structural paragraphs such as hidden markers/separators."); +} + +static void CoreImportWizardOrdersScenesBeforeCharacters() +{ + var root = Path.Combine(AppContext.BaseDirectory, "../../../../PlotLine"); + var controller = File.ReadAllText(Path.Combine(root, "Controllers/OnboardingController.cs")); + var review = File.ReadAllText(Path.Combine(root, "Views/Onboarding/ScanReview.cshtml")); + var complete = File.ReadAllText(Path.Combine(root, "Views/Onboarding/BuildComplete.cshtml")); + var onboarding = File.ReadAllText(Path.Combine(root, "Services/OnboardingService.cs")); + + Assert(review.Contains("value=\"structure\"", StringComparison.Ordinal), "Chapter review should submit to structural import, not character review."); + Assert(controller.Contains("var readyToImport = string.Equals(intent, \"structure\"", StringComparison.Ordinal), "Controller should treat structure intent as the scene-creation gate."); + Assert(controller.Contains("FinaliseCharacterReviewAsync", StringComparison.Ordinal), "Character review should have a distinct post-structure finalisation path."); + Assert(onboarding.Contains("Import the approved chapters and scenes before reviewing characters.", StringComparison.Ordinal), "Character finalisation should refuse to run before structural scenes exist."); + Assert(complete.Contains("Review characters", StringComparison.Ordinal), "After Word sync, the next onboarding step should be character review."); + Assert(complete.Contains("Word may now be closed; keep this browser workflow open", StringComparison.Ordinal), "Word safe-close must be distinct from browser/core-import completion."); +} + +static void StoryIntelligenceUsesPersistedSceneSourceAfterRestart() +{ + var root = Path.Combine(AppContext.BaseDirectory, "../../../../PlotLine"); + var migration = File.ReadAllText(Path.Combine(root, "Sql/150_Phase24B_SceneManuscriptSources.sql")); + var sourceRepository = File.ReadAllText(Path.Combine(root, "Data/StoryIntelligenceSourceRepository.cs")); + var onboardingStory = File.ReadAllText(Path.Combine(root, "Services/OnboardingStoryIntelligenceService.cs")); + var existingQueue = File.ReadAllText(Path.Combine(root, "Services/StoryIntelligenceExistingChapterQueueService.cs")); + + Assert(migration.Contains("INNER JOIN dbo.SceneManuscriptSources", StringComparison.Ordinal), "Chapter source SQL should aggregate persisted canonical scene source text."); + Assert(migration.Contains("STRING_AGG", StringComparison.Ordinal), "Chapter source SQL should reconstruct chapter text from persisted scene text in order."); + Assert(sourceRepository.Contains("StoryIntelligenceSource_ChapterText_GetAdmin", StringComparison.Ordinal), "Story Intelligence source repository should use the stored source procedure."); + Assert(onboardingStory.Contains("durableSource = await sources.GetChapterSourceAsync", StringComparison.Ordinal), "Onboarding Story Intelligence should read durable chapter text from scene sources."); + Assert(onboardingStory.Contains("CoreImportComplete", StringComparison.Ordinal), "Optional onboarding Story Intelligence should require core import completion."); + Assert(existingQueue.Contains("sources.GetChapterSourceAsync", StringComparison.Ordinal), "Later Story Intelligence queueing should start from persisted chapter/scene source, not onboarding DTOs."); +} + static void WordSyncRequiresSaveAcknowledgement() { var root = Path.Combine(AppContext.BaseDirectory, "../../../../PlotLine"); diff --git a/PlotLine/Controllers/OnboardingController.cs b/PlotLine/Controllers/OnboardingController.cs index 17e0974..57cec69 100644 --- a/PlotLine/Controllers/OnboardingController.cs +++ b/PlotLine/Controllers/OnboardingController.cs @@ -526,10 +526,23 @@ public sealed class OnboardingController(IOnboardingService onboarding, IOnboard var continueToCharacters = string.Equals(intent, "characters", StringComparison.OrdinalIgnoreCase); var continueCharacterReview = string.Equals(intent, "character-next", StringComparison.OrdinalIgnoreCase); var backCharacterReview = string.Equals(intent, "character-back", StringComparison.OrdinalIgnoreCase); - var readyToImport = string.Equals(intent, "continue", StringComparison.OrdinalIgnoreCase); + var readyToImport = string.Equals(intent, "structure", StringComparison.OrdinalIgnoreCase); + var completeCoreImport = string.Equals(intent, "continue", StringComparison.OrdinalIgnoreCase); var nextCharacterPage = Math.Max(1, characterPage + (backCharacterReview ? -1 : 1)); try { + if (completeCoreImport) + { + var result = await onboarding.FinaliseCharacterReviewAsync(form); + if (result is null) + { + return NotFound(); + } + + TempData["OnboardingReviewMessage"] = "Characters saved. Core manuscript import is complete."; + return RedirectToAction(nameof(BuildComplete), new { previewId = form.PreviewID }); + } + var model = await onboarding.SaveScanReviewAsync(form, readyToImport); if (model is null) { @@ -537,12 +550,12 @@ public sealed class OnboardingController(IOnboardingService onboarding, IOnboard } TempData["OnboardingReviewMessage"] = readyToImport - ? "Review saved. Next: story and portrait context." + ? "Chapter review saved. Next: import scenes and update Word." : continueToCharacters ? "Chapter and scene review saved. Next: review detected characters." : continueCharacterReview || backCharacterReview ? "Character review choices saved." : "Review choices saved."; return readyToImport - ? RedirectToAction(nameof(StoryIntelligence)) + ? RedirectToAction(nameof(Index)) : continueCharacterReview || backCharacterReview ? RedirectToAction(nameof(ScanCharacters), new { previewId = form.PreviewID, page = nextCharacterPage }) : continueToCharacters ? RedirectToAction(nameof(ScanCharacters), new { previewId = form.PreviewID }) : RedirectToAction(nameof(ScanReview), new { previewId = form.PreviewID }); diff --git a/PlotLine/Data/OnboardingBuildRepository.cs b/PlotLine/Data/OnboardingBuildRepository.cs index 1442094..f1339c8 100644 --- a/PlotLine/Data/OnboardingBuildRepository.cs +++ b/PlotLine/Data/OnboardingBuildRepository.cs @@ -12,6 +12,7 @@ public interface IOnboardingBuildRepository Task MarkMarkerSyncCompletedAsync(int buildId); Task MarkMarkerSyncFailedAsync(int buildId, string message); Task SaveStructuralChapterSourceAsync(StructuralChapterSourceSaveRequest request); + Task SaveSceneManuscriptSourceAsync(SceneManuscriptSourceSaveRequest request); } public sealed class OnboardingBuildRepository(ISqlConnectionFactory connectionFactory) : IOnboardingBuildRepository @@ -110,6 +111,29 @@ public sealed class OnboardingBuildRepository(ISqlConnectionFactory connectionFa }, commandType: CommandType.StoredProcedure); } + + public async Task SaveSceneManuscriptSourceAsync(SceneManuscriptSourceSaveRequest request) + { + using var connection = connectionFactory.CreateConnection(); + await connection.ExecuteAsync( + "dbo.SceneManuscriptSource_Save", + new + { + request.SceneID, + request.ProjectID, + request.BookID, + request.ChapterID, + request.SourceText, + request.SourceWordCount, + request.SourceCharacterCount, + request.SourceStartParagraph, + request.SourceEndParagraph, + request.WordStartParagraphIndex, + request.WordEndParagraphIndex, + request.SourceKind + }, + commandType: CommandType.StoredProcedure); + } } public sealed class OnboardingManuscriptBuildRequest @@ -157,3 +181,19 @@ public sealed class StructuralChapterSourceSaveRequest public int SourceCharacterCount { get; init; } public string? ParagraphMapJson { get; init; } } + +public sealed class SceneManuscriptSourceSaveRequest +{ + public int SceneID { get; init; } + public int ProjectID { get; init; } + public int BookID { get; init; } + public int ChapterID { get; init; } + public string SourceText { get; init; } = string.Empty; + public int SourceWordCount { get; init; } + public int SourceCharacterCount { get; init; } + public int? SourceStartParagraph { get; init; } + public int? SourceEndParagraph { get; init; } + public int? WordStartParagraphIndex { get; init; } + public int? WordEndParagraphIndex { get; init; } + public string SourceKind { get; init; } = "StructuralImport"; +} diff --git a/PlotLine/Models/ManuscriptScanModels.cs b/PlotLine/Models/ManuscriptScanModels.cs index b97c209..c31244d 100644 --- a/PlotLine/Models/ManuscriptScanModels.cs +++ b/PlotLine/Models/ManuscriptScanModels.cs @@ -19,6 +19,7 @@ public static class ManuscriptScanReviewStatuses public const string ReviewInProgress = "ReviewInProgress"; public const string ReviewComplete = "ReviewComplete"; public const string ReadyToImport = "ReadyToImport"; + public const string CoreImportComplete = "CoreImportComplete"; } public sealed class ManuscriptScanCommand @@ -79,6 +80,7 @@ public sealed class ManuscriptScanChapterPreview public int? StartPosition { get; init; } public int? ExistingChapterID { get; init; } public IReadOnlyList ParagraphMap { get; init; } = []; + public IReadOnlyList SourceParagraphs { get; init; } = []; } public sealed class ManuscriptScanParagraphMap @@ -88,6 +90,17 @@ public sealed class ManuscriptScanParagraphMap public string Text { get; init; } = string.Empty; } +public sealed class ManuscriptScanSourceParagraph +{ + public int WordParagraphIndex { get; init; } + public string Text { get; init; } = string.Empty; + public bool IsBlank { get; init; } + public bool IsChapterHeading { get; init; } + public bool IsSceneHeading { get; init; } + public bool IsSceneSeparator { get; init; } + public bool IncludeInSourceText { get; init; } +} + public sealed class ManuscriptScanScenePreview { public string TemporarySceneKey { get; init; } = string.Empty; diff --git a/PlotLine/Services/ManuscriptStructureAnalysisService.cs b/PlotLine/Services/ManuscriptStructureAnalysisService.cs index 4ca89bf..41d27ac 100644 --- a/PlotLine/Services/ManuscriptStructureAnalysisService.cs +++ b/PlotLine/Services/ManuscriptStructureAnalysisService.cs @@ -72,7 +72,8 @@ public sealed class ManuscriptStructureAnalysisService( } var startMap = request.ParagraphMap.FirstOrDefault(item => item.StructuralParagraphOrdinal == start); - if (startMap is null || startMap.WordParagraphIndex < 0) + var endMap = request.ParagraphMap.FirstOrDefault(item => item.StructuralParagraphOrdinal == end); + if (startMap is null || endMap is null || startMap.WordParagraphIndex < 0 || endMap.WordParagraphIndex < 0) { return ManuscriptStructureAnalysisResult.Failed( request.TemporaryChapterKey, @@ -81,6 +82,7 @@ public sealed class ManuscriptStructureAnalysisService( } var sceneParagraphs = paragraphs.Skip(start - 1).Take(end - start + 1).ToList(); + var sourceText = BuildSourceText(request.SourceParagraphs, startMap.WordParagraphIndex, endMap.WordParagraphIndex); scenes.Add(new ManuscriptStructureSceneBoundary { TemporarySceneKey = $"{request.TemporaryChapterKey}-ai-scene-{sceneNumber}", @@ -88,9 +90,11 @@ public sealed class ManuscriptStructureAnalysisService( SceneNumberWithinChapter = sceneNumber, Title = $"Scene {sceneNumber:N0}", WordCount = sceneParagraphs.Sum(CountWords), + SourceText = sourceText, StartParagraph = start, EndParagraph = end, WordStartPosition = startMap.WordParagraphIndex, + WordEndPosition = endMap.WordParagraphIndex, Reason = boundary.Reason, Confidence = boundary.Confidence }); @@ -157,6 +161,29 @@ public sealed class ManuscriptStructureAnalysisService( return outputText.Trim(); } + private static string BuildSourceText(IReadOnlyList sourceParagraphs, int wordStartPosition, int wordEndPosition) + { + var lines = sourceParagraphs + .Where(item => item.WordParagraphIndex >= wordStartPosition + && item.WordParagraphIndex <= wordEndPosition + && item.IncludeInSourceText) + .OrderBy(item => item.WordParagraphIndex) + .Select(item => item.IsBlank ? string.Empty : item.Text.Trim()) + .ToList(); + + while (lines.Count > 0 && string.IsNullOrWhiteSpace(lines[0])) + { + lines.RemoveAt(0); + } + + while (lines.Count > 0 && string.IsNullOrWhiteSpace(lines[^1])) + { + lines.RemoveAt(lines.Count - 1); + } + + return string.Join(Environment.NewLine + Environment.NewLine, lines); + } + private static string BuildValidationMessage(ValidationResult validation) { var first = validation.Errors.FirstOrDefault()?.Message @@ -180,6 +207,7 @@ public sealed class ManuscriptStructureAnalysisRequest public string SourceLabel { get; init; } = string.Empty; public string ChapterText { get; init; } = string.Empty; public IReadOnlyList ParagraphMap { get; init; } = []; + public IReadOnlyList SourceParagraphs { get; init; } = []; } public sealed class ManuscriptStructureAnalysisResult @@ -204,9 +232,11 @@ public sealed class ManuscriptStructureSceneBoundary public int SceneNumberWithinChapter { get; init; } public string Title { get; init; } = string.Empty; public int WordCount { get; init; } + public string SourceText { get; init; } = string.Empty; public int StartParagraph { get; init; } public int EndParagraph { get; init; } public int WordStartPosition { get; init; } + public int WordEndPosition { get; init; } public string? Reason { get; init; } public decimal? Confidence { get; init; } } diff --git a/PlotLine/Services/OnboardingService.cs b/PlotLine/Services/OnboardingService.cs index f15804b..8c9b6a2 100644 --- a/PlotLine/Services/OnboardingService.cs +++ b/PlotLine/Services/OnboardingService.cs @@ -11,6 +11,7 @@ public interface IOnboardingService Task GetWizardAsync(); Task GetScanReviewAsync(Guid? previewId = null); Task SaveScanReviewAsync(ManuscriptScanReviewForm form, bool readyToImport); + Task FinaliseCharacterReviewAsync(ManuscriptScanReviewForm form); Task BuildApprovedStructureAsync(Guid previewId, Func? progress = null); Task GetBuildResultAsync(Guid previewId); Task ShouldShowOnboardingAsync(); @@ -262,7 +263,8 @@ public sealed class OnboardingService( ChapterNumber = chapterDecision.ChapterNumber, SourceLabel = $"{preview.DocumentTitle ?? "Word manuscript"}: {chapterDecision.Title}", ChapterText = chapterPreview.ChapterText, - ParagraphMap = chapterPreview.ParagraphMap + ParagraphMap = chapterPreview.ParagraphMap, + SourceParagraphs = chapterPreview.SourceParagraphs }); if (!analysis.Success) @@ -274,11 +276,6 @@ public sealed class OnboardingService( } await PublishBuildProgress(userId, preview.PreviewID, "Preparing chapters and scenes for import...", 45, progress); - var characterDecisions = review.Characters - .Where(IsIncludedCanonicalCharacter) - .OrderBy(character => character.Name) - .ThenBy(character => character.TemporaryCharacterKey) - .ToList(); var request = new OnboardingManuscriptBuildRequest { PreviewID = preview.PreviewID, @@ -300,12 +297,7 @@ public sealed class OnboardingService( SortOrder = scene.SceneNumberWithinChapter * 10, WordCount = scene.WordCount }).ToList(), - Characters = characterDecisions.Select(character => new OnboardingBuildCharacterRequest - { - TemporaryCharacterKey = character.TemporaryCharacterKey, - Name = character.Name, - ExistingCharacterID = character.ExistingCharacterID - }).ToList() + Characters = [] }; await PublishBuildProgress(userId, preview.PreviewID, "Creating PlotDirector structure...", 60, progress); @@ -335,7 +327,7 @@ public sealed class OnboardingService( MarkerWarning = result.MarkerWarning }; await SaveStructuralChapterSourcesAsync(preview, result); - await AddScanCharacterAliasesAsync(review, result); + await SaveSceneManuscriptSourcesAsync(preview, structuralScenes, result); await scanPreviews.SaveBuildResultAsync(userId, result); await PublishBuildProgress(userId, preview.PreviewID, "Updating and saving Word markers. Keep Word and this browser page open.", 90, progress); return result; @@ -344,6 +336,113 @@ public sealed class OnboardingService( public async Task GetBuildResultAsync(Guid previewId) => await scanPreviews.GetBuildResultAsync(RequireUserId(), previewId); + public async Task FinaliseCharacterReviewAsync(ManuscriptScanReviewForm form) + { + var context = await GetScanReviewContextAsync(form.PreviewID); + if (context is null) + { + return null; + } + + var (userId, preview, review, _, _, existingCharacters) = context.Value; + var build = await scanPreviews.GetBuildResultAsync(userId, preview.PreviewID); + if (build is null || build.SceneMappings.Count == 0) + { + throw new InvalidOperationException("Import the approved chapters and scenes before reviewing characters."); + } + + var candidateLookup = preview.CharacterCandidates.ToDictionary(candidate => candidate.TemporaryCharacterKey, StringComparer.Ordinal); + var existingCharacterIds = existingCharacters.Select(character => character.CharacterID).ToHashSet(); + var characterDecisions = form.Characters + .Where(item => candidateLookup.ContainsKey(item.TemporaryCharacterKey)) + .Select(item => + { + var cleanName = Clean(item.Name); + var aliasTargetKey = CleanOptional(item.AliasTargetTemporaryCharacterKey); + if (!string.IsNullOrWhiteSpace(aliasTargetKey) + && (!candidateLookup.ContainsKey(aliasTargetKey) || string.Equals(aliasTargetKey, item.TemporaryCharacterKey, StringComparison.Ordinal))) + { + aliasTargetKey = null; + } + + var matchedExistingId = string.IsNullOrWhiteSpace(aliasTargetKey) && item.ExistingCharacterID.HasValue && existingCharacterIds.Contains(item.ExistingCharacterID.Value) + ? item.ExistingCharacterID + : string.IsNullOrWhiteSpace(aliasTargetKey) ? MatchExistingCharacter(cleanName, existingCharacters)?.CharacterID : null; + return new ManuscriptScanCharacterReviewDecision + { + TemporaryCharacterKey = item.TemporaryCharacterKey, + Include = item.Include, + Name = cleanName, + Category = item.Category, + ExistingCharacterID = matchedExistingId, + AliasTargetTemporaryCharacterKey = aliasTargetKey, + SexHint = CharacterSexHintService.Normalise(item.SexHint, cleanName) + }; + }) + .ToList(); + + var characterMappings = new List(); + foreach (var decision in characterDecisions.Where(IsIncludedCanonicalCharacter).OrderBy(item => item.Name).ThenBy(item => item.TemporaryCharacterKey)) + { + var characterId = decision.ExistingCharacterID + ?? MatchExistingCharacter(decision.Name, existingCharacters)?.CharacterID + ?? await characters.SaveCharacterAsync(new Character + { + ProjectID = preview.ProjectID, + CharacterName = decision.Name, + Sex = string.IsNullOrWhiteSpace(decision.SexHint) ? null : decision.SexHint, + CharacterImportance = 8, + ShowInQuickAddBar = true + }); + + characterMappings.Add(new OnboardingBuildCharacterMap + { + TemporaryCharacterKey = decision.TemporaryCharacterKey, + CharacterID = characterId, + Created = !decision.ExistingCharacterID.HasValue + && existingCharacters.All(character => character.CharacterID != characterId) + }); + } + + var completedReview = new ManuscriptScanReviewDecision + { + PreviewID = review.PreviewID, + Status = ManuscriptScanReviewStatuses.CoreImportComplete, + Chapters = review.Chapters, + Scenes = review.Scenes, + Characters = characterDecisions + }; + + var completedResult = new OnboardingManuscriptBuildResult + { + BuildID = build.BuildID, + PreviewID = build.PreviewID, + ProjectID = build.ProjectID, + BookID = build.BookID, + ManuscriptDocumentID = build.ManuscriptDocumentID, + Status = "CoreImportComplete", + MarkerStatus = build.MarkerStatus, + ChaptersCreated = build.ChaptersCreated, + ScenesCreated = build.ScenesCreated, + CharactersCreated = characterMappings.Count(item => item.Created), + CharactersReused = characterMappings.Count(item => !item.Created), + SceneAppearancesCreated = build.SceneAppearancesCreated, + AlreadyBuilt = build.AlreadyBuilt, + Message = "Core manuscript import complete.", + ChapterMappings = build.ChapterMappings, + SceneMappings = build.SceneMappings, + CharacterMappings = characterMappings, + ChapterMarkerInstructions = build.ChapterMarkerInstructions, + SceneMarkerInstructions = build.SceneMarkerInstructions, + MarkerWarning = build.MarkerWarning + }; + + await scanPreviews.SaveReviewAsync(userId, completedReview); + await scanPreviews.SaveBuildResultAsync(userId, completedResult); + await AddScanCharacterAliasesAsync(completedReview, completedResult); + return completedResult; + } + private static IReadOnlyList BuildChapterMarkerInstructions( ManuscriptScanPreview preview, IReadOnlyList mappings) @@ -414,6 +513,43 @@ public sealed class OnboardingService( } } + private async Task SaveSceneManuscriptSourcesAsync( + ManuscriptScanPreview preview, + IReadOnlyList structuralScenes, + OnboardingManuscriptBuildResult result) + { + var sceneIds = result.SceneMappings.ToDictionary(item => item.TemporarySceneKey, item => item.SceneID, StringComparer.Ordinal); + var chapterIds = result.ChapterMappings.ToDictionary(item => item.TemporaryChapterKey, item => item.ChapterID, StringComparer.Ordinal); + foreach (var scene in structuralScenes) + { + if (!sceneIds.TryGetValue(scene.TemporarySceneKey, out var sceneId) + || !chapterIds.TryGetValue(scene.TemporaryChapterKey, out var chapterId)) + { + throw new InvalidOperationException("The imported scene source could not be matched to a canonical PlotDirector scene."); + } + + if (string.IsNullOrWhiteSpace(scene.SourceText)) + { + throw new InvalidOperationException($"Scene {scene.SceneNumberWithinChapter:N0} source text could not be persisted because the structural source text is empty."); + } + + await builds.SaveSceneManuscriptSourceAsync(new SceneManuscriptSourceSaveRequest + { + SceneID = sceneId, + ProjectID = preview.ProjectID, + BookID = preview.BookID, + ChapterID = chapterId, + SourceText = scene.SourceText, + SourceWordCount = CountWords(scene.SourceText), + SourceCharacterCount = scene.SourceText.Length, + SourceStartParagraph = scene.StartParagraph, + SourceEndParagraph = scene.EndParagraph, + WordStartParagraphIndex = scene.WordStartPosition, + WordEndParagraphIndex = scene.WordEndPosition + }); + } + } + private async Task PublishBuildProgress(int userId, Guid previewId, string message, int percent, Func? publish, OnboardingManuscriptBuildResult? result = null) { var state = await scanPreviews.BuildProgressAsync(userId, previewId, message, percent, result); diff --git a/PlotLine/Services/OnboardingStoryIntelligenceService.cs b/PlotLine/Services/OnboardingStoryIntelligenceService.cs index 407762d..f0fa816 100644 --- a/PlotLine/Services/OnboardingStoryIntelligenceService.cs +++ b/PlotLine/Services/OnboardingStoryIntelligenceService.cs @@ -39,6 +39,7 @@ public sealed class OnboardingStoryIntelligenceService( IStoryIntelligenceAssetImportService assetImport, IStoryIntelligenceRelationshipImportService relationshipImport, IStoryIntelligenceKnowledgeImportService knowledgeImport, + IStoryIntelligenceSourceRepository sources, IStoryIntelligencePipelineStateService pipelineState, IStoryMemoryRepository storyMemory, IStoryIntelligenceClient client, @@ -108,8 +109,10 @@ public sealed class OnboardingStoryIntelligenceService( return existing; } - build ??= await onboarding.BuildApprovedStructureAsync(preview.PreviewID) - ?? throw new InvalidOperationException("The approved chapters could not be prepared for analysis."); + if (build is null || build.SceneMappings.Count == 0) + { + throw new InvalidOperationException("Import the manuscript structure before starting Story Intelligence."); + } var chapterIds = build.ChapterMappings.ToDictionary(chapter => chapter.TemporaryChapterKey, chapter => chapter.ChapterID, StringComparer.Ordinal); var included = IncludedChapters(preview, review); @@ -118,11 +121,6 @@ public sealed class OnboardingStoryIntelligenceService( throw new InvalidOperationException("Choose at least one chapter before analysing the manuscript."); } - if (included.Any(chapter => string.IsNullOrWhiteSpace(chapter.Preview.ChapterText))) - { - throw new InvalidOperationException("The Word Companion scan did not include chapter text. Scan the manuscript again before analysing it."); - } - var clientStatus = client.GetConfigurationStatus(); var modelSummary = StoryIntelligenceOptions.BuildModelSummary( clientStatus.ChapterStructureModel, @@ -142,7 +140,18 @@ public sealed class OnboardingStoryIntelligenceService( throw new InvalidOperationException($"PlotDirector could not prepare chapter '{chapter.Decision.Title}' for analysis."); } - var text = chapter.Preview.ChapterText!.Trim(); + var durableSource = await sources.GetChapterSourceAsync(chapterId); + var text = durableSource?.SourceText?.Trim(); + if (string.IsNullOrWhiteSpace(text)) + { + text = chapter.Preview.ChapterText?.Trim(); + } + + if (string.IsNullOrWhiteSpace(text)) + { + throw new InvalidOperationException($"PlotDirector does not have persisted scene source text for chapter '{chapter.Decision.Title}'."); + } + var paragraphs = StoryIntelligenceParagraphs.Split(text); var runId = await runs.QueueAdminTextAsync(new StoryIntelligenceRunQueueRequest { @@ -155,8 +164,8 @@ public sealed class OnboardingStoryIntelligenceService( SourceLabel = $"{wizard.SelectedBookTitle}: {chapter.Decision.Title}", ChapterText = text, SourceFileName = preview.DocumentTitle, - SourceWordCount = CountWords(text), - SourceCharacterCount = text.Length, + SourceWordCount = durableSource?.SourceWordCount ?? CountWords(text), + SourceCharacterCount = durableSource?.SourceCharacterCount ?? text.Length, SourceParagraphCount = paragraphs.Count, SourceChapterCount = 1, Model = modelSummary, @@ -838,7 +847,7 @@ public sealed class OnboardingStoryIntelligenceService( } var review = await scanStore.GetReviewAsync(userId, preview.PreviewID); - if (review is null || !string.Equals(review.Status, ManuscriptScanReviewStatuses.ReadyToImport, StringComparison.Ordinal)) + if (review is null || !string.Equals(review.Status, ManuscriptScanReviewStatuses.CoreImportComplete, StringComparison.Ordinal)) { return null; } diff --git a/PlotLine/Sql/150_Phase24B_SceneManuscriptSources.sql b/PlotLine/Sql/150_Phase24B_SceneManuscriptSources.sql new file mode 100644 index 0000000..6027ec1 --- /dev/null +++ b/PlotLine/Sql/150_Phase24B_SceneManuscriptSources.sql @@ -0,0 +1,152 @@ +SET ANSI_NULLS ON; +GO +SET QUOTED_IDENTIFIER ON; +GO + +IF OBJECT_ID(N'dbo.SceneManuscriptSources', N'U') IS NULL +BEGIN + CREATE TABLE dbo.SceneManuscriptSources + ( + SceneID int NOT NULL CONSTRAINT PK_SceneManuscriptSources PRIMARY KEY, + ProjectID int NOT NULL, + BookID int NOT NULL, + ChapterID int NOT NULL, + SourceText nvarchar(max) NOT NULL, + SourceWordCount int NOT NULL, + SourceCharacterCount int NOT NULL, + SourceStartParagraph int NULL, + SourceEndParagraph int NULL, + WordStartParagraphIndex int NULL, + WordEndParagraphIndex int NULL, + SourceKind nvarchar(80) NOT NULL CONSTRAINT DF_SceneManuscriptSources_SourceKind DEFAULT N'StructuralImport', + CreatedUtc datetime2 NOT NULL CONSTRAINT DF_SceneManuscriptSources_CreatedUtc DEFAULT SYSUTCDATETIME(), + UpdatedUtc datetime2 NOT NULL CONSTRAINT DF_SceneManuscriptSources_UpdatedUtc DEFAULT SYSUTCDATETIME(), + CONSTRAINT FK_SceneManuscriptSources_Scenes FOREIGN KEY (SceneID) REFERENCES dbo.Scenes(SceneID), + CONSTRAINT FK_SceneManuscriptSources_Projects FOREIGN KEY (ProjectID) REFERENCES dbo.Projects(ProjectID), + CONSTRAINT FK_SceneManuscriptSources_Books FOREIGN KEY (BookID) REFERENCES dbo.Books(BookID), + CONSTRAINT FK_SceneManuscriptSources_Chapters FOREIGN KEY (ChapterID) REFERENCES dbo.Chapters(ChapterID) + ); +END; +GO + +IF NOT EXISTS (SELECT 1 FROM sys.indexes WHERE name = N'IX_SceneManuscriptSources_BookChapter' AND object_id = OBJECT_ID(N'dbo.SceneManuscriptSources')) + CREATE INDEX IX_SceneManuscriptSources_BookChapter ON dbo.SceneManuscriptSources(BookID, ChapterID, SceneID); +GO + +CREATE OR ALTER PROCEDURE dbo.SceneManuscriptSource_Save + @SceneID int, + @ProjectID int, + @BookID int, + @ChapterID int, + @SourceText nvarchar(max), + @SourceWordCount int, + @SourceCharacterCount int, + @SourceStartParagraph int = NULL, + @SourceEndParagraph int = NULL, + @WordStartParagraphIndex int = NULL, + @WordEndParagraphIndex int = NULL, + @SourceKind nvarchar(80) = N'StructuralImport' +AS +BEGIN + SET NOCOUNT ON; + + MERGE dbo.SceneManuscriptSources AS target + USING (SELECT @SceneID AS SceneID) AS source + ON target.SceneID = source.SceneID + WHEN MATCHED THEN + UPDATE SET ProjectID = @ProjectID, + BookID = @BookID, + ChapterID = @ChapterID, + SourceText = @SourceText, + SourceWordCount = @SourceWordCount, + SourceCharacterCount = @SourceCharacterCount, + SourceStartParagraph = @SourceStartParagraph, + SourceEndParagraph = @SourceEndParagraph, + WordStartParagraphIndex = @WordStartParagraphIndex, + WordEndParagraphIndex = @WordEndParagraphIndex, + SourceKind = @SourceKind, + UpdatedUtc = SYSUTCDATETIME() + WHEN NOT MATCHED THEN + INSERT + ( + SceneID, ProjectID, BookID, ChapterID, SourceText, SourceWordCount, SourceCharacterCount, + SourceStartParagraph, SourceEndParagraph, WordStartParagraphIndex, WordEndParagraphIndex, SourceKind + ) + VALUES + ( + @SceneID, @ProjectID, @BookID, @ChapterID, @SourceText, @SourceWordCount, @SourceCharacterCount, + @SourceStartParagraph, @SourceEndParagraph, @WordStartParagraphIndex, @WordEndParagraphIndex, @SourceKind + ); +END; +GO + +CREATE OR ALTER PROCEDURE dbo.SceneManuscriptSource_GetByScene + @SceneID int +AS +BEGIN + SET NOCOUNT ON; + + SELECT SceneID, ProjectID, BookID, ChapterID, SourceText, SourceWordCount, SourceCharacterCount, + SourceStartParagraph, SourceEndParagraph, WordStartParagraphIndex, WordEndParagraphIndex, SourceKind, + CreatedUtc, UpdatedUtc + FROM dbo.SceneManuscriptSources + WHERE SceneID = @SceneID; +END; +GO + +CREATE OR ALTER PROCEDURE dbo.SceneManuscriptSource_ListByChapter + @ChapterID int +AS +BEGIN + SET NOCOUNT ON; + + SELECT source.SceneID, source.ProjectID, source.BookID, source.ChapterID, source.SourceText, + source.SourceWordCount, source.SourceCharacterCount, source.SourceStartParagraph, source.SourceEndParagraph, + source.WordStartParagraphIndex, source.WordEndParagraphIndex, source.SourceKind, source.CreatedUtc, source.UpdatedUtc + FROM dbo.SceneManuscriptSources source + INNER JOIN dbo.Scenes scene ON scene.SceneID = source.SceneID + WHERE source.ChapterID = @ChapterID + AND scene.IsArchived = 0 + ORDER BY scene.SortOrder, scene.SceneNumber, scene.SceneID; +END; +GO + +CREATE OR ALTER PROCEDURE dbo.StoryIntelligenceSource_ChapterText_GetAdmin + @ChapterID int +AS +BEGIN + SET NOCOUNT ON; + + SELECT + p.ProjectID, + p.ProjectName, + b.BookID, + b.BookTitle, + b.Subtitle AS BookSubtitle, + c.ChapterID, + c.ChapterNumber, + c.ChapterTitle, + sourceText.SourceText, + sourceText.SourceWordCount, + sourceText.SourceCharacterCount + FROM dbo.Chapters c + INNER JOIN dbo.Books b ON b.BookID = c.BookID + INNER JOIN dbo.Projects p ON p.ProjectID = b.ProjectID + OUTER APPLY + ( + SELECT + STRING_AGG(CONVERT(nvarchar(max), sms.SourceText), CHAR(13) + CHAR(10) + CHAR(13) + CHAR(10)) + WITHIN GROUP (ORDER BY s.SortOrder, s.SceneNumber, s.SceneID) AS SourceText, + SUM(sms.SourceWordCount) AS SourceWordCount, + SUM(sms.SourceCharacterCount) AS SourceCharacterCount + FROM dbo.Scenes s + INNER JOIN dbo.SceneManuscriptSources sms ON sms.SceneID = s.SceneID + WHERE s.ChapterID = c.ChapterID + AND s.IsArchived = 0 + ) sourceText + WHERE c.ChapterID = @ChapterID + AND c.IsArchived = 0 + AND b.IsArchived = 0 + AND p.IsArchived = 0; +END; +GO diff --git a/PlotLine/Views/Onboarding/BuildComplete.cshtml b/PlotLine/Views/Onboarding/BuildComplete.cshtml index ac18b04..14f777f 100644 --- a/PlotLine/Views/Onboarding/BuildComplete.cshtml +++ b/PlotLine/Views/Onboarding/BuildComplete.cshtml @@ -1,6 +1,7 @@ @model OnboardingManuscriptBuildResult @{ ViewData["Title"] = "Manuscript imported"; + var coreComplete = string.Equals(Model.Status, "CoreImportComplete", StringComparison.OrdinalIgnoreCase); }
@@ -20,7 +21,20 @@

Your manuscript has been imported.

-

@(Model.AlreadyBuilt ? "This manuscript structure has already been created in PlotDirector, so nothing was duplicated." : "PlotDirector created the approved chapters and scenes, updated the Word markers, and saved the manuscript. Word and this browser page may now be closed.")

+

+ @if (coreComplete) + { + @:PlotDirector created the approved chapters and scenes, saved the Word markers, and applied your character review. Core import is complete. + } + else if (Model.AlreadyBuilt) + { + @:This manuscript structure has already been created in PlotDirector, so nothing was duplicated. Review characters to complete the core import. + } + else + { + @:PlotDirector created the approved chapters and scenes, persisted their manuscript text, updated the Word markers, and saved the manuscript. Word may now be closed; keep this browser workflow open to review characters. + } +

@if (!string.IsNullOrWhiteSpace(Model.MarkerWarning)) {

@Model.MarkerWarning

@@ -46,14 +60,31 @@ -
-

Optional: analyse manuscript

-

Story Intelligence can now enrich the existing scenes with summaries, context, relationships, knowledge, timeline details and metrics. You can analyse now or return later.

-
+ @if (coreComplete) + { +
+

Optional: analyse manuscript

+

Story Intelligence can now enrich the existing scenes with summaries, context, relationships, knowledge, timeline details and metrics. You can analyse now or return later.

+
+ } + else + { +
+

Next: review characters

+

Character detection remains part of core import. Review detected names, aliases and existing matches to finish importing the manuscript.

+
+ }
- Analyse my book - Not now + @if (coreComplete) + { + Analyse my book + Not now + } + else + { + Review characters + } Open Project Overview Open Writer Workspace
diff --git a/PlotLine/Views/Onboarding/ScanReview.cshtml b/PlotLine/Views/Onboarding/ScanReview.cshtml index f5c53a2..572bbc4 100644 --- a/PlotLine/Views/Onboarding/ScanReview.cshtml +++ b/PlotLine/Views/Onboarding/ScanReview.cshtml @@ -188,7 +188,7 @@ }
- +
@@ -353,13 +353,13 @@ } else { - + } } else { Back to setup - + } diff --git a/PlotLine/wwwroot/js/word-companion-host.js b/PlotLine/wwwroot/js/word-companion-host.js index 21c505b..f3ba666 100644 --- a/PlotLine/wwwroot/js/word-companion-host.js +++ b/PlotLine/wwwroot/js/word-companion-host.js @@ -2171,6 +2171,7 @@ wordCount: 0, chapterTextParagraphs: [], chapterText: "", + sourceParagraphs: [], startPosition: paragraph?.index ?? null, existingChapterID: isOpening ? null : paragraphAnchorId(paragraph, "PD-CHAPTER") }; @@ -2199,6 +2200,23 @@ return scene; }; + const recordSourceParagraph = (paragraph, text, options = {}) => { + if (!chapter || !paragraph || !Number.isInteger(paragraph.index)) { + return; + } + + chapter.sourceParagraphs = chapter.sourceParagraphs || []; + chapter.sourceParagraphs.push({ + wordParagraphIndex: paragraph.index, + text: String(text || ""), + isBlank: !!options.isBlank, + isChapterHeading: !!options.isChapterHeading, + isSceneHeading: !!options.isSceneHeading, + isSceneSeparator: !!options.isSceneSeparator, + includeInSourceText: options.includeInSourceText !== false + }); + }; + const addWords = (paragraph, text, words) => { if (!chapter || !scene || words <= 0) { return; @@ -2220,8 +2238,12 @@ paragraphs.forEach((paragraph, index) => { paragraph.index = index; - const text = String(paragraph.text || "").trim(); + const rawText = String(paragraph.text || ""); + const text = rawText.trim(); if (!text) { + if (chapter) { + recordSourceParagraph(paragraph, "", { isBlank: true, includeInSourceText: true }); + } return; } @@ -2236,6 +2258,7 @@ documentText.push(text); detectedHeadingChapters += 1; startChapter(paragraph, text); + recordSourceParagraph(paragraph, text, { isChapterHeading: true, includeInSourceText: false }); totalWordCount += words; chapter.wordCount += words; chapter.chapterTextParagraphs.push(text); @@ -2260,16 +2283,19 @@ if (isBuiltInHeading(paragraph, 2)) { startScene(paragraph, text); + recordSourceParagraph(paragraph, text, { isSceneHeading: true, includeInSourceText: true }); totalWordCount += words; addWords(paragraph, text, words); return; } if (sceneSeparatorTexts.has(text)) { + recordSourceParagraph(paragraph, text, { isSceneSeparator: true, includeInSourceText: false }); startScene(paragraph, null); return; } + recordSourceParagraph(paragraph, text, { includeInSourceText: true }); totalWordCount += words; addWords(paragraph, text, words); });