From 9d8d4430673ba4bee0c1a535f6f14360156a33cd Mon Sep 17 00:00:00 2001 From: Nick Beckley Date: Tue, 25 Aug 2026 13:14:16 +0000 Subject: [PATCH] Improve core import character mapping --- PlotLine.Tests/Program.cs | 68 ++++++ PlotLine/Data/OnboardingBuildRepository.cs | 38 +++ PlotLine/Data/Repositories.cs | 7 +- .../Docs/AI/Chapter-Structure-Prompt-V2.md | 10 +- PlotLine/Models/ChapterStructureModels.cs | 1 + PlotLine/Models/CoreModels.cs | 5 + PlotLine/Program.cs | 1 + .../ChapterStructureBoundaryNormaliser.cs | 1 + .../Services/ChapterStructureValidator.cs | 9 + .../CoreImportCharacterMappingService.cs | 217 ++++++++++++++++++ .../ManuscriptStructureAnalysisService.cs | 13 ++ PlotLine/Services/OnboardingService.cs | 8 + .../PersistedStoryIntelligenceRunner.cs | 1 + ...CoreImportCharacterMappingAndSummaries.sql | 169 ++++++++++++++ PlotLine/Views/Onboarding/ScanReview.cshtml | 21 +- 15 files changed, 557 insertions(+), 12 deletions(-) create mode 100644 PlotLine/Services/CoreImportCharacterMappingService.cs create mode 100644 PlotLine/Sql/154_Phase24C_CoreImportCharacterMappingAndSummaries.sql diff --git a/PlotLine.Tests/Program.cs b/PlotLine.Tests/Program.cs index 6c50ef2..08d0185 100644 --- a/PlotLine.Tests/Program.cs +++ b/PlotLine.Tests/Program.cs @@ -65,7 +65,11 @@ var tests = new (string Name, Action Test)[] ("Structural import creates scenes before Word sync", StructuralImportCreatesScenesBeforeWordSync), ("Structural import persists paragraph mapping and source text", StructuralImportPersistsParagraphMappingAndSourceText), ("Structural import persists canonical scene source text", StructuralImportPersistsCanonicalSceneSourceText), + ("Structural import adds summaries without extra scene AI calls", StructuralImportAddsSummariesWithoutExtraSceneAiCalls), + ("Core import deterministically maps known characters", CoreImportDeterministicallyMapsKnownCharacters), ("Core import wizard orders scenes before characters", CoreImportWizardOrdersScenesBeforeCharacters), + ("Core import remaps characters after review", CoreImportRemapsCharactersAfterReview), + ("Scan review separates known matches from unresolved names", ScanReviewSeparatesKnownMatchesFromUnresolvedNames), ("Story Intelligence uses persisted scene source after restart", StoryIntelligenceUsesPersistedSceneSourceAfterRestart), ("Word sync requires save acknowledgement", WordSyncRequiresSaveAcknowledgement), ("Story Intelligence enriches existing structural scenes", StoryIntelligenceEnrichesExistingStructuralScenes), @@ -1381,6 +1385,46 @@ static void StructuralImportPersistsCanonicalSceneSourceText() Assert(analyzer.Contains("item.IncludeInSourceText", StringComparison.Ordinal), "Scene source text should exclude non-prose structural paragraphs such as hidden markers/separators."); } +static void StructuralImportAddsSummariesWithoutExtraSceneAiCalls() +{ + var root = Path.Combine(AppContext.BaseDirectory, "../../../../PlotLine"); + var prompt = File.ReadAllText(Path.Combine(root, "Docs/AI/Chapter-Structure-Prompt-V2.md")); + var model = File.ReadAllText(Path.Combine(root, "Models/ChapterStructureModels.cs")); + var analyzer = File.ReadAllText(Path.Combine(root, "Services/ManuscriptStructureAnalysisService.cs")); + var onboarding = File.ReadAllText(Path.Combine(root, "Services/OnboardingService.cs")); + var migration = File.ReadAllText(Path.Combine(root, "Sql/154_Phase24C_CoreImportCharacterMappingAndSummaries.sql")); + + Assert(prompt.Contains("structuralSummary", StringComparison.Ordinal), "Chapter boundary prompt should request a per-scene structural summary."); + Assert(prompt.Contains("25 to 40 words", StringComparison.Ordinal), "Structural summaries should be short factual summaries."); + Assert(model.Contains("public string? StructuralSummary", StringComparison.Ordinal), "Chapter boundary model should carry structural summaries."); + Assert(analyzer.Contains("StructuralSummary = Trim(boundary.StructuralSummary", StringComparison.Ordinal), "Boundary analysis should carry summaries into structural scenes."); + Assert(onboarding.Contains("SaveSceneStructuralSummaryAsync", StringComparison.Ordinal), "Core import should persist structural summaries after canonical SceneIDs exist."); + Assert(migration.Contains("SceneStructuralSummary_Save", StringComparison.Ordinal), "Structural summaries should be saved through a stored procedure."); + Assert(analyzer.Split("ExecutePromptAsync", StringSplitOptions.None).Length - 1 == 1, "Structural summaries must be produced by the existing chapter-boundary call, not extra scene calls."); +} + +static void CoreImportDeterministicallyMapsKnownCharacters() +{ + var root = Path.Combine(AppContext.BaseDirectory, "../../../../PlotLine"); + var service = File.ReadAllText(Path.Combine(root, "Services/CoreImportCharacterMappingService.cs")); + var repository = File.ReadAllText(Path.Combine(root, "Data/OnboardingBuildRepository.cs")); + var migration = File.ReadAllText(Path.Combine(root, "Sql/154_Phase24C_CoreImportCharacterMappingAndSummaries.sql")); + + Assert(service.Contains("ICoreImportCharacterMappingService", StringComparison.Ordinal), "Deterministic character mapping service should be explicit."); + Assert(service.Contains("ListSceneManuscriptSourcesByBookAsync", StringComparison.Ordinal), "Mapping should scan persisted canonical scene source text."); + Assert(service.Contains("characters.ListCharactersAsync(projectId)", StringComparison.Ordinal), "Mapping should use known project characters before AI discovery."); + Assert(service.Contains("characters.ListAliasesAsync", StringComparison.Ordinal), "Mapping should include accepted aliases."); + Assert(service.Contains("GenericReferences", StringComparison.Ordinal), "Mapping should reject generic family/role labels."); + Assert(service.Contains("AddUniqueTokenTerm(index, character.CharacterID, tokens[0], tokenOwners)", StringComparison.Ordinal), "First-name matching should require unique ownership."); + Assert(service.Contains("AddUniqueTokenTerm(index, character.CharacterID, tokens[^1], tokenOwners)", StringComparison.Ordinal), "Surname-only matching should require unique ownership."); + Assert(service.Contains("tokenOwners.GetValueOrDefault(token) == 1", StringComparison.Ordinal), "Token matching should only add unique first/surname terms."); + Assert(service.Contains(@"(?= 0, "Character review should save accepted aliases."); + Assert(remap > aliasSave, "Character review should rerun deterministic mapping after accepted aliases are saved."); +} + +static void ScanReviewSeparatesKnownMatchesFromUnresolvedNames() +{ + var root = Path.Combine(AppContext.BaseDirectory, "../../../../PlotLine"); + var review = File.ReadAllText(Path.Combine(root, "Views/Onboarding/ScanReview.cshtml")); + + Assert(review.Contains("Already matched known characters", StringComparison.Ordinal), "Character review should separate already-matched known characters."); + Assert(review.Contains("Likely new characters", StringComparison.Ordinal), "Character review should focus unresolved likely characters separately."); + Assert(review.Contains("Possible new names or aliases", StringComparison.Ordinal), "Character review should label possible alias/new-name cases clearly."); + Assert(review.Contains("Generic or ambiguous references", StringComparison.Ordinal), "Character review should label generic/ambiguous references distinctly."); +} + static void StoryIntelligenceUsesPersistedSceneSourceAfterRestart() { var root = Path.Combine(AppContext.BaseDirectory, "../../../../PlotLine"); @@ -1404,6 +1470,7 @@ static void StoryIntelligenceUsesPersistedSceneSourceAfterRestart() var sourceRepository = File.ReadAllText(Path.Combine(root, "Data/StoryIntelligenceSourceRepository.cs")); var onboardingStory = File.ReadAllText(Path.Combine(root, "Services/OnboardingStoryIntelligenceService.cs")); var existingQueue = File.ReadAllText(Path.Combine(root, "Services/StoryIntelligenceExistingChapterQueueService.cs")); + var runner = File.ReadAllText(Path.Combine(root, "Services/PersistedStoryIntelligenceRunner.cs")); Assert(migration.Contains("INNER JOIN dbo.SceneManuscriptSources", StringComparison.Ordinal), "Chapter source SQL should aggregate persisted canonical scene source text."); Assert(migration.Contains("STRING_AGG", StringComparison.Ordinal), "Chapter source SQL should reconstruct chapter text from persisted scene text in order."); @@ -1411,6 +1478,7 @@ static void StoryIntelligenceUsesPersistedSceneSourceAfterRestart() Assert(onboardingStory.Contains("durableSource = await sources.GetChapterSourceAsync", StringComparison.Ordinal), "Onboarding Story Intelligence should read durable chapter text from scene sources."); Assert(onboardingStory.Contains("CoreImportComplete", StringComparison.Ordinal), "Optional onboarding Story Intelligence should require core import completion."); Assert(existingQueue.Contains("sources.GetChapterSourceAsync", StringComparison.Ordinal), "Later Story Intelligence queueing should start from persisted chapter/scene source, not onboarding DTOs."); + Assert(runner.Contains("structuralSummary = boundary.StructuralSummary", StringComparison.Ordinal), "Scene Intelligence context should receive structural summaries from the boundary pass."); } static void WordSyncRequiresSaveAcknowledgement() diff --git a/PlotLine/Data/OnboardingBuildRepository.cs b/PlotLine/Data/OnboardingBuildRepository.cs index f1339c8..3811bd1 100644 --- a/PlotLine/Data/OnboardingBuildRepository.cs +++ b/PlotLine/Data/OnboardingBuildRepository.cs @@ -13,6 +13,8 @@ public interface IOnboardingBuildRepository Task MarkMarkerSyncFailedAsync(int buildId, string message); Task SaveStructuralChapterSourceAsync(StructuralChapterSourceSaveRequest request); Task SaveSceneManuscriptSourceAsync(SceneManuscriptSourceSaveRequest request); + Task SaveSceneStructuralSummaryAsync(int sceneId, string structuralSummary); + Task> ListSceneManuscriptSourcesByBookAsync(int bookId); } public sealed class OnboardingBuildRepository(ISqlConnectionFactory connectionFactory) : IOnboardingBuildRepository @@ -134,6 +136,25 @@ public sealed class OnboardingBuildRepository(ISqlConnectionFactory connectionFa }, commandType: CommandType.StoredProcedure); } + + public async Task SaveSceneStructuralSummaryAsync(int sceneId, string structuralSummary) + { + using var connection = connectionFactory.CreateConnection(); + await connection.ExecuteAsync( + "dbo.SceneStructuralSummary_Save", + new { SceneID = sceneId, StructuralSummary = structuralSummary }, + commandType: CommandType.StoredProcedure); + } + + public async Task> ListSceneManuscriptSourcesByBookAsync(int bookId) + { + using var connection = connectionFactory.CreateConnection(); + var rows = await connection.QueryAsync( + "dbo.SceneManuscriptSource_ListByBook", + new { BookID = bookId }, + commandType: CommandType.StoredProcedure); + return rows.ToList(); + } } public sealed class OnboardingManuscriptBuildRequest @@ -197,3 +218,20 @@ public sealed class SceneManuscriptSourceSaveRequest public int? WordEndParagraphIndex { get; init; } public string SourceKind { get; init; } = "StructuralImport"; } + +public sealed class SceneManuscriptSourceRecord +{ + public int SceneID { get; init; } + public int ProjectID { get; init; } + public int BookID { get; init; } + public int ChapterID { get; init; } + public string SourceText { get; init; } = string.Empty; + public int SourceWordCount { get; init; } + public int SourceCharacterCount { get; init; } + public int? SourceStartParagraph { get; init; } + public int? SourceEndParagraph { get; init; } + public int? WordStartParagraphIndex { get; init; } + public int? WordEndParagraphIndex { get; init; } + public string SourceKind { get; init; } = string.Empty; + public string? StructuralSummary { get; init; } +} diff --git a/PlotLine/Data/Repositories.cs b/PlotLine/Data/Repositories.cs index 1edd164..badc217 100644 --- a/PlotLine/Data/Repositories.cs +++ b/PlotLine/Data/Repositories.cs @@ -3864,7 +3864,12 @@ public sealed class CharacterRepository(ISqlConnectionFactory connectionFactory) sceneCharacter.OutfitDescription, sceneCharacter.PhysicalCondition, sceneCharacter.EmotionalState, - sceneCharacter.KnowledgeNotes + sceneCharacter.KnowledgeNotes, + sceneCharacter.MatchSource, + sceneCharacter.MatchedText, + sceneCharacter.OccurrenceCount, + sceneCharacter.FirstOccurrenceIndex, + PreserveExistingWhenNull = string.Equals(sceneCharacter.MatchSource, "CoreImportDeterministic", StringComparison.Ordinal) }, commandType: CommandType.StoredProcedure); } diff --git a/PlotLine/Docs/AI/Chapter-Structure-Prompt-V2.md b/PlotLine/Docs/AI/Chapter-Structure-Prompt-V2.md index 027e8fe..974d202 100644 --- a/PlotLine/Docs/AI/Chapter-Structure-Prompt-V2.md +++ b/PlotLine/Docs/AI/Chapter-Structure-Prompt-V2.md @@ -10,9 +10,9 @@ Replace the insertion blocks marked with `{{...}}` before sending. You are PlotDirector's Chapter Structure engine. -You are an assistant archivist for a novelist. You observe chapter structure only. You do not critique, rewrite, summarise scenes, create story entities or infer canon. +You are an assistant archivist for a novelist. You observe chapter structure only. You do not critique, rewrite, create story entities or infer canon. -Your only task is to identify likely scene boundaries in one chapter. +Your only task is to identify likely scene boundaries in one chapter and provide a light factual summary for each detected scene. ## Operating Principles @@ -117,11 +117,14 @@ Each `sceneBoundaries` item must contain exactly: - `sceneNumber` - `startParagraph` - `endParagraph` +- `structuralSummary` - `confidence` - `reason` Use confidence values from `0.0` to `1.0`. +Use `structuralSummary` for a factual, neutral summary of what happens in that scene. It should normally be 25 to 40 words, avoid interpretation, and only use information present in the supplied text. + Use short factual reasons, such as: `Location changes from house to train.` @@ -133,7 +136,7 @@ Do not output: - assets; - relationships; - metrics; -- scene summaries; +- interpretive scene analysis; - observations; - continuity warnings; - plot threads. @@ -149,6 +152,7 @@ Do not output: "sceneNumber": 1, "startParagraph": 1, "endParagraph": 3, + "structuralSummary": "The opening scene follows the same continuous action across the supplied paragraphs, keeping the characters in one time and place while the immediate situation is established.", "confidence": 0.86, "reason": "Opening scene remains in the same place and time." } diff --git a/PlotLine/Models/ChapterStructureModels.cs b/PlotLine/Models/ChapterStructureModels.cs index f3de63c..d2f5e7a 100644 --- a/PlotLine/Models/ChapterStructureModels.cs +++ b/PlotLine/Models/ChapterStructureModels.cs @@ -18,6 +18,7 @@ public sealed class ChapterSceneBoundary public int? SceneNumber { get; init; } public int? StartParagraph { get; init; } public int? EndParagraph { get; init; } + public string? StructuralSummary { get; init; } public decimal? Confidence { get; init; } public string? Reason { get; init; } diff --git a/PlotLine/Models/CoreModels.cs b/PlotLine/Models/CoreModels.cs index 0fcdcf6..778ebc8 100644 --- a/PlotLine/Models/CoreModels.cs +++ b/PlotLine/Models/CoreModels.cs @@ -377,6 +377,7 @@ public sealed class Scene public decimal SceneNumber { get; set; } public string SceneTitle { get; set; } = string.Empty; public string? Summary { get; set; } + public string? StructuralSummary { get; set; } public int? POVCharacterID { get; set; } public int? PrimaryLocationID { get; set; } public int? FloorPlanID { get; set; } @@ -997,6 +998,10 @@ public sealed class SceneCharacter public string? PhysicalCondition { get; set; } public string? EmotionalState { get; set; } public string? KnowledgeNotes { get; set; } + public string? MatchSource { get; set; } + public string? MatchedText { get; set; } + public int? OccurrenceCount { get; set; } + public int? FirstOccurrenceIndex { get; set; } public decimal SceneNumber { get; set; } public string SceneTitle { get; set; } = string.Empty; public decimal ChapterNumber { get; set; } diff --git a/PlotLine/Program.cs b/PlotLine/Program.cs index 036474c..6470b25 100644 --- a/PlotLine/Program.cs +++ b/PlotLine/Program.cs @@ -216,6 +216,7 @@ public class Program builder.Services.AddScoped(); builder.Services.AddScoped(); builder.Services.AddScoped(); + builder.Services.AddScoped(); builder.Services.AddScoped(); builder.Services.AddScoped(); builder.Services.AddScoped(); diff --git a/PlotLine/Services/ChapterStructureBoundaryNormaliser.cs b/PlotLine/Services/ChapterStructureBoundaryNormaliser.cs index 0ba7fee..6f438c3 100644 --- a/PlotLine/Services/ChapterStructureBoundaryNormaliser.cs +++ b/PlotLine/Services/ChapterStructureBoundaryNormaliser.cs @@ -92,6 +92,7 @@ public static class ChapterStructureBoundaryNormaliser SceneNumber = sceneNumber, StartParagraph = boundary.StartParagraph, EndParagraph = boundary.EndParagraph, + StructuralSummary = boundary.StructuralSummary, Confidence = boundary.Confidence, Reason = boundary.Reason, ExtensionData = boundary.ExtensionData diff --git a/PlotLine/Services/ChapterStructureValidator.cs b/PlotLine/Services/ChapterStructureValidator.cs index 5262753..4c70dda 100644 --- a/PlotLine/Services/ChapterStructureValidator.cs +++ b/PlotLine/Services/ChapterStructureValidator.cs @@ -119,6 +119,15 @@ public sealed class ChapterStructureValidator : IChapterStructureValidator } Confidence(boundary.Confidence, $"{path}.confidence", result); + RequiredString(boundary.StructuralSummary, $"{path}.structuralSummary", result); + if (!string.IsNullOrWhiteSpace(boundary.StructuralSummary)) + { + var wordCount = boundary.StructuralSummary.Split((char[]?)null, StringSplitOptions.RemoveEmptyEntries).Length; + if (wordCount is < 15 or > 80) + { + AddWarning(result, $"{path}.structuralSummary", "Structural summary should be concise.", "Return a factual scene summary of roughly 25 to 40 words."); + } + } RequiredString(boundary.Reason, $"{path}.reason", result); Unknown(boundary.ExtensionData, path, result); } diff --git a/PlotLine/Services/CoreImportCharacterMappingService.cs b/PlotLine/Services/CoreImportCharacterMappingService.cs new file mode 100644 index 0000000..323d5c3 --- /dev/null +++ b/PlotLine/Services/CoreImportCharacterMappingService.cs @@ -0,0 +1,217 @@ +using System.Text.RegularExpressions; +using PlotLine.Data; +using PlotLine.Models; + +namespace PlotLine.Services; + +public interface ICoreImportCharacterMappingService +{ + Task MapBookAsync(int projectId, int bookId); +} + +public sealed class CoreImportCharacterMappingService( + IOnboardingBuildRepository builds, + ICharacterRepository characters, + ILogger logger) : ICoreImportCharacterMappingService +{ + private const string MatchSource = "CoreImportDeterministic"; + + private static readonly HashSet GenericReferences = new(StringComparer.OrdinalIgnoreCase) + { + "aunt", "boy", "brother", "child", "children", "dad", "daddy", "doctor", "examiner", "father", + "friend", "girl", "grandfather", "grandmother", "he", "him", "his", "lady", "man", "miss", + "mother", "mum", "mummy", "officer", "parent", "professor", "she", "sister", "sir", "son", + "teacher", "uncle", "woman" + }; + + public async Task MapBookAsync(int projectId, int bookId) + { + var sourceScenes = await builds.ListSceneManuscriptSourcesByBookAsync(bookId); + if (sourceScenes.Count == 0) + { + return new CoreImportCharacterMappingResult(0, 0); + } + + var projectCharacters = (await characters.ListCharactersAsync(projectId)) + .Where(character => !character.IsArchived && !string.IsNullOrWhiteSpace(character.CharacterName)) + .ToList(); + if (projectCharacters.Count == 0) + { + return new CoreImportCharacterMappingResult(sourceScenes.Count, 0); + } + + var aliases = new Dictionary>(); + foreach (var character in projectCharacters) + { + aliases[character.CharacterID] = await characters.ListAliasesAsync(character.CharacterID); + } + + var matchIndex = BuildMatchIndex(projectCharacters, aliases); + var lookup = await characters.GetLookupsAsync(projectId); + var roleTypeId = lookup.RoleTypes.FirstOrDefault(item => string.Equals(item.TypeName, "Mentioned Only", StringComparison.OrdinalIgnoreCase))?.CharacterRoleInSceneTypeID + ?? lookup.RoleTypes.FirstOrDefault()?.CharacterRoleInSceneTypeID; + var presenceTypeId = lookup.PresenceTypes.FirstOrDefault(item => string.Equals(item.TypeName, "Mentioned Only", StringComparison.OrdinalIgnoreCase))?.PresenceTypeID + ?? lookup.PresenceTypes.FirstOrDefault(item => string.Equals(item.TypeName, "Present", StringComparison.OrdinalIgnoreCase))?.PresenceTypeID + ?? lookup.PresenceTypes.FirstOrDefault()?.PresenceTypeID; + + var saved = 0; + foreach (var scene in sourceScenes) + { + foreach (var match in MatchScene(scene.SourceText, matchIndex).OrderBy(item => item.CharacterID)) + { + await characters.SaveSceneCharacterAsync(new SceneCharacter + { + SceneID = scene.SceneID, + CharacterID = match.CharacterID, + RoleInSceneTypeID = roleTypeId, + PresenceTypeID = presenceTypeId, + MatchSource = MatchSource, + MatchedText = match.MatchedText, + OccurrenceCount = match.OccurrenceCount, + FirstOccurrenceIndex = match.FirstOccurrenceIndex + }); + saved++; + } + } + + logger.LogInformation( + "Core import deterministic character mapping completed. ProjectID={ProjectID} BookID={BookID} Scenes={SceneCount} LinksSaved={LinksSaved}", + projectId, + bookId, + sourceScenes.Count, + saved); + + return new CoreImportCharacterMappingResult(sourceScenes.Count, saved); + } + + internal static IReadOnlyList BuildMatchIndex( + IReadOnlyList projectCharacters, + IReadOnlyDictionary> aliases) + { + var termsByCharacter = new Dictionary>(); + foreach (var character in projectCharacters) + { + var terms = new HashSet(StringComparer.OrdinalIgnoreCase); + AddTerm(terms, character.CharacterName); + AddTerm(terms, character.ShortName); + foreach (var alias in aliases.GetValueOrDefault(character.CharacterID) ?? []) + { + AddTerm(terms, alias.Alias); + } + + termsByCharacter[character.CharacterID] = terms; + } + + var tokenOwners = termsByCharacter + .SelectMany(pair => pair.Value.SelectMany(TokeniseName).Select(token => (Token: token, CharacterID: pair.Key))) + .GroupBy(item => item.Token, StringComparer.OrdinalIgnoreCase) + .ToDictionary( + group => group.Key, + group => group.Select(item => item.CharacterID).Distinct().Count(), + StringComparer.OrdinalIgnoreCase); + + var index = new List(); + foreach (var character in projectCharacters) + { + foreach (var term in termsByCharacter[character.CharacterID]) + { + var tokens = TokeniseName(term).ToList(); + if (tokens.Count == 0) + { + continue; + } + + if (tokens.Count == 1 && (GenericReferences.Contains(tokens[0]) || tokenOwners.GetValueOrDefault(tokens[0]) > 1)) + { + continue; + } + + index.Add(new DeterministicCharacterMatchTerm(character.CharacterID, term, RegexForTerm(term))); + + if (tokens.Count > 1) + { + AddUniqueTokenTerm(index, character.CharacterID, tokens[0], tokenOwners); + AddUniqueTokenTerm(index, character.CharacterID, tokens[^1], tokenOwners); + } + } + } + + return index + .GroupBy(item => (item.CharacterID, Key: item.Term.ToLowerInvariant())) + .Select(group => group.First()) + .OrderByDescending(item => item.Term.Length) + .ThenBy(item => item.Term, StringComparer.OrdinalIgnoreCase) + .ToList(); + } + + internal static IReadOnlyList MatchScene( + string? sourceText, + IReadOnlyList matchIndex) + { + if (string.IsNullOrWhiteSpace(sourceText) || matchIndex.Count == 0) + { + return []; + } + + var matches = new Dictionary(); + foreach (var term in matchIndex) + { + var regexMatches = term.Regex.Matches(sourceText); + if (regexMatches.Count == 0) + { + continue; + } + + if (!matches.TryGetValue(term.CharacterID, out var existing) + || regexMatches[0].Index < existing.FirstOccurrenceIndex + || (regexMatches[0].Index == existing.FirstOccurrenceIndex && term.Term.Length > existing.MatchedText.Length)) + { + matches[term.CharacterID] = new DeterministicSceneCharacterMatch( + term.CharacterID, + term.Term, + regexMatches.Count, + regexMatches[0].Index); + } + } + + return matches.Values.ToList(); + } + + private static void AddUniqueTokenTerm( + List index, + int characterId, + string token, + IReadOnlyDictionary tokenOwners) + { + if (!GenericReferences.Contains(token) && tokenOwners.GetValueOrDefault(token) == 1) + { + index.Add(new DeterministicCharacterMatchTerm(characterId, token, RegexForTerm(token))); + } + } + + private static void AddTerm(HashSet terms, string? value) + { + var clean = CleanName(value); + if (!string.IsNullOrWhiteSpace(clean) && !GenericReferences.Contains(clean)) + { + terms.Add(clean); + } + } + + private static IEnumerable TokeniseName(string value) + => value.Split([' ', '-', '\''], StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries) + .Select(CleanName) + .Where(token => token.Length > 1); + + private static Regex RegexForTerm(string term) + => new($@"(? Regex.Replace(value?.Trim() ?? string.Empty, @"\s+", " "); +} + +public sealed record CoreImportCharacterMappingResult(int ScenesScanned, int LinksSaved); + +public sealed record DeterministicCharacterMatchTerm(int CharacterID, string Term, Regex Regex); + +public sealed record DeterministicSceneCharacterMatch(int CharacterID, string MatchedText, int OccurrenceCount, int FirstOccurrenceIndex); diff --git a/PlotLine/Services/ManuscriptStructureAnalysisService.cs b/PlotLine/Services/ManuscriptStructureAnalysisService.cs index 41d27ac..063de55 100644 --- a/PlotLine/Services/ManuscriptStructureAnalysisService.cs +++ b/PlotLine/Services/ManuscriptStructureAnalysisService.cs @@ -95,6 +95,7 @@ public sealed class ManuscriptStructureAnalysisService( EndParagraph = end, WordStartPosition = startMap.WordParagraphIndex, WordEndPosition = endMap.WordParagraphIndex, + StructuralSummary = Trim(boundary.StructuralSummary, 600), Reason = boundary.Reason, Confidence = boundary.Confidence }); @@ -196,6 +197,17 @@ public sealed class ManuscriptStructureAnalysisService( => string.IsNullOrWhiteSpace(value) ? 0 : value.Split((char[]?)null, StringSplitOptions.RemoveEmptyEntries).Length; + + private static string? Trim(string? value, int maxLength) + { + var clean = value?.Trim(); + if (string.IsNullOrWhiteSpace(clean)) + { + return null; + } + + return clean.Length <= maxLength ? clean : clean[..maxLength].Trim(); + } } public sealed class ManuscriptStructureAnalysisRequest @@ -237,6 +249,7 @@ public sealed class ManuscriptStructureSceneBoundary public int EndParagraph { get; init; } public int WordStartPosition { get; init; } public int WordEndPosition { get; init; } + public string? StructuralSummary { get; init; } public string? Reason { get; init; } public decimal? Confidence { get; init; } } diff --git a/PlotLine/Services/OnboardingService.cs b/PlotLine/Services/OnboardingService.cs index 8c9b6a2..d9694e8 100644 --- a/PlotLine/Services/OnboardingService.cs +++ b/PlotLine/Services/OnboardingService.cs @@ -40,6 +40,7 @@ public sealed class OnboardingService( ICharacterRepository characters, IOnboardingBuildRepository builds, IManuscriptStructureAnalysisService structureAnalysis, + ICoreImportCharacterMappingService coreImportCharacterMapping, ICurrentUserService currentUser) : IOnboardingService { private static readonly HashSet WritingJourneys = new(StringComparer.Ordinal) @@ -328,6 +329,7 @@ public sealed class OnboardingService( }; await SaveStructuralChapterSourcesAsync(preview, result); await SaveSceneManuscriptSourcesAsync(preview, structuralScenes, result); + await coreImportCharacterMapping.MapBookAsync(preview.ProjectID, preview.BookID); await scanPreviews.SaveBuildResultAsync(userId, result); await PublishBuildProgress(userId, preview.PreviewID, "Updating and saving Word markers. Keep Word and this browser page open.", 90, progress); return result; @@ -440,6 +442,7 @@ public sealed class OnboardingService( await scanPreviews.SaveReviewAsync(userId, completedReview); await scanPreviews.SaveBuildResultAsync(userId, completedResult); await AddScanCharacterAliasesAsync(completedReview, completedResult); + await coreImportCharacterMapping.MapBookAsync(preview.ProjectID, preview.BookID); return completedResult; } @@ -547,6 +550,11 @@ public sealed class OnboardingService( WordStartParagraphIndex = scene.WordStartPosition, WordEndParagraphIndex = scene.WordEndPosition }); + + if (!string.IsNullOrWhiteSpace(scene.StructuralSummary)) + { + await builds.SaveSceneStructuralSummaryAsync(sceneId, scene.StructuralSummary); + } } } diff --git a/PlotLine/Services/PersistedStoryIntelligenceRunner.cs b/PlotLine/Services/PersistedStoryIntelligenceRunner.cs index 9931bce..88b84b0 100644 --- a/PlotLine/Services/PersistedStoryIntelligenceRunner.cs +++ b/PlotLine/Services/PersistedStoryIntelligenceRunner.cs @@ -886,6 +886,7 @@ public sealed class PersistedStoryIntelligenceRunner( sceneId = (int?)null, chapterNumber = ChapterNumber(run), sceneNumber = boundary.SceneNumber, + structuralSummary = boundary.StructuralSummary, sourceLabel = $"{SourceLabel(run)}, suggested scene {boundary.SceneNumber}", knownCharacters = KnownCharacters(run) }, JsonOptions); diff --git a/PlotLine/Sql/154_Phase24C_CoreImportCharacterMappingAndSummaries.sql b/PlotLine/Sql/154_Phase24C_CoreImportCharacterMappingAndSummaries.sql new file mode 100644 index 0000000..4a177b5 --- /dev/null +++ b/PlotLine/Sql/154_Phase24C_CoreImportCharacterMappingAndSummaries.sql @@ -0,0 +1,169 @@ +SET ANSI_NULLS ON; +GO +SET QUOTED_IDENTIFIER ON; +GO + +IF COL_LENGTH(N'dbo.Scenes', N'StructuralSummary') IS NULL +BEGIN + ALTER TABLE dbo.Scenes ADD StructuralSummary nvarchar(max) NULL; +END; +GO + +IF COL_LENGTH(N'dbo.SceneCharacters', N'MatchSource') IS NULL +BEGIN + ALTER TABLE dbo.SceneCharacters ADD MatchSource nvarchar(80) NULL; +END; +GO + +IF COL_LENGTH(N'dbo.SceneCharacters', N'MatchedText') IS NULL +BEGIN + ALTER TABLE dbo.SceneCharacters ADD MatchedText nvarchar(200) NULL; +END; +GO + +IF COL_LENGTH(N'dbo.SceneCharacters', N'OccurrenceCount') IS NULL +BEGIN + ALTER TABLE dbo.SceneCharacters ADD OccurrenceCount int NULL; +END; +GO + +IF COL_LENGTH(N'dbo.SceneCharacters', N'FirstOccurrenceIndex') IS NULL +BEGIN + ALTER TABLE dbo.SceneCharacters ADD FirstOccurrenceIndex int NULL; +END; +GO + +CREATE OR ALTER PROCEDURE dbo.SceneStructuralSummary_Save + @SceneID int, + @StructuralSummary nvarchar(max) +AS +BEGIN + SET NOCOUNT ON; + + UPDATE dbo.Scenes + SET StructuralSummary = NULLIF(LTRIM(RTRIM(@StructuralSummary)), N''), + Summary = CASE + WHEN NULLIF(LTRIM(RTRIM(Summary)), N'') IS NULL THEN NULLIF(LTRIM(RTRIM(@StructuralSummary)), N'') + ELSE Summary + END, + UpdatedDate = SYSUTCDATETIME() + WHERE SceneID = @SceneID; +END; +GO + +CREATE OR ALTER PROCEDURE dbo.SceneManuscriptSource_ListByBook + @BookID int +AS +BEGIN + SET NOCOUNT ON; + + SELECT source.SceneID, source.ProjectID, source.BookID, source.ChapterID, source.SourceText, + source.SourceWordCount, source.SourceCharacterCount, source.SourceStartParagraph, source.SourceEndParagraph, + source.WordStartParagraphIndex, source.WordEndParagraphIndex, source.SourceKind, scene.StructuralSummary + FROM dbo.SceneManuscriptSources source + INNER JOIN dbo.Scenes scene ON scene.SceneID = source.SceneID + INNER JOIN dbo.Chapters chapter ON chapter.ChapterID = source.ChapterID + WHERE source.BookID = @BookID + AND scene.IsArchived = 0 + AND chapter.IsArchived = 0 + ORDER BY chapter.SortOrder, chapter.ChapterNumber, scene.SortOrder, scene.SceneNumber, scene.SceneID; +END; +GO + +CREATE OR ALTER PROCEDURE dbo.SceneCharacter_ListByScene + @SceneID int +AS +BEGIN + SET NOCOUNT ON; + SELECT sc.SceneCharacterID, sc.SceneID, sc.CharacterID, c.ProjectID, c.CharacterName, c.ShortName, c.BirthDate, c.AgeAtSeriesStart, + sc.RoleInSceneTypeID, role.TypeName AS RoleInSceneTypeName, sc.PresenceTypeID, presence.TypeName AS PresenceTypeName, + sc.LocationID, loc.LocationName, sc.EntryLocationID, entryLoc.LocationName AS EntryLocationName, + sc.ExitLocationID, exitLoc.LocationName AS ExitLocationName, + sc.AppearanceNotes, sc.OutfitDescription, sc.PhysicalCondition, sc.EmotionalState, sc.KnowledgeNotes, + sc.MatchSource, sc.MatchedText, sc.OccurrenceCount, sc.FirstOccurrenceIndex, + sc.CreatedDate, sc.UpdatedDate + FROM dbo.SceneCharacters sc + INNER JOIN dbo.Characters c ON c.CharacterID = sc.CharacterID + LEFT JOIN dbo.CharacterRoleInSceneTypes role ON role.CharacterRoleInSceneTypeID = sc.RoleInSceneTypeID + LEFT JOIN dbo.PresenceTypes presence ON presence.PresenceTypeID = sc.PresenceTypeID + LEFT JOIN dbo.Locations loc ON loc.LocationID = sc.LocationID + LEFT JOIN dbo.Locations entryLoc ON entryLoc.LocationID = sc.EntryLocationID + LEFT JOIN dbo.Locations exitLoc ON exitLoc.LocationID = sc.ExitLocationID + WHERE sc.SceneID = @SceneID AND c.IsArchived = 0 + ORDER BY role.SortOrder, c.CharacterName; +END; +GO + +CREATE OR ALTER PROCEDURE dbo.SceneCharacter_Save + @SceneCharacterID int = NULL, + @SceneID int, + @CharacterID int, + @RoleInSceneTypeID int = NULL, + @PresenceTypeID int = NULL, + @LocationID int = NULL, + @EntryLocationID int = NULL, + @ExitLocationID int = NULL, + @AppearanceNotes nvarchar(max) = NULL, + @OutfitDescription nvarchar(max) = NULL, + @PhysicalCondition nvarchar(max) = NULL, + @EmotionalState nvarchar(max) = NULL, + @KnowledgeNotes nvarchar(max) = NULL, + @MatchSource nvarchar(80) = NULL, + @MatchedText nvarchar(200) = NULL, + @OccurrenceCount int = NULL, + @FirstOccurrenceIndex int = NULL, + @PreserveExistingWhenNull bit = 0 +AS +BEGIN + SET NOCOUNT ON; + + IF @SceneCharacterID IS NULL OR @SceneCharacterID = 0 + BEGIN + SELECT @SceneCharacterID = SceneCharacterID + FROM dbo.SceneCharacters + WHERE SceneID = @SceneID AND CharacterID = @CharacterID; + END + + IF @SceneCharacterID IS NULL OR @SceneCharacterID = 0 + BEGIN + INSERT dbo.SceneCharacters + ( + SceneID, CharacterID, RoleInSceneTypeID, PresenceTypeID, LocationID, EntryLocationID, ExitLocationID, + AppearanceNotes, OutfitDescription, PhysicalCondition, EmotionalState, KnowledgeNotes, + MatchSource, MatchedText, OccurrenceCount, FirstOccurrenceIndex + ) + VALUES + ( + @SceneID, @CharacterID, @RoleInSceneTypeID, @PresenceTypeID, @LocationID, @EntryLocationID, @ExitLocationID, + @AppearanceNotes, @OutfitDescription, @PhysicalCondition, @EmotionalState, @KnowledgeNotes, + @MatchSource, @MatchedText, @OccurrenceCount, @FirstOccurrenceIndex + ); + + SET @SceneCharacterID = CAST(SCOPE_IDENTITY() AS int); + END + ELSE + BEGIN + UPDATE dbo.SceneCharacters + SET SceneID = @SceneID, + CharacterID = @CharacterID, + RoleInSceneTypeID = CASE WHEN @PreserveExistingWhenNull = 1 THEN COALESCE(@RoleInSceneTypeID, RoleInSceneTypeID) ELSE @RoleInSceneTypeID END, + PresenceTypeID = CASE WHEN @PreserveExistingWhenNull = 1 THEN COALESCE(@PresenceTypeID, PresenceTypeID) ELSE @PresenceTypeID END, + LocationID = CASE WHEN @PreserveExistingWhenNull = 1 THEN COALESCE(@LocationID, LocationID) ELSE @LocationID END, + EntryLocationID = CASE WHEN @PreserveExistingWhenNull = 1 THEN COALESCE(@EntryLocationID, EntryLocationID) ELSE @EntryLocationID END, + ExitLocationID = CASE WHEN @PreserveExistingWhenNull = 1 THEN COALESCE(@ExitLocationID, ExitLocationID) ELSE @ExitLocationID END, + AppearanceNotes = CASE WHEN @PreserveExistingWhenNull = 1 THEN COALESCE(@AppearanceNotes, AppearanceNotes) ELSE @AppearanceNotes END, + OutfitDescription = CASE WHEN @PreserveExistingWhenNull = 1 THEN COALESCE(@OutfitDescription, OutfitDescription) ELSE @OutfitDescription END, + PhysicalCondition = CASE WHEN @PreserveExistingWhenNull = 1 THEN COALESCE(@PhysicalCondition, PhysicalCondition) ELSE @PhysicalCondition END, + EmotionalState = CASE WHEN @PreserveExistingWhenNull = 1 THEN COALESCE(@EmotionalState, EmotionalState) ELSE @EmotionalState END, + KnowledgeNotes = CASE WHEN @PreserveExistingWhenNull = 1 THEN COALESCE(@KnowledgeNotes, KnowledgeNotes) ELSE @KnowledgeNotes END, + MatchSource = CASE WHEN @PreserveExistingWhenNull = 1 THEN COALESCE(@MatchSource, MatchSource) ELSE @MatchSource END, + MatchedText = CASE WHEN @PreserveExistingWhenNull = 1 THEN COALESCE(@MatchedText, MatchedText) ELSE @MatchedText END, + OccurrenceCount = CASE WHEN @PreserveExistingWhenNull = 1 THEN COALESCE(@OccurrenceCount, OccurrenceCount) ELSE @OccurrenceCount END, + FirstOccurrenceIndex = CASE WHEN @PreserveExistingWhenNull = 1 THEN COALESCE(@FirstOccurrenceIndex, FirstOccurrenceIndex) ELSE @FirstOccurrenceIndex END, + UpdatedDate = SYSUTCDATETIME() + WHERE SceneCharacterID = @SceneCharacterID; + END + + SELECT @SceneCharacterID; +END; +GO diff --git a/PlotLine/Views/Onboarding/ScanReview.cshtml b/PlotLine/Views/Onboarding/ScanReview.cshtml index 572bbc4..b9c15fa 100644 --- a/PlotLine/Views/Onboarding/ScanReview.cshtml +++ b/PlotLine/Views/Onboarding/ScanReview.cshtml @@ -1,25 +1,29 @@ @model ManuscriptScanReviewViewModel @{ ViewData["Title"] = "Review manuscript scan"; + var alreadyMatchedCharacters = Model.CharacterCandidates + .Where(candidate => candidate.IsExistingCharacterMatch) + .ToList(); var probableCharacters = Model.CharacterCandidates - .Where(candidate => string.Equals(candidate.Category, "ProbableCharacter", StringComparison.OrdinalIgnoreCase)) + .Where(candidate => !candidate.IsExistingCharacterMatch && string.Equals(candidate.Category, "ProbableCharacter", StringComparison.OrdinalIgnoreCase)) .ToList(); var possibleCharacters = Model.CharacterCandidates - .Where(candidate => string.Equals(candidate.Category, "PossibleCharacter", StringComparison.OrdinalIgnoreCase)) + .Where(candidate => !candidate.IsExistingCharacterMatch && string.Equals(candidate.Category, "PossibleCharacter", StringComparison.OrdinalIgnoreCase)) .ToList(); var relationshipTitles = Model.CharacterCandidates - .Where(candidate => string.Equals(candidate.Category, "RelationshipTitle", StringComparison.OrdinalIgnoreCase)) + .Where(candidate => !candidate.IsExistingCharacterMatch && string.Equals(candidate.Category, "RelationshipTitle", StringComparison.OrdinalIgnoreCase)) .ToList(); var primaryCharacters = probableCharacters .Concat(possibleCharacters) .Concat(relationshipTitles) + .Concat(alreadyMatchedCharacters) .OrderBy(candidate => candidate.ReviewName) .ThenBy(candidate => candidate.Name) .ToList(); var detectedAliasTargets = primaryCharacters .OrderBy(candidate => candidate.ReviewName) .ToList(); - var visibleCharacterCount = probableCharacters.Count + possibleCharacters.Count + relationshipTitles.Count; + var visibleCharacterCount = probableCharacters.Count + possibleCharacters.Count + relationshipTitles.Count + alreadyMatchedCharacters.Count; var isCharacterStep = string.Equals(ViewData["ScanReviewStep"] as string, "characters", StringComparison.OrdinalIgnoreCase); const int characterPageSize = 10; var primaryCharacterPageCount = Math.Max(1, (int)Math.Ceiling(primaryCharacters.Count / (double)characterPageSize)); @@ -214,7 +218,7 @@
- Primary candidates + Detected names @visibleCharacterCount.ToString("N0")
@@ -237,9 +241,10 @@
@foreach (var group in new[] { - new { Title = "Probable characters", Candidates = currentCharacters.Where(candidate => string.Equals(candidate.Category, "ProbableCharacter", StringComparison.OrdinalIgnoreCase)).ToList() }, - new { Title = "Possible characters", Candidates = currentCharacters.Where(candidate => string.Equals(candidate.Category, "PossibleCharacter", StringComparison.OrdinalIgnoreCase)).ToList() }, - new { Title = "Relationship titles", Candidates = currentCharacters.Where(candidate => string.Equals(candidate.Category, "RelationshipTitle", StringComparison.OrdinalIgnoreCase)).ToList() } + new { Title = "Already matched known characters", Candidates = currentCharacters.Where(candidate => candidate.IsExistingCharacterMatch).ToList() }, + new { Title = "Likely new characters", Candidates = currentCharacters.Where(candidate => !candidate.IsExistingCharacterMatch && string.Equals(candidate.Category, "ProbableCharacter", StringComparison.OrdinalIgnoreCase)).ToList() }, + new { Title = "Possible new names or aliases", Candidates = currentCharacters.Where(candidate => !candidate.IsExistingCharacterMatch && string.Equals(candidate.Category, "PossibleCharacter", StringComparison.OrdinalIgnoreCase)).ToList() }, + new { Title = "Generic or ambiguous references", Candidates = currentCharacters.Where(candidate => !candidate.IsExistingCharacterMatch && string.Equals(candidate.Category, "RelationshipTitle", StringComparison.OrdinalIgnoreCase)).ToList() } }) { if (!group.Candidates.Any())