diff --git a/PlotLine.Tests/Program.cs b/PlotLine.Tests/Program.cs index e9b1af0..cc282c3 100644 --- a/PlotLine.Tests/Program.cs +++ b/PlotLine.Tests/Program.cs @@ -10,7 +10,9 @@ var tests = new (string Name, Action Test)[] ("Truncated JSON is rejected", RejectsTruncatedJson), ("Repaired JSON deserialises into SceneIntelligenceScene", RepairedJsonDeserialises), ("Chapter Structure confidence 0. repairs and deserialises", ChapterStructureConfidenceRepairs), - ("Shared parser reports raw chapter output on unrecoverable JSON", SharedParserReportsRawOutput) + ("Shared parser reports raw chapter output on unrecoverable JSON", SharedParserReportsRawOutput), + ("Character filtering rejects generic groups", CharacterFilteringRejectsGenericGroups), + ("Character filtering preserves titled names", CharacterFilteringPreservesTitledNames) }; foreach (var test in tests) @@ -106,6 +108,42 @@ static void SharedParserReportsRawOutput() Assert(result.RawJson == raw, "Raw malformed output was not retained."); } +static void CharacterFilteringRejectsGenericGroups() +{ + Assert(IsGenericGroupReference("four examiners"), "Quantity + generic group should be rejected."); + Assert(IsGenericGroupReference("groups of men"), "Generic group phrase should be rejected."); + Assert(IsGenericGroupReference("neighbours"), "Generic plural group should be rejected."); + Assert(IsGenericGroupReference("passenger (blue Volkswagen Golf GTi)"), "Unnamed passenger descriptor should be rejected."); + Assert(IsGenericGroupReference("tubby lad"), "Unnamed descriptive person should be rejected."); + Assert(IsGenericGroupReference("lady examiner"), "Unnamed role descriptor should be rejected."); + Assert(IsGenericGroupReference("her mother"), "Determiner + family role should be rejected."); + Assert(IsGenericGroupReference("the child"), "Determiner + generic person should be rejected."); +} + +static void CharacterFilteringPreservesTitledNames() +{ + Assert(!IsGenericGroupReference("Reverend Potter"), "Reverend Potter should not be treated as generic."); + Assert(IsNamedOrTitledPerson("Reverend Potter"), "Reverend Potter should be recognised as a titled person."); + Assert(IsNamedOrTitledPerson("Detective Sergeant Summerhill"), "Detective Sergeant Summerhill should be recognised as a titled person."); + Assert(IsNamedOrTitledPerson("Mrs Patterson"), "Mrs Patterson should be recognised as a titled person."); + Assert(IsNamedOrTitledPerson("Aunt Elen"), "Aunt Elen should be recognised as a titled person."); +} + +static bool IsGenericGroupReference(string name) + => InvokePrivateCharacterFilter("IsGenericGroupReference", name); + +static bool IsNamedOrTitledPerson(string name) + => InvokePrivateCharacterFilter("IsNamedOrTitledPerson", name); + +static bool InvokePrivateCharacterFilter(string methodName, string name) +{ + var method = typeof(StoryIntelligenceCharacterImportService).GetMethod( + methodName, + System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Static); + Assert(method is not null, $"{methodName} was not found."); + return method!.Invoke(null, [name]) is true; +} + static JsonSerializerOptions JsonOptions() => new() { diff --git a/PlotLine/Services/OnboardingStoryIntelligenceService.cs b/PlotLine/Services/OnboardingStoryIntelligenceService.cs index 007e258..fe3f55b 100644 --- a/PlotLine/Services/OnboardingStoryIntelligenceService.cs +++ b/PlotLine/Services/OnboardingStoryIntelligenceService.cs @@ -370,6 +370,7 @@ public sealed class OnboardingStoryIntelligenceService( BookID = batch.BookID, CharactersCreated = batch.LastCharacterImportResult?.CharactersCreated ?? 0, CharactersLinked = batch.LastCharacterImportResult?.CharactersLinked ?? 0, + CharactersAliased = batch.LastCharacterImportResult?.CharactersAliased ?? 0, CharactersIgnored = batch.LastCharacterImportResult?.CharactersIgnored ?? 0, PovLinksResolved = batch.LastCharacterImportResult?.PovLinksResolved ?? 0, ScenePeoplePanelsUpdated = batch.LastCharacterImportResult?.ScenePeoplePanelsUpdated ?? 0 @@ -711,6 +712,7 @@ public sealed class StoryIntelligenceCharacterImportBatchResult { public int CharactersCreated { get; init; } public int CharactersLinked { get; init; } + public int CharactersAliased { get; init; } public int CharactersIgnored { get; init; } public int PovLinksResolved { get; init; } public int ScenePeoplePanelsUpdated { get; init; } diff --git a/PlotLine/Services/StoryIntelligenceCharacterImportService.cs b/PlotLine/Services/StoryIntelligenceCharacterImportService.cs index 03332b9..fb5fbc9 100644 --- a/PlotLine/Services/StoryIntelligenceCharacterImportService.cs +++ b/PlotLine/Services/StoryIntelligenceCharacterImportService.cs @@ -1,4 +1,5 @@ using System.Text.Json; +using System.Text.RegularExpressions; using PlotLine.Data; using PlotLine.Models; using PlotLine.ViewModels; @@ -31,11 +32,38 @@ public sealed class StoryIntelligenceCharacterImportService( private static readonly HashSet GenericPeople = CreateSet( "man", "woman", "boy", "girl", "child", "children", "person", "people", "doctor", "nurse", "police officer", "driver", "receptionist", "attendant", "guard", "security guard", "waiter", "waitress", - "clerk", "shopkeeper", "cashier", "crowd", "family", "group", "small group", "police"); + "clerk", "shopkeeper", "cashier", "crowd", "family", "group", "small group", "police", "officer", + "officers", "staff", "customers", "boys", "girls", "men", "women", "neighbour", "neighbours", + "passenger", "passengers", "examiner", "examiners", "instructor", "instructors", "candidate", "candidates", + "social worker", "social workers", "responders"); + + private static readonly HashSet GenericGroupHeads = CreateSet( + "man", "men", "woman", "women", "boy", "boys", "girl", "girls", "child", "children", "person", "people", + "group", "crowd", "family", "neighbour", "neighbours", "passenger", "passengers", "examiner", "examiners", + "driver", "drivers", "instructor", "instructors", "candidate", "candidates", "officer", "officers", "staff", "customers", + "responders", "worker", "workers", "lad", "lads"); + + private static readonly HashSet QuantityWords = CreateSet( + "one", "two", "three", "four", "five", "six", "seven", "eight", "nine", "ten", "several", "some", + "many", "few", "couple", "various", "other", "groups"); + + private static readonly HashSet DeterminerWords = CreateSet( + "the", "a", "an", "my", "your", "his", "her", "our", "their"); + + private static readonly HashSet FamilyRoleNames = CreateSet( + "aunt", "uncle", "mother", "father", "mum", "mom", "mam", "dad", "brother", "sister", "grandmother", + "grandfather", "grandma", "grandad", "granddad"); + + private static readonly HashSet CharacterTitlePrefixes = CreateSet( + "mr", "mrs", "miss", "ms", "dr", "reverend", "rev", "detective", "sergeant", "ds", "aunt", "uncle"); private static readonly HashSet NonCharacterNames = CreateSet( "christmas", "house", "street", "door", "window", "car", "church", "building", "lobby", "room", - "hallway", "kitchen", "garage", "pavement", "road", "desk", "table", "phone", "letter", "note"); + "hallway", "kitchen", "garage", "pavement", "road", "desk", "table", "phone", "letter", "note", + "social services", "under-flyover / link road"); + + private static readonly Regex ParentheticalSuffix = new(@"\s*\([^)]*\)\s*$", RegexOptions.Compiled); + private static readonly Regex LeadingQuantity = new(@"^\s*(\d+|one|two|three|four|five|six|seven|eight|nine|ten|several|some|many|few|couple|various|other)\s+", RegexOptions.IgnoreCase | RegexOptions.Compiled); public async Task BuildReviewAsync(OnboardingStoryIntelligenceBatch batch) { @@ -86,9 +114,12 @@ public sealed class StoryIntelligenceCharacterImportService( var created = 0; var linkedExisting = 0; var ignored = 0; + var aliased = 0; var linkedAppearances = 0; var povLinks = 0; var resolvedCharacters = new Dictionary(StringComparer.OrdinalIgnoreCase); + var resolvedCandidateIds = new Dictionary(StringComparer.OrdinalIgnoreCase); + var candidateByKey = data.Candidates.ToDictionary(candidate => candidate.Key, StringComparer.OrdinalIgnoreCase); foreach (var candidate in data.Candidates) { @@ -104,6 +135,11 @@ public sealed class StoryIntelligenceCharacterImportService( continue; } + if (string.Equals(choice.Action, StoryIntelligenceCharacterImportActions.Alias, StringComparison.OrdinalIgnoreCase)) + { + continue; + } + var requestedName = Clean(choice.ImportName); var importName = string.IsNullOrWhiteSpace(requestedName) ? candidate.DisplayName : requestedName; var forceCreateSeparate = string.Equals(choice.Action, StoryIntelligenceCharacterImportActions.CreateSeparate, StringComparison.OrdinalIgnoreCase); @@ -151,9 +187,60 @@ public sealed class StoryIntelligenceCharacterImportService( linkedAppearances++; } + resolvedCandidateIds[candidate.Key] = characterId.Value; AddDecision(batch, candidate.Key, choice.Action, importName, characterId.Value, true); } + foreach (var candidate in data.Candidates) + { + if (!choices.TryGetValue(candidate.Key, out var choice) + || !string.Equals(choice.Action, StoryIntelligenceCharacterImportActions.Alias, StringComparison.OrdinalIgnoreCase)) + { + continue; + } + + var targetKey = Clean(choice.AliasTargetKey); + if (string.IsNullOrWhiteSpace(targetKey) + || string.Equals(targetKey, candidate.Key, StringComparison.OrdinalIgnoreCase) + || !choices.TryGetValue(targetKey, out var targetChoice) + || string.Equals(targetChoice.Action, StoryIntelligenceCharacterImportActions.Ignore, StringComparison.OrdinalIgnoreCase) + || string.Equals(targetChoice.Action, StoryIntelligenceCharacterImportActions.Alias, StringComparison.OrdinalIgnoreCase) + || !candidateByKey.TryGetValue(targetKey, out var targetCandidate) + || !resolvedCandidateIds.TryGetValue(targetKey, out var targetCharacterId)) + { + logger.LogWarning( + "Story Intelligence character alias candidate {CandidateKey} could not be imported because target {TargetKey} was not a resolved create/link target.", + candidate.Key, + targetKey); + continue; + } + + var targetName = targetCandidate.ExistingCharacterName ?? targetChoice.ImportName ?? targetCandidate.DisplayName; + await TryAddAliasAsync(targetCharacterId, candidate.DisplayName, targetName); + AddResolvedName(resolvedCharacters, candidate.DisplayName, targetCharacterId); + foreach (var alias in candidate.Aliases) + { + await TryAddAliasAsync(targetCharacterId, alias, targetName); + AddResolvedName(resolvedCharacters, alias, targetCharacterId); + } + + foreach (var appearance in candidate.PresentAppearances.Concat(candidate.MentionedOnlyAppearances).Where(appearance => !appearance.AlreadyLinked)) + { + await characters.SaveSceneCharacterAsync(new SceneCharacter + { + SceneID = appearance.SceneID, + CharacterID = targetCharacterId, + RoleInSceneTypeID = MatchRole(roleTypes, appearance.RoleInScene), + PresenceTypeID = MatchPresence(presenceTypes, appearance.MentionedOnly), + AppearanceNotes = BuildAliasAppearanceNote(candidate.DisplayName, targetCandidate.DisplayName, appearance.Notes) + }); + linkedAppearances++; + } + + aliased++; + AddDecision(batch, candidate.Key, StoryIntelligenceCharacterImportActions.Alias, candidate.DisplayName, targetCharacterId, true); + } + foreach (var scene in data.SceneAnalyses) { var povName = Clean(scene.Parsed.PointOfView?.CharacterName); @@ -172,16 +259,18 @@ public sealed class StoryIntelligenceCharacterImportService( CharactersCreated = created, CharactersLinked = linkedExisting, CharactersIgnored = ignored, + CharactersAliased = aliased, PovLinksResolved = povLinks, ScenePeoplePanelsUpdated = linkedAppearances }; await pipelineState.RecordCharacterImportAsync(batch.ProjectID, batch.BookID); logger.LogInformation( - "Imported Story Intelligence characters for batch {BatchID}. Created={Created} LinkedExisting={LinkedExisting} Ignored={Ignored} LinkedAppearances={LinkedAppearances} PovLinks={PovLinks}", + "Imported Story Intelligence characters for batch {BatchID}. Created={Created} LinkedExisting={LinkedExisting} Aliased={Aliased} Ignored={Ignored} LinkedAppearances={LinkedAppearances} PovLinks={PovLinks}", batch.BatchID, created, linkedExisting, + aliased, ignored, linkedAppearances, povLinks); @@ -190,7 +279,7 @@ public sealed class StoryIntelligenceCharacterImportService( { Success = true, ScenesCreated = linkedAppearances, - Message = $"Characters imported. {created:N0} created, {linkedExisting:N0} linked, {ignored:N0} ignored, {linkedAppearances:N0} scene appearance(s) updated, {povLinks:N0} POV link(s) resolved." + Message = $"Characters imported. {created:N0} created, {linkedExisting:N0} linked, {aliased:N0} alias decision(s) applied, {ignored:N0} ignored, {linkedAppearances:N0} scene appearance(s) updated, {povLinks:N0} POV link(s) resolved." }; } @@ -258,6 +347,7 @@ public sealed class StoryIntelligenceCharacterImportService( var candidates = groups.Values .Where(candidate => candidate.PresentAppearances.Count > 0 || candidate.MentionedOnlyAppearances.Count > 0) .Where(candidate => candidate.PresentAppearances.Concat(candidate.MentionedOnlyAppearances).Any(appearance => !appearance.AlreadyLinked)) + .Where(IsVisibleCharacterCandidate) .OrderByDescending(candidate => candidate.PresentAppearances.Select(appearance => appearance.SceneID).Distinct().Count()) .ThenByDescending(candidate => candidate.MentionedOnlyAppearances.Select(appearance => appearance.SceneID).Distinct().Count()) .ThenBy(candidate => candidate.DisplayName) @@ -515,6 +605,12 @@ public sealed class StoryIntelligenceCharacterImportService( } } + private static string? BuildAliasAppearanceNote(string aliasName, string targetName, string? notes) + { + var prefix = $"Imported as alias '{aliasName}' of {targetName}."; + return string.IsNullOrWhiteSpace(notes) ? prefix : $"{prefix} {notes}"; + } + private static string? BuildFirstAppearance(Scene scene, SceneIntelligenceScene parsed, SceneIntelligenceCharacter character) { var note = Clean(character.Notes); @@ -551,6 +647,30 @@ public sealed class StoryIntelligenceCharacterImportService( private static bool IsCharacterEntity(string? entityType) => string.Equals(Clean(entityType), "Character", StringComparison.OrdinalIgnoreCase); + private static bool IsVisibleCharacterCandidate(CharacterCandidate candidate) + { + var name = CleanCharacterName(candidate.DisplayName); + if (!IsCharacterNameCandidate(name)) + { + return false; + } + + if (IsNamedOrTitledPerson(name)) + { + return true; + } + + if (IsFamilyRoleOnly(name)) + { + return candidate.PresentAppearances.Concat(candidate.MentionedOnlyAppearances) + .Select(appearance => appearance.SceneID) + .Distinct() + .Count() >= 2; + } + + return !IsGenericGroupReference(name); + } + private static bool IsCharacterNameCandidate(string? name) { var clean = CleanCharacterName(name); @@ -570,6 +690,80 @@ public sealed class StoryIntelligenceCharacterImportService( return true; } + private static bool IsGenericGroupReference(string? name) + { + var clean = CleanCharacterName(name); + if (string.IsNullOrWhiteSpace(clean)) + { + return true; + } + + var withoutParentheses = Clean(ParentheticalSuffix.Replace(clean, string.Empty)); + var normalised = Normalise(withoutParentheses); + if (GenericPeople.Contains(normalised) || NonCharacterNames.Contains(normalised)) + { + return true; + } + + var words = WordsForFiltering(withoutParentheses); + if (words.Count == 0) + { + return true; + } + + if (LeadingQuantity.IsMatch(withoutParentheses) + && words.Any(word => GenericGroupHeads.Contains(word))) + { + return true; + } + + if (words.Count > 1 + && words.Any(word => GenericGroupHeads.Contains(word)) + && !HasSpecificPossessiveFamilyReference(words)) + { + return true; + } + + if (words.Count > 1 + && DeterminerWords.Contains(words[0]) + && words.Skip(1).Any(word => FamilyRoleNames.Contains(word) || GenericGroupHeads.Contains(word))) + { + return true; + } + + if (words.Count > 1 + && QuantityWords.Contains(words[0]) + && words.Skip(1).Any(word => GenericGroupHeads.Contains(word))) + { + return true; + } + + return words.All(word => GenericGroupHeads.Contains(word) || QuantityWords.Contains(word)); + } + + private static bool IsNamedOrTitledPerson(string name) + { + var words = WordsForFiltering(name); + if (words.Count == 0) + { + return false; + } + + return CharacterTitlePrefixes.Contains(words[0]) && words.Count > 1; + } + + private static bool IsFamilyRoleOnly(string name) + { + var words = WordsForFiltering(name); + return words.Count == 1 && FamilyRoleNames.Contains(words[0]); + } + + private static bool HasSpecificPossessiveFamilyReference(IReadOnlyList words) + => words.Count == 2 + && FamilyRoleNames.Contains(words[1]) + && words[0].EndsWith("s", StringComparison.OrdinalIgnoreCase) + && !DeterminerWords.Contains(words[0]); + private static string ResolveCandidateKey( IReadOnlyDictionary groups, CharacterIndex existingIndex, @@ -636,6 +830,15 @@ public sealed class StoryIntelligenceCharacterImportService( return words; } + private static List WordsForFiltering(string? name) + => CleanCharacterName(name) + .Replace("/", " ", StringComparison.Ordinal) + .Replace("-", " ", StringComparison.Ordinal) + .Split(' ', StringSplitOptions.RemoveEmptyEntries) + .Select(word => word.Trim('.', ',', ';', ':', '!', '?', '"', '\'', '(', ')').ToLowerInvariant()) + .Where(word => !string.IsNullOrWhiteSpace(word)) + .ToList(); + private static bool IsMergeableTitle(string value) => value.Equals("instructor", StringComparison.OrdinalIgnoreCase) || value.Equals("detective", StringComparison.OrdinalIgnoreCase) diff --git a/PlotLine/ViewModels/OnboardingViewModels.cs b/PlotLine/ViewModels/OnboardingViewModels.cs index 0150c90..e513efc 100644 --- a/PlotLine/ViewModels/OnboardingViewModels.cs +++ b/PlotLine/ViewModels/OnboardingViewModels.cs @@ -204,6 +204,7 @@ public sealed class StoryIntelligenceCharacterImportChoiceForm public string Key { get; set; } = string.Empty; public string Action { get; set; } = StoryIntelligenceCharacterImportActions.Approve; public string? ImportName { get; set; } + public string? AliasTargetKey { get; set; } } public static class StoryIntelligenceCharacterImportActions @@ -211,6 +212,7 @@ public static class StoryIntelligenceCharacterImportActions public const string Approve = "Approve"; public const string Ignore = "Ignore"; public const string CreateSeparate = "CreateSeparate"; + public const string Alias = "Alias"; } public sealed class StoryIntelligenceCompletionViewModel @@ -235,6 +237,7 @@ public sealed class StoryIntelligenceCharacterImportResultViewModel public int BookID { get; init; } public int CharactersCreated { get; init; } public int CharactersLinked { get; init; } + public int CharactersAliased { get; init; } public int CharactersIgnored { get; init; } public int PovLinksResolved { get; init; } public int ScenePeoplePanelsUpdated { get; init; } diff --git a/PlotLine/Views/Onboarding/StoryIntelligenceCharacterComplete.cshtml b/PlotLine/Views/Onboarding/StoryIntelligenceCharacterComplete.cshtml index cbe4b4f..f5aea2a 100644 --- a/PlotLine/Views/Onboarding/StoryIntelligenceCharacterComplete.cshtml +++ b/PlotLine/Views/Onboarding/StoryIntelligenceCharacterComplete.cshtml @@ -24,6 +24,10 @@ Characters linked @Model.CharactersLinked.ToString("N0") +
+ Aliases added + @Model.CharactersAliased.ToString("N0") +
Characters ignored @Model.CharactersIgnored.ToString("N0") diff --git a/PlotLine/Views/Onboarding/StoryIntelligenceCharacters.cshtml b/PlotLine/Views/Onboarding/StoryIntelligenceCharacters.cshtml index 1f64a62..d37d271 100644 --- a/PlotLine/Views/Onboarding/StoryIntelligenceCharacters.cshtml +++ b/PlotLine/Views/Onboarding/StoryIntelligenceCharacters.cshtml @@ -69,7 +69,7 @@ -
+
First appearance
@Display(candidate.ExampleFirstAppearance)
Possible aliases
@(candidate.PossibleAliases.Count == 0 ? "None detected" : string.Join(", ", candidate.PossibleAliases))
@@ -101,10 +101,27 @@ Ignore + - - +
+ + +
+ +
} @@ -129,7 +146,44 @@ const cards = () => Array.from(form.querySelectorAll("[data-character-card]")); const setAction = (card, action) => { const input = card.querySelector(`[data-character-action][value="${action}"]`); - if (input) input.checked = true; + if (input) { + input.checked = true; + updateCard(card); + } + }; + const actionFor = (card) => card.querySelector("[data-character-action]:checked")?.value || "@StoryIntelligenceCharacterImportActions.Approve"; + const isCanonicalTarget = (card) => { + const action = actionFor(card); + return action !== "@StoryIntelligenceCharacterImportActions.Ignore" && action !== "@StoryIntelligenceCharacterImportActions.Alias"; + }; + const updateAliasTargets = () => { + const allCards = cards(); + for (const card of allCards) { + const body = card.querySelector("[data-character-key]"); + const ownKey = body?.getAttribute("data-character-key") || ""; + const select = card.querySelector("[data-character-alias-target]"); + if (!select) continue; + + const previous = select.value; + for (const option of Array.from(select.options)) { + if (!option.value) continue; + const targetCard = allCards.find(item => item.querySelector("[data-character-key]")?.getAttribute("data-character-key") === option.value); + option.disabled = option.value === ownKey || !targetCard || !isCanonicalTarget(targetCard); + } + if (select.selectedOptions[0]?.disabled) { + select.value = ""; + } else { + select.value = previous; + } + } + }; + const updateCard = (card) => { + const action = actionFor(card); + const namePanel = card.querySelector("[data-character-import-name-panel]"); + const aliasPanel = card.querySelector("[data-character-alias-panel]"); + if (namePanel) namePanel.hidden = action === "@StoryIntelligenceCharacterImportActions.Ignore" || action === "@StoryIntelligenceCharacterImportActions.Alias"; + if (aliasPanel) aliasPanel.hidden = action !== "@StoryIntelligenceCharacterImportActions.Alias"; + updateAliasTargets(); }; document.querySelectorAll("[data-character-bulk]").forEach(button => { button.addEventListener("click", () => { @@ -142,6 +196,10 @@ if (action === "collapse") cards().forEach(card => card.open = false); }); }); + form.querySelectorAll("[data-character-action]").forEach(input => { + input.addEventListener("change", () => updateCard(input.closest("[data-character-card]"))); + }); + cards().forEach(updateCard); form.addEventListener("submit", () => { cards() .filter(card => !card.querySelector("[data-character-selected]")?.checked)