From 3cf68740dd4b561d9addf50e15f44122ec39d483 Mon Sep 17 00:00:00 2001 From: Nick Beckley Date: Wed, 8 Jul 2026 21:39:34 +0100 Subject: [PATCH] =?UTF-8?q?Phase=2020AK=20=E2=80=93=20Character=20Intellig?= =?UTF-8?q?ence=20(AI=20Extraction=20and=20Review)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ...StoryIntelligenceCharacterImportService.cs | 229 +++++++++++++++++- 1 file changed, 216 insertions(+), 13 deletions(-) diff --git a/PlotLine/Services/StoryIntelligenceCharacterImportService.cs b/PlotLine/Services/StoryIntelligenceCharacterImportService.cs index bdb89dc..03332b9 100644 --- a/PlotLine/Services/StoryIntelligenceCharacterImportService.cs +++ b/PlotLine/Services/StoryIntelligenceCharacterImportService.cs @@ -24,12 +24,18 @@ public sealed class StoryIntelligenceCharacterImportService( PropertyNameCaseInsensitive = true }; - private static readonly HashSet IgnoredNames = CreateSet("narrator", "unknown"); + private static readonly HashSet IgnoredNames = CreateSet( + "narrator", "unknown", "unknown narrator", "i", "me", "my", "mine", "we", "us", "our", "ours", + "you", "he", "him", "his", "she", "her", "hers", "they", "them", "their", "theirs"); private static readonly HashSet GenericPeople = CreateSet( "man", "woman", "boy", "girl", "child", "children", "person", "people", "doctor", "nurse", "police officer", "driver", "receptionist", "attendant", "guard", "security guard", "waiter", "waitress", - "clerk", "shopkeeper", "cashier", "crowd", "family", "group", "small group"); + "clerk", "shopkeeper", "cashier", "crowd", "family", "group", "small group", "police"); + + private static readonly HashSet NonCharacterNames = CreateSet( + "christmas", "house", "street", "door", "window", "car", "church", "building", "lobby", "room", + "hallway", "kitchen", "garage", "pavement", "road", "desk", "table", "phone", "letter", "note"); public async Task BuildReviewAsync(OnboardingStoryIntelligenceBatch batch) { @@ -132,7 +138,7 @@ public sealed class StoryIntelligenceCharacterImportService( await TryAddAliasAsync(characterId.Value, alias, importName); } - foreach (var appearance in candidate.PresentAppearances) + foreach (var appearance in candidate.PresentAppearances.Concat(candidate.MentionedOnlyAppearances).Where(appearance => !appearance.AlreadyLinked)) { await characters.SaveSceneCharacterAsync(new SceneCharacter { @@ -245,17 +251,15 @@ public sealed class StoryIntelligenceCharacterImportService( var existingAppearances = await characters.ListSceneCharactersAsync(importedScene.SceneID); alreadyLinked += existingAppearances.Count; - foreach (var character in parsed.Characters ?? []) - { - AddMention(groups, existingIndex, importedScene, parsed, character, existingAppearances); - } + AddSceneCharacterMentions(groups, existingIndex, importedScene, parsed, existingAppearances); } } var candidates = groups.Values .Where(candidate => candidate.PresentAppearances.Count > 0 || candidate.MentionedOnlyAppearances.Count > 0) - .Where(candidate => candidate.PresentAppearances.Any(appearance => !appearance.AlreadyLinked)) + .Where(candidate => candidate.PresentAppearances.Concat(candidate.MentionedOnlyAppearances).Any(appearance => !appearance.AlreadyLinked)) .OrderByDescending(candidate => candidate.PresentAppearances.Select(appearance => appearance.SceneID).Distinct().Count()) + .ThenByDescending(candidate => candidate.MentionedOnlyAppearances.Select(appearance => appearance.SceneID).Distinct().Count()) .ThenBy(candidate => candidate.DisplayName) .ToList(); @@ -270,24 +274,28 @@ public sealed class StoryIntelligenceCharacterImportService( SceneIntelligenceCharacter character, IReadOnlyList existingAppearances) { - var name = Clean(character.Name); - if (string.IsNullOrWhiteSpace(name) || IgnoredNames.Contains(name) || IsGenericPerson(name)) + var name = CleanCharacterName(character.Name); + if (!IsCharacterNameCandidate(name)) { return; } - var key = Normalise(name); + var key = ResolveCandidateKey(groups, existingIndex, name); if (!groups.TryGetValue(key, out var candidate)) { var match = existingIndex.Find(name); candidate = new CharacterCandidate(key, name, match?.CharacterID, match?.CharacterName); groups[key] = candidate; } + else if (!string.Equals(candidate.DisplayName, name, StringComparison.OrdinalIgnoreCase)) + { + candidate.Aliases.Add(name); + } foreach (var alias in character.Aliases ?? []) { - var cleanAlias = Clean(alias); - if (!string.IsNullOrWhiteSpace(cleanAlias) && !string.Equals(cleanAlias, candidate.DisplayName, StringComparison.OrdinalIgnoreCase)) + var cleanAlias = CleanCharacterName(alias); + if (IsCharacterNameCandidate(cleanAlias) && !string.Equals(cleanAlias, candidate.DisplayName, StringComparison.OrdinalIgnoreCase)) { candidate.Aliases.Add(cleanAlias); } @@ -322,16 +330,97 @@ public sealed class StoryIntelligenceCharacterImportService( candidate.FirstAppearanceNote ??= BuildFirstAppearance(importedScene, parsed, character); } + private static void AddSceneCharacterMentions( + Dictionary groups, + CharacterIndex existingIndex, + Scene importedScene, + SceneIntelligenceScene parsed, + IReadOnlyList existingAppearances) + { + foreach (var character in parsed.Characters ?? []) + { + AddMention(groups, existingIndex, importedScene, parsed, character, existingAppearances); + } + + var povName = CleanCharacterName(parsed.PointOfView?.CharacterName); + if (IsCharacterNameCandidate(povName)) + { + AddMention(groups, existingIndex, importedScene, parsed, new SceneIntelligenceCharacter + { + Name = povName, + RoleInScene = "POV", + MentionedOnly = false, + Confidence = parsed.PointOfView?.Confidence ?? 0.7m, + Notes = "Point of view candidate from Scene Intelligence." + }, existingAppearances); + } + + foreach (var relationship in parsed.Relationships ?? []) + { + AddObservedName(groups, existingIndex, importedScene, parsed, relationship.CharacterA, "Relationship participant", false, relationship.Confidence, relationship.RelationshipSignal, existingAppearances); + AddObservedName(groups, existingIndex, importedScene, parsed, relationship.CharacterB, "Relationship participant", false, relationship.Confidence, relationship.RelationshipSignal, existingAppearances); + } + + foreach (var knowledge in parsed.KnowledgeChanges ?? []) + { + AddObservedName(groups, existingIndex, importedScene, parsed, knowledge.RecipientCharacter, "Knowledge recipient", false, knowledge.Confidence, knowledge.KnowledgeItem, existingAppearances); + AddObservedName(groups, existingIndex, importedScene, parsed, knowledge.SourceCharacter, "Knowledge source", true, knowledge.Confidence, knowledge.KnowledgeItem, existingAppearances); + } + + foreach (var observation in parsed.Observations ?? []) + { + if (IsCharacterEntity(observation.SubjectEntityType)) + { + AddObservedName(groups, existingIndex, importedScene, parsed, observation.SubjectName, observation.ObservationType, false, observation.Confidence, observation.Description, existingAppearances); + } + + if (IsCharacterEntity(observation.ObjectEntityType)) + { + AddObservedName(groups, existingIndex, importedScene, parsed, observation.ObjectName, observation.ObservationType, true, observation.Confidence, observation.Description, existingAppearances); + } + } + } + + private static void AddObservedName( + Dictionary groups, + CharacterIndex existingIndex, + Scene importedScene, + SceneIntelligenceScene parsed, + string? name, + string? role, + bool mentionedOnly, + decimal? confidence, + string? notes, + IReadOnlyList existingAppearances) + { + var cleanName = CleanCharacterName(name); + if (!IsCharacterNameCandidate(cleanName)) + { + return; + } + + AddMention(groups, existingIndex, importedScene, parsed, new SceneIntelligenceCharacter + { + Name = cleanName, + RoleInScene = role, + MentionedOnly = mentionedOnly, + Confidence = confidence ?? 0.65m, + Notes = notes + }, existingAppearances); + } + private async Task BuildCharacterIndexAsync(int projectId) { var index = new CharacterIndex(); foreach (var character in await characters.ListCharactersAsync(projectId)) { index.Add(character.CharacterName, character.CharacterID, character.CharacterName); + index.Add(FirstName(character.CharacterName), character.CharacterID, character.CharacterName); index.Add(character.ShortName, character.CharacterID, character.CharacterName); foreach (var alias in await characters.ListAliasesAsync(character.CharacterID)) { index.Add(alias.Alias, character.CharacterID, character.CharacterName); + index.Add(FirstName(alias.Alias), character.CharacterID, character.CharacterName); } } @@ -459,6 +548,120 @@ public sealed class StoryIntelligenceCharacterImportService( private static bool IsGenericPerson(string name) => GenericPeople.Contains(name) || GenericPeople.Contains(Normalise(name)); + private static bool IsCharacterEntity(string? entityType) + => string.Equals(Clean(entityType), "Character", StringComparison.OrdinalIgnoreCase); + + private static bool IsCharacterNameCandidate(string? name) + { + var clean = CleanCharacterName(name); + if (string.IsNullOrWhiteSpace(clean) + || IgnoredNames.Contains(clean) + || IsGenericPerson(clean) + || NonCharacterNames.Contains(clean)) + { + return false; + } + + if (!clean.Any(char.IsLetter) || clean.Length < 2) + { + return false; + } + + return true; + } + + private static string ResolveCandidateKey( + IReadOnlyDictionary groups, + CharacterIndex existingIndex, + string name) + { + var canonical = CanonicalCharacterKey(name); + if (groups.ContainsKey(canonical)) + { + return canonical; + } + + var existingMatch = existingIndex.Find(name); + if (existingMatch is not null) + { + var existingKey = CanonicalCharacterKey(existingMatch.CharacterName); + if (groups.ContainsKey(existingKey)) + { + return existingKey; + } + } + + var first = FirstNameWithoutTitle(name); + if (!string.IsNullOrWhiteSpace(first)) + { + foreach (var candidate in groups.Values) + { + if (string.Equals(FirstNameWithoutTitle(candidate.DisplayName), first, StringComparison.OrdinalIgnoreCase) + && IsSafeFirstNameMerge(candidate.DisplayName, name)) + { + return candidate.Key; + } + } + } + + return canonical; + } + + private static bool IsSafeFirstNameMerge(string current, string next) + { + var currentWords = WordsWithoutTitles(current); + var nextWords = WordsWithoutTitles(next); + return currentWords.Count == 1 || nextWords.Count == 1 || currentWords.SequenceEqual(nextWords, StringComparer.OrdinalIgnoreCase); + } + + private static string CanonicalCharacterKey(string name) + => Normalise(string.Join(' ', WordsWithoutTitles(CleanCharacterName(name)))); + + private static string? FirstNameWithoutTitle(string? name) + => WordsWithoutTitles(name).FirstOrDefault(); + + private static List WordsWithoutTitles(string? name) + { + var words = CleanCharacterName(name) + .Split(' ', StringSplitOptions.RemoveEmptyEntries) + .Select(word => word.Trim('.', ',', ';', ':', '!', '?', '"', '\'')) + .Where(word => !string.IsNullOrWhiteSpace(word)) + .ToList(); + + if (words.Count > 1 && IsMergeableTitle(words[0])) + { + words.RemoveAt(0); + } + + return words; + } + + private static bool IsMergeableTitle(string value) + => value.Equals("instructor", StringComparison.OrdinalIgnoreCase) + || value.Equals("detective", StringComparison.OrdinalIgnoreCase) + || value.Equals("sergeant", StringComparison.OrdinalIgnoreCase) + || value.Equals("ds", StringComparison.OrdinalIgnoreCase); + + private static string CleanCharacterName(string? value) + { + var clean = Clean(value) + .Trim(' ', '.', ',', ';', ':', '!', '?', '"', '\''); + + if (clean.EndsWith("'s", StringComparison.OrdinalIgnoreCase)) + { + clean = clean[..^2]; + } + else if (clean.EndsWith("’s", StringComparison.OrdinalIgnoreCase)) + { + clean = clean[..^2]; + } + + return Clean(clean.Replace("Mrs.", "Mrs", StringComparison.OrdinalIgnoreCase) + .Replace("Mr.", "Mr", StringComparison.OrdinalIgnoreCase) + .Replace("Ms.", "Ms", StringComparison.OrdinalIgnoreCase) + .Replace("Dr.", "Dr", StringComparison.OrdinalIgnoreCase)); + } + private static string Normalise(string? value) => Clean(value).ToLowerInvariant();