Phase 20AK – Character Intelligence (AI Extraction and Review)

PlotLine/Services/StoryIntelligenceCharacterImportService.cs

@@ -24,12 +24,18 @@ public sealed class StoryIntelligenceCharacterImportService(
         PropertyNameCaseInsensitive = true
     };
 
-    private static readonly HashSet<string> IgnoredNames = CreateSet("narrator", "unknown");
+    private static readonly HashSet<string> IgnoredNames = CreateSet(
+        "narrator", "unknown", "unknown narrator", "i", "me", "my", "mine", "we", "us", "our", "ours",
+        "you", "he", "him", "his", "she", "her", "hers", "they", "them", "their", "theirs");
 
     private static readonly HashSet<string> GenericPeople = CreateSet(
         "man", "woman", "boy", "girl", "child", "children", "person", "people", "doctor", "nurse",
         "police officer", "driver", "receptionist", "attendant", "guard", "security guard", "waiter", "waitress",
-        "clerk", "shopkeeper", "cashier", "crowd", "family", "group", "small group");
+        "clerk", "shopkeeper", "cashier", "crowd", "family", "group", "small group", "police");
+
+    private static readonly HashSet<string> NonCharacterNames = CreateSet(
+        "christmas", "house", "street", "door", "window", "car", "church", "building", "lobby", "room",
+        "hallway", "kitchen", "garage", "pavement", "road", "desk", "table", "phone", "letter", "note");
 
     public async Task<StoryIntelligenceCharacterReviewViewModel> BuildReviewAsync(OnboardingStoryIntelligenceBatch batch)
     {
@@ -132,7 +138,7 @@ public sealed class StoryIntelligenceCharacterImportService(
                 await TryAddAliasAsync(characterId.Value, alias, importName);
             }
 
-            foreach (var appearance in candidate.PresentAppearances)
+            foreach (var appearance in candidate.PresentAppearances.Concat(candidate.MentionedOnlyAppearances).Where(appearance => !appearance.AlreadyLinked))
             {
                 await characters.SaveSceneCharacterAsync(new SceneCharacter
                 {
@@ -245,17 +251,15 @@ public sealed class StoryIntelligenceCharacterImportService(
                 var existingAppearances = await characters.ListSceneCharactersAsync(importedScene.SceneID);
                 alreadyLinked += existingAppearances.Count;
 
-                foreach (var character in parsed.Characters ?? [])
+                AddSceneCharacterMentions(groups, existingIndex, importedScene, parsed, existingAppearances);
-                {
-                    AddMention(groups, existingIndex, importedScene, parsed, character, existingAppearances);
-                }
             }
         }
 
         var candidates = groups.Values
             .Where(candidate => candidate.PresentAppearances.Count > 0 || candidate.MentionedOnlyAppearances.Count > 0)
-            .Where(candidate => candidate.PresentAppearances.Any(appearance => !appearance.AlreadyLinked))
+            .Where(candidate => candidate.PresentAppearances.Concat(candidate.MentionedOnlyAppearances).Any(appearance => !appearance.AlreadyLinked))
             .OrderByDescending(candidate => candidate.PresentAppearances.Select(appearance => appearance.SceneID).Distinct().Count())
+            .ThenByDescending(candidate => candidate.MentionedOnlyAppearances.Select(appearance => appearance.SceneID).Distinct().Count())
             .ThenBy(candidate => candidate.DisplayName)
             .ToList();
 
@@ -270,24 +274,28 @@ public sealed class StoryIntelligenceCharacterImportService(
         SceneIntelligenceCharacter character,
         IReadOnlyList<SceneCharacter> existingAppearances)
     {
-        var name = Clean(character.Name);
+        var name = CleanCharacterName(character.Name);
-        if (string.IsNullOrWhiteSpace(name) || IgnoredNames.Contains(name) || IsGenericPerson(name))
+        if (!IsCharacterNameCandidate(name))
         {
             return;
         }
 
-        var key = Normalise(name);
+        var key = ResolveCandidateKey(groups, existingIndex, name);
         if (!groups.TryGetValue(key, out var candidate))
         {
             var match = existingIndex.Find(name);
             candidate = new CharacterCandidate(key, name, match?.CharacterID, match?.CharacterName);
             groups[key] = candidate;
         }
+        else if (!string.Equals(candidate.DisplayName, name, StringComparison.OrdinalIgnoreCase))
+        {
+            candidate.Aliases.Add(name);
+        }
 
         foreach (var alias in character.Aliases ?? [])
         {
-            var cleanAlias = Clean(alias);
+            var cleanAlias = CleanCharacterName(alias);
-            if (!string.IsNullOrWhiteSpace(cleanAlias) && !string.Equals(cleanAlias, candidate.DisplayName, StringComparison.OrdinalIgnoreCase))
+            if (IsCharacterNameCandidate(cleanAlias) && !string.Equals(cleanAlias, candidate.DisplayName, StringComparison.OrdinalIgnoreCase))
             {
                 candidate.Aliases.Add(cleanAlias);
             }
@@ -322,16 +330,97 @@ public sealed class StoryIntelligenceCharacterImportService(
         candidate.FirstAppearanceNote ??= BuildFirstAppearance(importedScene, parsed, character);
     }
 
+    private static void AddSceneCharacterMentions(
+        Dictionary<string, CharacterCandidate> groups,
+        CharacterIndex existingIndex,
+        Scene importedScene,
+        SceneIntelligenceScene parsed,
+        IReadOnlyList<SceneCharacter> existingAppearances)
+    {
+        foreach (var character in parsed.Characters ?? [])
+        {
+            AddMention(groups, existingIndex, importedScene, parsed, character, existingAppearances);
+        }
+
+        var povName = CleanCharacterName(parsed.PointOfView?.CharacterName);
+        if (IsCharacterNameCandidate(povName))
+        {
+            AddMention(groups, existingIndex, importedScene, parsed, new SceneIntelligenceCharacter
+            {
+                Name = povName,
+                RoleInScene = "POV",
+                MentionedOnly = false,
+                Confidence = parsed.PointOfView?.Confidence ?? 0.7m,
+                Notes = "Point of view candidate from Scene Intelligence."
+            }, existingAppearances);
+        }
+
+        foreach (var relationship in parsed.Relationships ?? [])
+        {
+            AddObservedName(groups, existingIndex, importedScene, parsed, relationship.CharacterA, "Relationship participant", false, relationship.Confidence, relationship.RelationshipSignal, existingAppearances);
+            AddObservedName(groups, existingIndex, importedScene, parsed, relationship.CharacterB, "Relationship participant", false, relationship.Confidence, relationship.RelationshipSignal, existingAppearances);
+        }
+
+        foreach (var knowledge in parsed.KnowledgeChanges ?? [])
+        {
+            AddObservedName(groups, existingIndex, importedScene, parsed, knowledge.RecipientCharacter, "Knowledge recipient", false, knowledge.Confidence, knowledge.KnowledgeItem, existingAppearances);
+            AddObservedName(groups, existingIndex, importedScene, parsed, knowledge.SourceCharacter, "Knowledge source", true, knowledge.Confidence, knowledge.KnowledgeItem, existingAppearances);
+        }
+
+        foreach (var observation in parsed.Observations ?? [])
+        {
+            if (IsCharacterEntity(observation.SubjectEntityType))
+            {
+                AddObservedName(groups, existingIndex, importedScene, parsed, observation.SubjectName, observation.ObservationType, false, observation.Confidence, observation.Description, existingAppearances);
+            }
+
+            if (IsCharacterEntity(observation.ObjectEntityType))
+            {
+                AddObservedName(groups, existingIndex, importedScene, parsed, observation.ObjectName, observation.ObservationType, true, observation.Confidence, observation.Description, existingAppearances);
+            }
+        }
+    }
+
+    private static void AddObservedName(
+        Dictionary<string, CharacterCandidate> groups,
+        CharacterIndex existingIndex,
+        Scene importedScene,
+        SceneIntelligenceScene parsed,
+        string? name,
+        string? role,
+        bool mentionedOnly,
+        decimal? confidence,
+        string? notes,
+        IReadOnlyList<SceneCharacter> existingAppearances)
+    {
+        var cleanName = CleanCharacterName(name);
+        if (!IsCharacterNameCandidate(cleanName))
+        {
+            return;
+        }
+
+        AddMention(groups, existingIndex, importedScene, parsed, new SceneIntelligenceCharacter
+        {
+            Name = cleanName,
+            RoleInScene = role,
+            MentionedOnly = mentionedOnly,
+            Confidence = confidence ?? 0.65m,
+            Notes = notes
+        }, existingAppearances);
+    }
+
     private async Task<CharacterIndex> BuildCharacterIndexAsync(int projectId)
     {
         var index = new CharacterIndex();
         foreach (var character in await characters.ListCharactersAsync(projectId))
         {
             index.Add(character.CharacterName, character.CharacterID, character.CharacterName);
+            index.Add(FirstName(character.CharacterName), character.CharacterID, character.CharacterName);
             index.Add(character.ShortName, character.CharacterID, character.CharacterName);
             foreach (var alias in await characters.ListAliasesAsync(character.CharacterID))
             {
                 index.Add(alias.Alias, character.CharacterID, character.CharacterName);
+                index.Add(FirstName(alias.Alias), character.CharacterID, character.CharacterName);
             }
         }
 
@@ -459,6 +548,120 @@ public sealed class StoryIntelligenceCharacterImportService(
     private static bool IsGenericPerson(string name)
         => GenericPeople.Contains(name) || GenericPeople.Contains(Normalise(name));
 
+    private static bool IsCharacterEntity(string? entityType)
+        => string.Equals(Clean(entityType), "Character", StringComparison.OrdinalIgnoreCase);
+
+    private static bool IsCharacterNameCandidate(string? name)
+    {
+        var clean = CleanCharacterName(name);
+        if (string.IsNullOrWhiteSpace(clean)
+            || IgnoredNames.Contains(clean)
+            || IsGenericPerson(clean)
+            || NonCharacterNames.Contains(clean))
+        {
+            return false;
+        }
+
+        if (!clean.Any(char.IsLetter) || clean.Length < 2)
+        {
+            return false;
+        }
+
+        return true;
+    }
+
+    private static string ResolveCandidateKey(
+        IReadOnlyDictionary<string, CharacterCandidate> groups,
+        CharacterIndex existingIndex,
+        string name)
+    {
+        var canonical = CanonicalCharacterKey(name);
+        if (groups.ContainsKey(canonical))
+        {
+            return canonical;
+        }
+
+        var existingMatch = existingIndex.Find(name);
+        if (existingMatch is not null)
+        {
+            var existingKey = CanonicalCharacterKey(existingMatch.CharacterName);
+            if (groups.ContainsKey(existingKey))
+            {
+                return existingKey;
+            }
+        }
+
+        var first = FirstNameWithoutTitle(name);
+        if (!string.IsNullOrWhiteSpace(first))
+        {
+            foreach (var candidate in groups.Values)
+            {
+                if (string.Equals(FirstNameWithoutTitle(candidate.DisplayName), first, StringComparison.OrdinalIgnoreCase)
+                    && IsSafeFirstNameMerge(candidate.DisplayName, name))
+                {
+                    return candidate.Key;
+                }
+            }
+        }
+
+        return canonical;
+    }
+
+    private static bool IsSafeFirstNameMerge(string current, string next)
+    {
+        var currentWords = WordsWithoutTitles(current);
+        var nextWords = WordsWithoutTitles(next);
+        return currentWords.Count == 1 || nextWords.Count == 1 || currentWords.SequenceEqual(nextWords, StringComparer.OrdinalIgnoreCase);
+    }
+
+    private static string CanonicalCharacterKey(string name)
+        => Normalise(string.Join(' ', WordsWithoutTitles(CleanCharacterName(name))));
+
+    private static string? FirstNameWithoutTitle(string? name)
+        => WordsWithoutTitles(name).FirstOrDefault();
+
+    private static List<string> WordsWithoutTitles(string? name)
+    {
+        var words = CleanCharacterName(name)
+            .Split(' ', StringSplitOptions.RemoveEmptyEntries)
+            .Select(word => word.Trim('.', ',', ';', ':', '!', '?', '"', '\''))
+            .Where(word => !string.IsNullOrWhiteSpace(word))
+            .ToList();
+
+        if (words.Count > 1 && IsMergeableTitle(words[0]))
+        {
+            words.RemoveAt(0);
+        }
+
+        return words;
+    }
+
+    private static bool IsMergeableTitle(string value)
+        => value.Equals("instructor", StringComparison.OrdinalIgnoreCase)
+            || value.Equals("detective", StringComparison.OrdinalIgnoreCase)
+            || value.Equals("sergeant", StringComparison.OrdinalIgnoreCase)
+            || value.Equals("ds", StringComparison.OrdinalIgnoreCase);
+
+    private static string CleanCharacterName(string? value)
+    {
+        var clean = Clean(value)
+            .Trim(' ', '.', ',', ';', ':', '!', '?', '"', '\'');
+
+        if (clean.EndsWith("'s", StringComparison.OrdinalIgnoreCase))
+        {
+            clean = clean[..^2];
+        }
+        else if (clean.EndsWith("’s", StringComparison.OrdinalIgnoreCase))
+        {
+            clean = clean[..^2];
+        }
+
+        return Clean(clean.Replace("Mrs.", "Mrs", StringComparison.OrdinalIgnoreCase)
+            .Replace("Mr.", "Mr", StringComparison.OrdinalIgnoreCase)
+            .Replace("Ms.", "Ms", StringComparison.OrdinalIgnoreCase)
+            .Replace("Dr.", "Dr", StringComparison.OrdinalIgnoreCase));
+    }
+
     private static string Normalise(string? value)
         => Clean(value).ToLowerInvariant();