Phase 16D.1 - Refine Major Character Discovery

PlotLine/Models/WordCompanionApiModels.cs

@@ -338,6 +338,8 @@ public sealed class WordCompanionCharacterCandidateDto
 {
     public string Text { get; init; } = string.Empty;
     public int MentionCount { get; init; }
+    public string Confidence { get; init; } = "Medium";
+    public string Reason { get; init; } = string.Empty;
 }
 
 public sealed class WordCompanionDiscoverCharactersRequest

PlotLine/Services/WordCompanionService.cs

@@ -35,19 +35,42 @@ public sealed class WordCompanionService(
     private const int CharacterDiscoveryMinimumMentions = 5;
     private const int CharacterDiscoverySuggestionLimit = 50;
     private static readonly Regex CharacterCandidateRegex = new(
-        @"\b(?:(?:Mr|Mrs|Ms|Miss|Dr|Rev|Fr|Sir|Lady|Lord|DS|DI|DC|PC)\.?\s+)?[A-Z][a-z]+(?:[-'][A-Z]?[a-z]+)?(?:\s+[A-Z][a-z]+(?:[-'][A-Z]?[a-z]+)?){0,2}\b",
+        @"(?<!['’\p{L}])(?:(?:Mr|Mrs|Ms|Miss|Dr|Rev|Fr|Sir|Lady|Lord|DS|DI|DC|PC|Professor|Aunt|Uncle)\.?\s+)?[A-Z][a-z]+(?:[-'][A-Z]?[a-z]+)?(?:\s+[A-Z][a-z]+(?:[-'][A-Z]?[a-z]+)?){0,2}(?!['’\p{L}])",
         RegexOptions.Compiled);
     private static readonly HashSet<string> CharacterDiscoveryStopWords = new(StringComparer.OrdinalIgnoreCase)
     {
         "A", "An", "And", "As", "At", "But", "By", "For", "From", "He", "Her", "His", "I", "If", "In", "Into",
         "It", "Its", "Of", "On", "Or", "Our", "She", "So", "That", "The", "Their", "Then", "There", "They",
         "This", "To", "We", "When", "Where", "Who", "With", "You", "Your",
-        "Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday", "Sunday",
-        "January", "February", "March", "April", "May", "June", "July", "August", "September", "October", "November", "December",
         "Christmas", "Easter", "Morning", "Afternoon", "Evening", "Night", "Today", "Tomorrow", "Yesterday",
         "Chapter", "Scene", "Part", "Prologue", "Epilogue", "Book", "Volume", "Act",
         "Mum", "Dad", "Mother", "Father", "Grandma", "Grandad", "Nan", "Nana", "Granny",
-        "Yes", "No", "Okay", "Ok", "Hello", "Goodbye"
+        "Yes", "No", "Okay", "Ok", "Hello", "Goodbye",
+        "What", "Not", "Just", "Yeah", "How", "Well", "Are", "Like", "One", "Can", "Thank", "Why", "Come",
+        "Let", "Maybe", "Please", "God", "Something", "After", "Did", "Even", "All", "Nothing", "Have",
+        "Don", "Won", "Could", "Would", "Should", "Do", "Does", "Had", "Has", "Was", "Were", "Is", "Am",
+        "Right", "Really", "Now", "Look", "Listen", "Sorry", "Thanks", "Fine", "Great", "Sure", "Course",
+        "Thing", "Things", "Everyone", "Everything", "Anything", "Someone", "Nobody", "People"
+    };
+    private static readonly HashSet<string> CharacterDiscoveryTemporalWords = new(StringComparer.OrdinalIgnoreCase)
+    {
+        "Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday", "Sunday",
+        "January", "February", "March", "April", "May", "June", "July", "August", "September", "October", "November", "December",
+        "Spring", "Summer", "Autumn", "Fall", "Winter"
+    };
+    private static readonly HashSet<string> CharacterDiscoveryNameLikeWords = new(StringComparer.OrdinalIgnoreCase)
+    {
+        "Grace", "Jack", "Rose", "May", "Will", "Mark", "Hope", "Faith", "Pat", "Rob", "Bill", "Harry",
+        "November", "Autumn", "Rain"
+    };
+    private static readonly HashSet<string> CharacterDiscoveryHonorifics = new(StringComparer.OrdinalIgnoreCase)
+    {
+        "Mr", "Mrs", "Ms", "Miss", "Dr", "Rev", "Fr", "Sir", "Lady", "Lord", "DS", "DI", "DC", "PC", "Professor", "Aunt", "Uncle"
+    };
+    private static readonly HashSet<string> CharacterDiscoveryDialogueTags = new(StringComparer.OrdinalIgnoreCase)
+    {
+        "said", "asked", "replied", "answered", "whispered", "shouted", "called", "muttered", "cried", "snapped",
+        "sighed", "laughed", "continued", "began", "told", "yelled"
     };
     public async Task<WordCompanionProjectsResponse> ListProjectsAsync()
     {
@@ -343,7 +366,7 @@ public sealed class WordCompanionService(
             return [];
         }
 
-        var mentions = new Dictionary<string, int>(StringComparer.Ordinal);
+        var candidates = new Dictionary<string, CharacterCandidateEvidence>(StringComparer.Ordinal);
         foreach (Match match in CharacterCandidateRegex.Matches(manuscriptText))
         {
             var candidate = NormalizeCandidateName(match.Value);
@@ -352,15 +375,33 @@ public sealed class WordCompanionService(
                 continue;
             }
 
-            mentions[candidate] = mentions.TryGetValue(candidate, out var count) ? count + 1 : 1;
+            if (!candidates.TryGetValue(candidate, out var evidence))
+            {
+                evidence = new CharacterCandidateEvidence(candidate);
+                candidates[candidate] = evidence;
             }
 
-        return mentions
-            .Where(item => item.Value >= CharacterDiscoveryMinimumMentions)
-            .OrderByDescending(item => item.Value)
-            .ThenBy(item => item.Key, StringComparer.OrdinalIgnoreCase)
+            evidence.MentionCount += 1;
+            if (AppearsAtSentenceStart(manuscriptText, match.Index))
+            {
+                evidence.SentenceStartMentions += 1;
+            }
+
+            if (HasDialogueTagEvidence(manuscriptText, match.Index, match.Length))
+            {
+                evidence.DialogueEvidenceCount += 1;
+            }
+        }
+
+        return candidates.Values
+            .Where(item => item.MentionCount >= CharacterDiscoveryMinimumMentions)
+            .Select(BuildCandidateDto)
+            .Where(item => item is not null)
+            .Select(item => item!)
+            .OrderByDescending(item => ConfidenceSortValue(item.Confidence))
+            .ThenByDescending(item => item.MentionCount)
+            .ThenBy(item => item.Text, StringComparer.OrdinalIgnoreCase)
             .Take(CharacterDiscoverySuggestionLimit)
-            .Select(item => new WordCompanionCharacterCandidateDto { Text = item.Key, MentionCount = item.Value })
             .ToList();
     }
 
@@ -381,9 +422,159 @@ public sealed class WordCompanionService(
         return words.Length > 1 || first.Length > 2;
     }
 
+    private static WordCompanionCharacterCandidateDto? BuildCandidateDto(CharacterCandidateEvidence evidence)
+    {
+        var score = 0;
+        var words = evidence.Text.Split(' ', StringSplitOptions.RemoveEmptyEntries);
+        var first = words.Length > 0 ? TrimHonorificPunctuation(words[0]) : string.Empty;
+        var hasHonorific = words.Length > 1 && CharacterDiscoveryHonorifics.Contains(first);
+        var hasMultiWordName = words.Length > 1;
+        var hasNameLikeWord = words.Any(word => CharacterDiscoveryNameLikeWords.Contains(TrimHonorificPunctuation(word)));
+        var hasTemporalWord = words.Any(word => CharacterDiscoveryTemporalWords.Contains(TrimHonorificPunctuation(word)));
+        var nonSentenceStartMentions = evidence.MentionCount - evidence.SentenceStartMentions;
+        var mostlySentenceStart = evidence.MentionCount > 0 && evidence.SentenceStartMentions >= Math.Ceiling(evidence.MentionCount * 0.8m);
+
+        if (evidence.MentionCount >= CharacterDiscoveryMinimumMentions)
+        {
+            score += 1;
+        }
+        if (evidence.MentionCount >= 10)
+        {
+            score += 1;
+        }
+        if (nonSentenceStartMentions > 0)
+        {
+            score += 2;
+        }
+        if (evidence.DialogueEvidenceCount > 0)
+        {
+            score += 3;
+        }
+        if (hasMultiWordName)
+        {
+            score += 2;
+        }
+        if (hasHonorific)
+        {
+            score += 2;
+        }
+        if (hasNameLikeWord)
+        {
+            score += 3;
+        }
+        if (hasTemporalWord)
+        {
+            score -= 1;
+        }
+        if (mostlySentenceStart)
+        {
+            score -= 2;
+        }
+        if (!hasMultiWordName && evidence.DialogueEvidenceCount == 0 && nonSentenceStartMentions == 0)
+        {
+            score -= 1;
+        }
+
+        if (score < 0)
+        {
+            return null;
+        }
+
+        var confidence = score >= 5 ? "High" : score >= 2 ? "Medium" : "Low";
+        return new WordCompanionCharacterCandidateDto
+        {
+            Text = evidence.Text,
+            MentionCount = evidence.MentionCount,
+            Confidence = confidence,
+            Reason = BuildCandidateReason(evidence, hasHonorific, hasMultiWordName, hasTemporalWord, mostlySentenceStart)
+        };
+    }
+
+    private static string BuildCandidateReason(
+        CharacterCandidateEvidence evidence,
+        bool hasHonorific,
+        bool hasMultiWordName,
+        bool hasTemporalWord,
+        bool mostlySentenceStart)
+    {
+        if (evidence.DialogueEvidenceCount > 0)
+        {
+            return "Dialogue evidence";
+        }
+        if (hasHonorific)
+        {
+            return "Title or honorific";
+        }
+        if (hasMultiWordName)
+        {
+            return "Proper-name phrase";
+        }
+        if (hasTemporalWord)
+        {
+            return "Possible name";
+        }
+        if (mostlySentenceStart)
+        {
+            return "Mostly sentence starts";
+        }
+        return evidence.MentionCount >= 10 ? "Frequent possible name" : "Possible name";
+    }
+
+    private static int ConfidenceSortValue(string confidence)
+    {
+        return string.Equals(confidence, "High", StringComparison.OrdinalIgnoreCase) ? 3
+            : string.Equals(confidence, "Medium", StringComparison.OrdinalIgnoreCase) ? 2
+            : 1;
+    }
+
+    private static bool AppearsAtSentenceStart(string manuscriptText, int matchIndex)
+    {
+        for (var i = matchIndex - 1; i >= 0; i--)
+        {
+            var character = manuscriptText[i];
+            if (char.IsWhiteSpace(character))
+            {
+                if (character is '\n' or '\r')
+                {
+                    return true;
+                }
+                continue;
+            }
+
+            if (character is '"' or '\'' or '’' or '“' or '”' or '(' or '[')
+            {
+                continue;
+            }
+
+            return character is '.' or '!' or '?' or ':' or ';';
+        }
+
+        return true;
+    }
+
+    private static bool HasDialogueTagEvidence(string manuscriptText, int matchIndex, int matchLength)
+    {
+        var before = manuscriptText[Math.Max(0, matchIndex - 48)..matchIndex];
+        var afterStart = matchIndex + matchLength;
+        var after = manuscriptText[afterStart..Math.Min(manuscriptText.Length, afterStart + 48)];
+        return HasTagAfterCandidate(after) || HasTagBeforeCandidate(before);
+    }
+
+    private static bool HasTagAfterCandidate(string textAfterCandidate)
+    {
+        var match = Regex.Match(textAfterCandidate, @"^\s*,?\s*(?<tag>[a-z]+)\b", RegexOptions.IgnoreCase);
+        return match.Success && CharacterDiscoveryDialogueTags.Contains(match.Groups["tag"].Value);
+    }
+
+    private static bool HasTagBeforeCandidate(string textBeforeCandidate)
+    {
+        var match = Regex.Match(textBeforeCandidate, @"\b(?<tag>[a-z]+)\s*,?\s*[""'\u2019\u201d]?\s*$", RegexOptions.IgnoreCase);
+        return match.Success && CharacterDiscoveryDialogueTags.Contains(match.Groups["tag"].Value);
+    }
+
     private static string NormalizeCandidateName(string candidate)
     {
-        return Regex.Replace(candidate ?? string.Empty, @"\s+", " ").Trim().Trim(',', '.', ';', ':', '!', '?', '"', '\'');
+        return Regex.Replace(candidate ?? string.Empty, @"\s+", " ").Trim().Trim(',', '.', ';', ':', '!', '?', '"', '\'', '’');
     }
 
     private static string TrimHonorificPunctuation(string word)
@@ -391,6 +582,14 @@ public sealed class WordCompanionService(
         return (word ?? string.Empty).Trim().TrimEnd('.');
     }
 
+    private sealed class CharacterCandidateEvidence(string text)
+    {
+        public string Text { get; } = text;
+        public int MentionCount { get; set; }
+        public int SentenceStartMentions { get; set; }
+        public int DialogueEvidenceCount { get; set; }
+    }
+
     private static WordCompanionManuscriptDocumentDto ToDto(ManuscriptDocumentModel document) => new()
     {
         ManuscriptDocumentId = document.ManuscriptDocumentID,

PlotLine/wwwroot/js/word-companion-host.js

@@ -1102,14 +1102,17 @@
             const checkbox = document.createElement("input");
             checkbox.type = "checkbox";
             checkbox.value = candidate.text || "";
-            checkbox.checked = true;
+            checkbox.checked = String(candidate.confidence || "").toLowerCase() !== "very low";
             checkbox.addEventListener("change", updateFirstRunButtons);
 
             const name = document.createElement("strong");
             name.textContent = candidate.text || "";
 
             const mentions = document.createElement("em");
-            mentions.textContent = `${Number(candidate.mentionCount || 0).toLocaleString()} mentions`;
+            const reason = String(candidate.reason || "").trim();
+            mentions.textContent = reason
+                ? `${Number(candidate.mentionCount || 0).toLocaleString()} mentions, ${reason.toLowerCase()}`
+                : `${Number(candidate.mentionCount || 0).toLocaleString()} mentions`;
 
             label.append(checkbox, name, mentions);
             firstRunCharacterCandidates.append(label);