Tighten manuscript character detection

PlotLine.Tests/Program.cs

@@ -66,6 +66,7 @@ var tests = new (string Name, Action Test)[]
     ("Story Intelligence visual polish keeps labels fixed and semantics specific", StoryIntelligenceVisualPolishKeepsLabelsFixedAndSemanticsSpecific),
     ("Character sex hints are conservative and reviewable", CharacterSexHintsAreConservativeAndReviewable),
     ("Character discovery does not truncate scan review candidates", CharacterDiscoveryDoesNotTruncateScanReviewCandidates),
+    ("Character discovery rejects sentence-start words", CharacterDiscoveryRejectsSentenceStartWords),
     ("Scan review aliases feed Story Intelligence known character context", ScanReviewAliasesFeedStoryIntelligenceKnownCharacterContext),
     ("Scan review post supports full-book form submissions", ScanReviewPostSupportsFullBookFormSubmissions),
     ("Story Intelligence experience boot does not serialise live model", StoryIntelligenceExperienceBootDoesNotSerialiseLiveModel),
@@ -1276,7 +1277,7 @@ static void CharacterDiscoveryDoesNotTruncateScanReviewCandidates()
     var builder = new StringBuilder();
     for (var index = 0; index < 60; index++)
     {
-        var name = $"Person{(char)('a' + index / 26)}{(char)('a' + index % 26)}";
+        var name = $"Mr Person{(char)('a' + index / 26)}{(char)('a' + index % 26)}";
         builder.Append(name).Append(" said hello. ").Append(name).Append(" smiled. ");
     }
 
@@ -1284,6 +1285,25 @@ static void CharacterDiscoveryDoesNotTruncateScanReviewCandidates()
     Assert(candidates.Count >= 60, $"Character discovery should not truncate the scan review to 50 candidates; got {candidates.Count}.");
 }
 
+static void CharacterDiscoveryRejectsSentenceStartWords()
+{
+    var method = typeof(WordCompanionService).GetMethod("DiscoverCharacterCandidates", System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Static);
+    Assert(method is not null, "Character discovery method could not be found.");
+
+    var text = """
+        Silence filled the room. Silence made everything worse.
+        Morning came too quickly. Morning dragged at the windows.
+        Eyes looked toward the door. Eyes followed the sound.
+        Colin said nothing. Debbie smiled at Colin. Mr Wilkinson opened the file.
+        """;
+
+    var candidates = (IReadOnlyList<WordCompanionCharacterCandidateDto>)method!.Invoke(null, [text, false])!;
+    Assert(candidates.Any(candidate => candidate.Text == "Colin"), "Real common character names should still be detected.");
+    Assert(candidates.Any(candidate => candidate.Text == "Debbie"), "Real common character names should still be detected.");
+    Assert(candidates.Any(candidate => candidate.Text == "Mr Wilkinson"), "Titled character names should still be detected.");
+    Assert(!candidates.Any(candidate => candidate.Text is "Silence" or "Morning" or "Eyes"), "Repeated sentence-start words must not become character candidates.");
+}
+
 static void StoryIntelligenceExperienceBootDoesNotSerialiseLiveModel()
 {
     var view = File.ReadAllText(Path.Combine(AppContext.BaseDirectory, "../../../../PlotLine/Views/Development/StoryIntelligenceExperience.cshtml"));

PlotLine/Hubs/WordCompanionFollowHub.cs

@@ -260,7 +260,9 @@ public sealed class WordCompanionFollowHub(
             || state.UsesWordCompanion == true;
 
     private static ManuscriptScanPreview NormalisePreviewContext(ManuscriptScanPreview preview, int userId, UserOnboardingState state)
-        => new()
+    {
+        var characterCandidates = WordCompanionService.DiscoverManuscriptScanCharacterCandidates(preview);
+        return new()
         {
             PreviewID = preview.PreviewID,
             UserID = preview.UserID == 0 ? userId : preview.UserID,
@@ -273,10 +275,11 @@ public sealed class WordCompanionFollowHub(
             TotalWordCount = preview.TotalWordCount,
             ChapterCount = preview.ChapterCount,
             SceneCount = preview.SceneCount,
-            CharacterCandidateCount = preview.CharacterCandidateCount,
+            CharacterCandidateCount = characterCandidates.Count,
             CreatedUtc = preview.CreatedUtc,
             Chapters = preview.Chapters,
             Scenes = preview.Scenes,
-            CharacterCandidates = preview.CharacterCandidates
+            CharacterCandidates = characterCandidates
         };
     }
+}

PlotLine/Services/WordCompanionService.cs

@@ -646,6 +646,32 @@ public sealed class WordCompanionService(
         return visible;
     }
 
+    public static IReadOnlyList<ManuscriptScanCharacterCandidatePreview> DiscoverManuscriptScanCharacterCandidates(ManuscriptScanPreview preview)
+    {
+        var manuscriptText = string.Join("\n\n", preview.Chapters
+            .OrderBy(chapter => chapter.ChapterNumber)
+            .Select(chapter => chapter.ChapterText)
+            .Where(text => !string.IsNullOrWhiteSpace(text)));
+        if (string.IsNullOrWhiteSpace(manuscriptText))
+        {
+            return FilterIncomingScanCharacterCandidates(preview.CharacterCandidates);
+        }
+
+        return DiscoverCharacterCandidates(manuscriptText)
+            .Select((candidate, index) => new ManuscriptScanCharacterCandidatePreview
+            {
+                TemporaryCharacterKey = $"c-{StableCandidateKey(candidate.Text, index)}",
+                Name = candidate.Text,
+                MentionCount = candidate.MentionCount,
+                QualityScore = candidate.QualityScore,
+                Category = candidate.Category,
+                Reason = candidate.Reason,
+                IsExistingCharacterMatch = candidate.IsExistingCharacterMatch,
+                SuggestedImportance = candidate.Confidence
+            })
+            .ToList();
+    }
+
     private static WordCompanionCharacterCandidateDto? BuildCandidateDto(CharacterCandidateEvidence evidence)
     {
         if (string.IsNullOrWhiteSpace(evidence.Text) || evidence.Text.Length > 200)
@@ -667,6 +693,8 @@ public sealed class WordCompanionService(
         var hasLocationIndicator = words.Any(word => CharacterDiscoveryLocationIndicators.Contains(TrimHonorificPunctuation(word)));
         var hasStopWord = words.Any(word => CharacterDiscoveryStopWords.Contains(TrimHonorificPunctuation(word)));
         var isRelationshipTitle = words.Length == 1 && CharacterDiscoveryRelationshipTitles.Contains(first);
+        var isSingleWord = words.Length == 1;
+        var hasKnownGivenName = CharacterSexHintService.Suggest(first) != CharacterSexHintService.Unknown;
         var nonSentenceStartMentions = evidence.MentionCount - evidence.SentenceStartMentions;
         var mostlySentenceStart = evidence.MentionCount > 0 && evidence.SentenceStartMentions >= Math.Ceiling(evidence.MentionCount * 0.8m);
 
@@ -707,10 +735,19 @@ public sealed class WordCompanionService(
         {
             score += 2;
         }
-        if (hasNameLikeWord)
+        if (hasNameLikeWord || hasKnownGivenName)
         {
             score += 3;
         }
+        if (isSingleWord
+            && mostlySentenceStart
+            && !hasKnownGivenName
+            && !hasNameLikeWord
+            && evidence.PossessiveEvidenceCount == 0
+            && nonSentenceStartMentions == 0)
+        {
+            score -= 5;
+        }
         if (hasTemporalWord)
         {
             score -= 1;
@@ -740,7 +777,7 @@ public sealed class WordCompanionService(
             score -= 3;
         }
 
-        var category = CandidateCategory(score, hasStopWord, isRelationshipTitle, hasTemporalWord, hasLocationIndicator, evidence);
+        var category = CandidateCategory(score, hasStopWord, isRelationshipTitle, hasTemporalWord, hasLocationIndicator, evidence, isSingleWord, mostlySentenceStart, hasKnownGivenName || hasNameLikeWord, hasHonorific);
         var confidence = score >= 4 ? "High" : score >= 2 ? "Medium" : "Low";
         if (hasLocationIndicator && evidence.DialogueEvidenceCount == 0 && !hasHonorific)
         {
@@ -768,12 +805,24 @@ public sealed class WordCompanionService(
         bool isRelationshipTitle,
         bool hasTemporalWord,
         bool hasLocationIndicator,
-        CharacterCandidateEvidence evidence)
+        CharacterCandidateEvidence evidence,
+        bool isSingleWord,
+        bool mostlySentenceStart,
+        bool hasNameEvidence,
+        bool hasHonorific)
     {
         if (hasStopWord)
         {
             return "Excluded";
         }
+        if (isSingleWord
+            && mostlySentenceStart
+            && !hasNameEvidence
+            && evidence.PossessiveEvidenceCount == 0
+            && evidence.MentionCount - evidence.SentenceStartMentions == 0)
+        {
+            return "Excluded";
+        }
         if (isRelationshipTitle)
         {
             return "RelationshipTitle";
@@ -786,7 +835,10 @@ public sealed class WordCompanionService(
         {
             return "Excluded";
         }
-        if (evidence.MentionCount < CharacterDiscoveryMinimumMentions && evidence.DialogueEvidenceCount == 0)
+        if (evidence.MentionCount < CharacterDiscoveryMinimumMentions
+            && evidence.DialogueEvidenceCount == 0
+            && !hasHonorific
+            && !(hasNameEvidence && (evidence.HumanActionEvidenceCount > 0 || evidence.PossessiveEvidenceCount > 0)))
         {
             return "Excluded";
         }
@@ -929,6 +981,29 @@ public sealed class WordCompanionService(
         return Regex.Replace(NormalizeApostrophes(candidate), @"\s+", " ").Trim().Trim(',', '.', ';', ':', '!', '?', '"', '\'');
     }
 
+    private static IReadOnlyList<ManuscriptScanCharacterCandidatePreview> FilterIncomingScanCharacterCandidates(
+        IReadOnlyList<ManuscriptScanCharacterCandidatePreview> candidates)
+        => candidates
+            .Where(candidate => !string.IsNullOrWhiteSpace(candidate.Name))
+            .Where(candidate => !string.Equals(candidate.Category, "Excluded", StringComparison.OrdinalIgnoreCase))
+            .Where(candidate =>
+                candidate.Name.Split(' ', StringSplitOptions.RemoveEmptyEntries).Length > 1
+                || candidate.QualityScore >= 4
+                || CharacterSexHintService.Suggest(candidate.Name) != CharacterSexHintService.Unknown)
+            .GroupBy(candidate => NormalizeCandidateName(candidate.Name), StringComparer.OrdinalIgnoreCase)
+            .Select(group => group.OrderByDescending(candidate => candidate.QualityScore).ThenByDescending(candidate => candidate.MentionCount).First())
+            .OrderByDescending(candidate => CategorySortValue(candidate.Category))
+            .ThenByDescending(candidate => candidate.QualityScore)
+            .ThenByDescending(candidate => candidate.MentionCount)
+            .ThenBy(candidate => candidate.Name, StringComparer.OrdinalIgnoreCase)
+            .ToList();
+
+    private static string StableCandidateKey(string name, int index)
+    {
+        var slug = Regex.Replace(name.ToLowerInvariant(), @"[^a-z0-9]+", "-").Trim('-');
+        return string.IsNullOrWhiteSpace(slug) ? $"candidate-{index + 1}" : slug;
+    }
+
     private static string NormalizeApostrophes(string? value)
     {
         return (value ?? string.Empty).Replace('’', '\'').Replace('‘', '\'');