diff --git a/PlotLine.Tests/Program.cs b/PlotLine.Tests/Program.cs index 1042f6a..2d1714c 100644 --- a/PlotLine.Tests/Program.cs +++ b/PlotLine.Tests/Program.cs @@ -66,6 +66,7 @@ var tests = new (string Name, Action Test)[] ("Story Intelligence visual polish keeps labels fixed and semantics specific", StoryIntelligenceVisualPolishKeepsLabelsFixedAndSemanticsSpecific), ("Character sex hints are conservative and reviewable", CharacterSexHintsAreConservativeAndReviewable), ("Character discovery does not truncate scan review candidates", CharacterDiscoveryDoesNotTruncateScanReviewCandidates), + ("Character discovery rejects sentence-start words", CharacterDiscoveryRejectsSentenceStartWords), ("Scan review aliases feed Story Intelligence known character context", ScanReviewAliasesFeedStoryIntelligenceKnownCharacterContext), ("Scan review post supports full-book form submissions", ScanReviewPostSupportsFullBookFormSubmissions), ("Story Intelligence experience boot does not serialise live model", StoryIntelligenceExperienceBootDoesNotSerialiseLiveModel), @@ -1276,7 +1277,7 @@ static void CharacterDiscoveryDoesNotTruncateScanReviewCandidates() var builder = new StringBuilder(); for (var index = 0; index < 60; index++) { - var name = $"Person{(char)('a' + index / 26)}{(char)('a' + index % 26)}"; + var name = $"Mr Person{(char)('a' + index / 26)}{(char)('a' + index % 26)}"; builder.Append(name).Append(" said hello. ").Append(name).Append(" smiled. "); } @@ -1284,6 +1285,25 @@ static void CharacterDiscoveryDoesNotTruncateScanReviewCandidates() Assert(candidates.Count >= 60, $"Character discovery should not truncate the scan review to 50 candidates; got {candidates.Count}."); } +static void CharacterDiscoveryRejectsSentenceStartWords() +{ + var method = typeof(WordCompanionService).GetMethod("DiscoverCharacterCandidates", System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Static); + Assert(method is not null, "Character discovery method could not be found."); + + var text = """ + Silence filled the room. Silence made everything worse. + Morning came too quickly. Morning dragged at the windows. + Eyes looked toward the door. Eyes followed the sound. + Colin said nothing. Debbie smiled at Colin. Mr Wilkinson opened the file. + """; + + var candidates = (IReadOnlyList)method!.Invoke(null, [text, false])!; + Assert(candidates.Any(candidate => candidate.Text == "Colin"), "Real common character names should still be detected."); + Assert(candidates.Any(candidate => candidate.Text == "Debbie"), "Real common character names should still be detected."); + Assert(candidates.Any(candidate => candidate.Text == "Mr Wilkinson"), "Titled character names should still be detected."); + Assert(!candidates.Any(candidate => candidate.Text is "Silence" or "Morning" or "Eyes"), "Repeated sentence-start words must not become character candidates."); +} + static void StoryIntelligenceExperienceBootDoesNotSerialiseLiveModel() { var view = File.ReadAllText(Path.Combine(AppContext.BaseDirectory, "../../../../PlotLine/Views/Development/StoryIntelligenceExperience.cshtml")); diff --git a/PlotLine/Hubs/WordCompanionFollowHub.cs b/PlotLine/Hubs/WordCompanionFollowHub.cs index cba30eb..5093655 100644 --- a/PlotLine/Hubs/WordCompanionFollowHub.cs +++ b/PlotLine/Hubs/WordCompanionFollowHub.cs @@ -260,7 +260,9 @@ public sealed class WordCompanionFollowHub( || state.UsesWordCompanion == true; private static ManuscriptScanPreview NormalisePreviewContext(ManuscriptScanPreview preview, int userId, UserOnboardingState state) - => new() + { + var characterCandidates = WordCompanionService.DiscoverManuscriptScanCharacterCandidates(preview); + return new() { PreviewID = preview.PreviewID, UserID = preview.UserID == 0 ? userId : preview.UserID, @@ -273,10 +275,11 @@ public sealed class WordCompanionFollowHub( TotalWordCount = preview.TotalWordCount, ChapterCount = preview.ChapterCount, SceneCount = preview.SceneCount, - CharacterCandidateCount = preview.CharacterCandidateCount, + CharacterCandidateCount = characterCandidates.Count, CreatedUtc = preview.CreatedUtc, Chapters = preview.Chapters, Scenes = preview.Scenes, - CharacterCandidates = preview.CharacterCandidates + CharacterCandidates = characterCandidates }; + } } diff --git a/PlotLine/Services/WordCompanionService.cs b/PlotLine/Services/WordCompanionService.cs index 527f545..d9ab1d4 100644 --- a/PlotLine/Services/WordCompanionService.cs +++ b/PlotLine/Services/WordCompanionService.cs @@ -646,6 +646,32 @@ public sealed class WordCompanionService( return visible; } + public static IReadOnlyList DiscoverManuscriptScanCharacterCandidates(ManuscriptScanPreview preview) + { + var manuscriptText = string.Join("\n\n", preview.Chapters + .OrderBy(chapter => chapter.ChapterNumber) + .Select(chapter => chapter.ChapterText) + .Where(text => !string.IsNullOrWhiteSpace(text))); + if (string.IsNullOrWhiteSpace(manuscriptText)) + { + return FilterIncomingScanCharacterCandidates(preview.CharacterCandidates); + } + + return DiscoverCharacterCandidates(manuscriptText) + .Select((candidate, index) => new ManuscriptScanCharacterCandidatePreview + { + TemporaryCharacterKey = $"c-{StableCandidateKey(candidate.Text, index)}", + Name = candidate.Text, + MentionCount = candidate.MentionCount, + QualityScore = candidate.QualityScore, + Category = candidate.Category, + Reason = candidate.Reason, + IsExistingCharacterMatch = candidate.IsExistingCharacterMatch, + SuggestedImportance = candidate.Confidence + }) + .ToList(); + } + private static WordCompanionCharacterCandidateDto? BuildCandidateDto(CharacterCandidateEvidence evidence) { if (string.IsNullOrWhiteSpace(evidence.Text) || evidence.Text.Length > 200) @@ -667,6 +693,8 @@ public sealed class WordCompanionService( var hasLocationIndicator = words.Any(word => CharacterDiscoveryLocationIndicators.Contains(TrimHonorificPunctuation(word))); var hasStopWord = words.Any(word => CharacterDiscoveryStopWords.Contains(TrimHonorificPunctuation(word))); var isRelationshipTitle = words.Length == 1 && CharacterDiscoveryRelationshipTitles.Contains(first); + var isSingleWord = words.Length == 1; + var hasKnownGivenName = CharacterSexHintService.Suggest(first) != CharacterSexHintService.Unknown; var nonSentenceStartMentions = evidence.MentionCount - evidence.SentenceStartMentions; var mostlySentenceStart = evidence.MentionCount > 0 && evidence.SentenceStartMentions >= Math.Ceiling(evidence.MentionCount * 0.8m); @@ -707,10 +735,19 @@ public sealed class WordCompanionService( { score += 2; } - if (hasNameLikeWord) + if (hasNameLikeWord || hasKnownGivenName) { score += 3; } + if (isSingleWord + && mostlySentenceStart + && !hasKnownGivenName + && !hasNameLikeWord + && evidence.PossessiveEvidenceCount == 0 + && nonSentenceStartMentions == 0) + { + score -= 5; + } if (hasTemporalWord) { score -= 1; @@ -740,7 +777,7 @@ public sealed class WordCompanionService( score -= 3; } - var category = CandidateCategory(score, hasStopWord, isRelationshipTitle, hasTemporalWord, hasLocationIndicator, evidence); + var category = CandidateCategory(score, hasStopWord, isRelationshipTitle, hasTemporalWord, hasLocationIndicator, evidence, isSingleWord, mostlySentenceStart, hasKnownGivenName || hasNameLikeWord, hasHonorific); var confidence = score >= 4 ? "High" : score >= 2 ? "Medium" : "Low"; if (hasLocationIndicator && evidence.DialogueEvidenceCount == 0 && !hasHonorific) { @@ -768,12 +805,24 @@ public sealed class WordCompanionService( bool isRelationshipTitle, bool hasTemporalWord, bool hasLocationIndicator, - CharacterCandidateEvidence evidence) + CharacterCandidateEvidence evidence, + bool isSingleWord, + bool mostlySentenceStart, + bool hasNameEvidence, + bool hasHonorific) { if (hasStopWord) { return "Excluded"; } + if (isSingleWord + && mostlySentenceStart + && !hasNameEvidence + && evidence.PossessiveEvidenceCount == 0 + && evidence.MentionCount - evidence.SentenceStartMentions == 0) + { + return "Excluded"; + } if (isRelationshipTitle) { return "RelationshipTitle"; @@ -786,7 +835,10 @@ public sealed class WordCompanionService( { return "Excluded"; } - if (evidence.MentionCount < CharacterDiscoveryMinimumMentions && evidence.DialogueEvidenceCount == 0) + if (evidence.MentionCount < CharacterDiscoveryMinimumMentions + && evidence.DialogueEvidenceCount == 0 + && !hasHonorific + && !(hasNameEvidence && (evidence.HumanActionEvidenceCount > 0 || evidence.PossessiveEvidenceCount > 0))) { return "Excluded"; } @@ -929,6 +981,29 @@ public sealed class WordCompanionService( return Regex.Replace(NormalizeApostrophes(candidate), @"\s+", " ").Trim().Trim(',', '.', ';', ':', '!', '?', '"', '\''); } + private static IReadOnlyList FilterIncomingScanCharacterCandidates( + IReadOnlyList candidates) + => candidates + .Where(candidate => !string.IsNullOrWhiteSpace(candidate.Name)) + .Where(candidate => !string.Equals(candidate.Category, "Excluded", StringComparison.OrdinalIgnoreCase)) + .Where(candidate => + candidate.Name.Split(' ', StringSplitOptions.RemoveEmptyEntries).Length > 1 + || candidate.QualityScore >= 4 + || CharacterSexHintService.Suggest(candidate.Name) != CharacterSexHintService.Unknown) + .GroupBy(candidate => NormalizeCandidateName(candidate.Name), StringComparer.OrdinalIgnoreCase) + .Select(group => group.OrderByDescending(candidate => candidate.QualityScore).ThenByDescending(candidate => candidate.MentionCount).First()) + .OrderByDescending(candidate => CategorySortValue(candidate.Category)) + .ThenByDescending(candidate => candidate.QualityScore) + .ThenByDescending(candidate => candidate.MentionCount) + .ThenBy(candidate => candidate.Name, StringComparer.OrdinalIgnoreCase) + .ToList(); + + private static string StableCandidateKey(string name, int index) + { + var slug = Regex.Replace(name.ToLowerInvariant(), @"[^a-z0-9]+", "-").Trim('-'); + return string.IsNullOrWhiteSpace(slug) ? $"candidate-{index + 1}" : slug; + } + private static string NormalizeApostrophes(string? value) { return (value ?? string.Empty).Replace('’', '\'').Replace('‘', '\'');