diff --git a/PlotLine/Models/StoryIntelligenceImportPreviewModels.cs b/PlotLine/Models/StoryIntelligenceImportPreviewModels.cs index 3773f7c..2def31d 100644 --- a/PlotLine/Models/StoryIntelligenceImportPreviewModels.cs +++ b/PlotLine/Models/StoryIntelligenceImportPreviewModels.cs @@ -6,6 +6,7 @@ public static class StoryImportResolutionStatuses public const string NewCharacter = "New Character"; public const string NewLocation = "New Location"; public const string NewAsset = "New Asset"; + public const string PossibleDuplicate = "Possible Duplicate"; public const string Ambiguous = "Ambiguous"; public const string UnresolvedPersonReference = "Unresolved Person Reference"; public const string GenericReference = "Generic Reference"; @@ -59,6 +60,7 @@ public sealed class StoryImportRelationshipPreview public string CharacterA { get; init; } = string.Empty; public string CharacterB { get; init; } = string.Empty; public string Signal { get; init; } = string.Empty; + public IReadOnlyList SupportingSceneNumbers { get; init; } = []; public decimal? Confidence { get; init; } public int? ConfidencePercent => Confidence.HasValue ? (int)Math.Round(Confidence.Value * 100m) : null; } @@ -76,7 +78,7 @@ public sealed class StoryImportKnowledgePreview public sealed class StoryImportTimelinePreview { - public string Action { get; init; } = "Would Create Scene Timeline Entry"; + public string Action { get; init; } = "Timeline Clue"; public int? SceneNumber { get; init; } public string SceneDate { get; init; } = string.Empty; public decimal? Confidence { get; init; } diff --git a/PlotLine/Services/SceneImportResolver.cs b/PlotLine/Services/SceneImportResolver.cs index 02573ac..027b8d3 100644 --- a/PlotLine/Services/SceneImportResolver.cs +++ b/PlotLine/Services/SceneImportResolver.cs @@ -23,16 +23,24 @@ public sealed class SceneImportResolver( private static readonly HashSet GenericPeople = CreateSet( "the receptionist", "receptionist", "old man", "young woman", "police officer", "doctor", "nurse", "driver", - "man", "woman", "boy", "girl", "child", "person", "guard", "waiter", "waitress", "clerk", "shopkeeper"); + "man", "woman", "women", "boy", "girl", "child", "children", "person", "people", "guard", "security guard", + "cashier", "elderly attendant", "attendant", "driver of car behind", "shop assistant", "crowd", "family", + "group of women", "small group", "women (small group)", "waiter", "waitress", "clerk", "shopkeeper"); private static readonly HashSet IgnoredPovNames = CreateSet("narrator", "unknown"); private static readonly HashSet GenericLocations = CreateSet( - "car park", "road", "house", "kitchen", "stairwell", "stair", "doorway", "corridor", "street", "room", - "bedroom", "bathroom", "office", "car", "church", "hall", "table", "chair"); + "car park", "house", "kitchen", "stairwell", "stair", "doorway", "corridor", "room", + "bedroom", "bathroom", "office", "reception", "sixth floor", "floor", "car", "hall", "table", "chair"); - private static readonly HashSet GenericAssets = CreateSet( - "the key", "key", "the letter", "letter", "the car", "car", "phone", "table", "chair", "door", "window"); + private static readonly HashSet EnvironmentalAssets = CreateSet( + "pink towel", "mug of tea", "magazine", "fire door", "old oak door", "iron handle", "reception desk", + "stained-glass windows", "stained glass windows", "chair", "desk", "window", "windows", "cup", "floor", + "wall", "door", "table", "towel", "mug", "handle"); + + private static readonly HashSet StoryAssetNames = CreateSet( + "car", "vehicle", "passport", "letter", "key", "photograph", "photo", "bank book", "bank account", + "necklace", "weapon", "folder", "document", "diary", "telephone number", "phone number", "scrap of paper", "wallet"); public async Task ResolveAsync(StoryIntelligenceSavedRun run, IReadOnlyList sceneResults) { @@ -59,7 +67,7 @@ public sealed class SceneImportResolver( var characterResolutions = ResolveCharacters(scenes, characterIndex).ToList(); var locationResolutions = ResolveLocations(scenes, locationIndex).ToList(); var assetResolutions = ResolveAssets(scenes, assetIndex).ToList(); - var relationshipPreview = ResolveRelationships(scenes, existingRelationships).ToList(); + var relationshipPreview = ResolveRelationships(scenes, existingRelationships, characterResolutions).ToList(); var knowledgePreview = ResolveKnowledge(scenes).ToList(); var timelinePreview = ResolveTimeline(scenes).ToList(); var warnings = BuildWarnings(characterResolutions, locationResolutions, assetResolutions).ToList(); @@ -99,7 +107,7 @@ public sealed class SceneImportResolver( continue; } - if (GenericPeople.Contains(name.ToLowerInvariant())) + if (IsGenericPerson(name)) { yield return LogResolution(new StoryImportEntityResolution { @@ -107,7 +115,7 @@ public sealed class SceneImportResolver( EntityType = "Character", Status = StoryImportResolutionStatuses.UnresolvedPersonReference, Confidence = item.Confidence, - Notes = "Generic person reference; would not create a Character record." + Notes = "Rejected: generic unnamed role; would not create a Character record." }); continue; } @@ -118,12 +126,12 @@ public sealed class SceneImportResolver( private IEnumerable ResolveLocations(IEnumerable scenes, EntityIndex index) { - foreach (var item in scenes + foreach (var group in scenes .SelectMany(scene => scene.Locations ?? []) .Where(location => !string.IsNullOrWhiteSpace(location.Name)) - .GroupBy(location => Clean(location.Name), StringComparer.OrdinalIgnoreCase) - .Select(group => group.First())) + .GroupBy(location => NormaliseLocationName(location.Name), StringComparer.OrdinalIgnoreCase)) { + var item = group.First(); var name = Clean(item.Name); if (IsGenericLocation(item)) { @@ -133,12 +141,22 @@ public sealed class SceneImportResolver( EntityType = "Location", Status = StoryImportResolutionStatuses.GenericReference, Confidence = item.Confidence, - Notes = "Generic location reference; would not create a Location record." + Notes = "Internal/generic location note; would not create a separate Location record." }); continue; } - yield return LogResolution(MatchEntity("Location", name, item.Confidence, index, StoryImportResolutionStatuses.NewLocation)); + var resolution = MatchEntity("Location", group.Key, item.Confidence, index, StoryImportResolutionStatuses.NewLocation); + var variants = group.Select(location => Clean(location.Name)).Distinct(StringComparer.OrdinalIgnoreCase).ToList(); + if (!string.Equals(group.Key, name, StringComparison.OrdinalIgnoreCase) || variants.Count > 1) + { + resolution = resolution.Status == StoryImportResolutionStatuses.NewLocation && variants.Count > 1 + ? WithStatusAndNotes(resolution, StoryImportResolutionStatuses.PossibleDuplicate, $"Possible duplicate location variant(s): {string.Join(", ", variants)}.") + : WithNotes(resolution, $"Merged location variant(s): {string.Join(", ", variants)}."); + logger.LogInformation("Merged Location {InputName} -> {NormalisedName}. Reason={Reason}", name, group.Key, "Location normalisation."); + } + + yield return LogResolution(resolution); } } @@ -151,15 +169,15 @@ public sealed class SceneImportResolver( .Select(group => group.First())) { var name = Clean(item.Name); - if (GenericAssets.Contains(name.ToLowerInvariant())) + if (IsRejectedAsset(item)) { yield return LogResolution(new StoryImportEntityResolution { InputName = name, EntityType = "Asset", - Status = StoryImportResolutionStatuses.GenericReference, + Status = StoryImportResolutionStatuses.Ignored, Confidence = item.Confidence, - Notes = "Generic object reference; would not create an Asset record unless later confirmed as meaningful." + Notes = "Rejected: environmental object or low-value prop; would not create an Asset record." }); continue; } @@ -168,26 +186,45 @@ public sealed class SceneImportResolver( } } - private static IEnumerable ResolveRelationships(IEnumerable scenes, IReadOnlyList existingRelationships) + private static IEnumerable ResolveRelationships( + IEnumerable scenes, + IReadOnlyList existingRelationships, + IReadOnlyList characterResolutions) { - foreach (var relationship in scenes.SelectMany(scene => scene.Relationships ?? [])) - { - if (string.IsNullOrWhiteSpace(relationship.CharacterA) || string.IsNullOrWhiteSpace(relationship.CharacterB)) - { - continue; - } + var resolvedCharacters = characterResolutions + .Where(item => item.Status == StoryImportResolutionStatuses.Matched) + .GroupBy(item => Clean(item.InputName), StringComparer.OrdinalIgnoreCase) + .ToDictionary(group => group.Key, group => group.First().MatchedName ?? group.First().InputName, StringComparer.OrdinalIgnoreCase); + var grouped = scenes + .SelectMany(scene => (scene.Relationships ?? []).Select(relationship => new + { + Relationship = relationship, + SceneNumber = scene.SceneReference?.SceneNumber.HasValue == true ? Convert.ToInt32(scene.SceneReference.SceneNumber.Value) : (int?)null + })) + .Where(item => !string.IsNullOrWhiteSpace(item.Relationship.CharacterA) + && !string.IsNullOrWhiteSpace(item.Relationship.CharacterB) + && resolvedCharacters.ContainsKey(Clean(item.Relationship.CharacterA)) + && resolvedCharacters.ContainsKey(Clean(item.Relationship.CharacterB))) + .GroupBy(item => RelationshipKey(resolvedCharacters[Clean(item.Relationship.CharacterA!)], resolvedCharacters[Clean(item.Relationship.CharacterB!)], item.Relationship.RelationshipSignal)); + + foreach (var group in grouped) + { + var first = group.First().Relationship; + var characterA = resolvedCharacters[Clean(first.CharacterA)]; + var characterB = resolvedCharacters[Clean(first.CharacterB)]; var exists = existingRelationships.Any(existing => - NamesMatch(existing.CharacterAName, relationship.CharacterA) && NamesMatch(existing.CharacterBName, relationship.CharacterB) - || NamesMatch(existing.CharacterAName, relationship.CharacterB) && NamesMatch(existing.CharacterBName, relationship.CharacterA)); + NamesMatch(existing.CharacterAName, characterA) && NamesMatch(existing.CharacterBName, characterB) + || NamesMatch(existing.CharacterAName, characterB) && NamesMatch(existing.CharacterBName, characterA)); yield return new StoryImportRelationshipPreview { Action = exists ? "Would Update Relationship" : "Would Create Relationship", - CharacterA = relationship.CharacterA, - CharacterB = relationship.CharacterB, - Signal = relationship.RelationshipSignal ?? "relationship signal", - Confidence = relationship.Confidence + CharacterA = characterA, + CharacterB = characterB, + Signal = first.RelationshipSignal ?? "relationship signal", + Confidence = group.Max(item => item.Relationship.Confidence), + SupportingSceneNumbers = group.Select(item => item.SceneNumber).Where(number => number.HasValue).Select(number => number!.Value).Distinct().Order().ToList() }; } } @@ -239,6 +276,22 @@ public sealed class SceneImportResolver( }; } + private static StoryImportEntityResolution WithNotes(StoryImportEntityResolution resolution, string notes) + => WithStatusAndNotes(resolution, resolution.Status, notes); + + private static StoryImportEntityResolution WithStatusAndNotes(StoryImportEntityResolution resolution, string status, string notes) + => new() + { + InputName = resolution.InputName, + Status = status, + EntityType = resolution.EntityType, + MatchedName = resolution.MatchedName, + MatchedID = resolution.MatchedID, + Confidence = resolution.Confidence, + Candidates = resolution.Candidates, + Notes = notes + }; + private async Task BuildCharacterIndexAsync(int projectId) { var rows = new List(); @@ -265,9 +318,20 @@ public sealed class SceneImportResolver( foreach (var location in await locations.ListByProjectAsync(projectId)) { rows.Add(new EntityIndexRow(location.LocationID, location.LocationName, location.LocationName, "Name")); + var normalised = NormaliseLocationName(location.LocationName); + if (!string.Equals(normalised, location.LocationName, StringComparison.OrdinalIgnoreCase)) + { + rows.Add(new EntityIndexRow(location.LocationID, location.LocationName, normalised, "Normalised name")); + } + foreach (var alias in await locations.ListAliasesAsync(location.LocationID)) { rows.Add(new EntityIndexRow(location.LocationID, location.LocationName, alias.Alias, "Alias")); + var normalisedAlias = NormaliseLocationName(alias.Alias); + if (!string.Equals(normalisedAlias, alias.Alias, StringComparison.OrdinalIgnoreCase)) + { + rows.Add(new EntityIndexRow(location.LocationID, location.LocationName, normalisedAlias, "Normalised alias")); + } } } @@ -292,13 +356,14 @@ public sealed class SceneImportResolver( private StoryImportEntityResolution LogResolution(StoryImportEntityResolution resolution) { logger.LogInformation( - "Story Intelligence Import Preview resolved {EntityType}. Input={InputName} Status={Status} Resolved={ResolvedName} Confidence={ConfidencePercent} Candidates={Candidates}", + "Story Intelligence Import Preview resolved {EntityType}. Input={InputName} Status={Status} Resolved={ResolvedName} Confidence={ConfidencePercent} Candidates={Candidates} Notes={Notes}", resolution.EntityType, resolution.InputName, resolution.Status, resolution.MatchedName, resolution.ConfidencePercent, - string.Join(", ", resolution.Candidates.Select(candidate => candidate.Name))); + string.Join(", ", resolution.Candidates.Select(candidate => candidate.Name)), + resolution.Notes); return resolution; } @@ -344,12 +409,78 @@ public sealed class SceneImportResolver( => string.Equals(location.LocationType, "generic", StringComparison.OrdinalIgnoreCase) || GenericLocations.Contains(Clean(location.Name).ToLowerInvariant()); + private static bool IsGenericPerson(string name) + { + var clean = Clean(name).ToLowerInvariant(); + return GenericPeople.Contains(clean) + || clean.StartsWith("the ", StringComparison.OrdinalIgnoreCase) && GenericPeople.Contains(clean[4..]) + || clean.Contains(" group", StringComparison.OrdinalIgnoreCase) + || clean.Contains("crowd", StringComparison.OrdinalIgnoreCase); + } + + private static bool IsRejectedAsset(SceneIntelligenceAsset asset) + { + var name = Clean(asset.Name).ToLowerInvariant(); + if (EnvironmentalAssets.Contains(name)) + { + return true; + } + + if (StoryAssetNames.Contains(RemoveLeadingArticle(name))) + { + return false; + } + + var signalText = $"{asset.Status} {asset.OwnerOrHolder} {asset.Notes} {asset.AssetType}".ToLowerInvariant(); + return !ContainsStoryAssetSignal(signalText); + } + + private static bool ContainsStoryAssetSignal(string value) + => value.Contains("sought", StringComparison.OrdinalIgnoreCase) + || value.Contains("lost", StringComparison.OrdinalIgnoreCase) + || value.Contains("hidden", StringComparison.OrdinalIgnoreCase) + || value.Contains("stolen", StringComparison.OrdinalIgnoreCase) + || value.Contains("exchanged", StringComparison.OrdinalIgnoreCase) + || value.Contains("missing", StringComparison.OrdinalIgnoreCase) + || value.Contains("important", StringComparison.OrdinalIgnoreCase) + || value.Contains("document", StringComparison.OrdinalIgnoreCase) + || value.Contains("evidence", StringComparison.OrdinalIgnoreCase); + + private static string NormaliseLocationName(string? value) + { + var clean = RemoveLeadingArticle(Clean(value)); + var lower = clean.ToLowerInvariant(); + var suffixes = new[] + { + " sixth floor reception", " sixth-floor reception", " reception", " sixth floor", " office", " corridor", " stairwell" + }; + + foreach (var suffix in suffixes) + { + if (lower.EndsWith(suffix, StringComparison.OrdinalIgnoreCase) && clean.Length > suffix.Length) + { + return clean[..^suffix.Length].Trim(); + } + } + + return clean; + } + private static string Clean(string? value) => string.Join(' ', (value ?? string.Empty).Trim().Split((char[]?)null, StringSplitOptions.RemoveEmptyEntries)); + private static string RemoveLeadingArticle(string value) + => value.StartsWith("the ", StringComparison.OrdinalIgnoreCase) ? value[4..].Trim() : value; + private static bool NamesMatch(string? first, string? second) => string.Equals(Clean(first), Clean(second), StringComparison.OrdinalIgnoreCase); + private static string RelationshipKey(string characterA, string characterB, string? signal) + { + var names = new[] { Clean(characterA), Clean(characterB) }.Order(StringComparer.OrdinalIgnoreCase).ToArray(); + return $"{names[0]}|{names[1]}|{Clean(signal).ToLowerInvariant()}"; + } + private static HashSet CreateSet(params string[] values) => new(values, StringComparer.OrdinalIgnoreCase); @@ -362,9 +493,11 @@ public sealed class SceneImportResolver( public IReadOnlyList Find(string value) { var clean = Clean(value); + var withoutArticle = RemoveLeadingArticle(clean); var matches = rows .Where(row => string.Equals(Clean(row.MatchValue), clean, StringComparison.Ordinal) - || string.Equals(Clean(row.MatchValue), clean, StringComparison.OrdinalIgnoreCase)) + || string.Equals(Clean(row.MatchValue), clean, StringComparison.OrdinalIgnoreCase) + || string.Equals(RemoveLeadingArticle(Clean(row.MatchValue)), withoutArticle, StringComparison.OrdinalIgnoreCase)) .GroupBy(row => row.EntityID) .Select(group => { diff --git a/PlotLine/Views/Admin/StoryIntelligenceRunDetails.cshtml b/PlotLine/Views/Admin/StoryIntelligenceRunDetails.cshtml index 737efe4..e446f73 100644 --- a/PlotLine/Views/Admin/StoryIntelligenceRunDetails.cshtml +++ b/PlotLine/Views/Admin/StoryIntelligenceRunDetails.cshtml @@ -269,8 +269,8 @@ else } -

Timeline

-

@preview.Timeline.Count.ToString("N0") scene timeline entr@(preview.Timeline.Count == 1 ? "y" : "ies") would be created.

+

Timeline Clues

+

@preview.Timeline.Count.ToString("N0") timeline clue@(preview.Timeline.Count == 1 ? string.Empty : "s") detected.

@if (preview.Timeline.Count > 0) {
    @@ -373,10 +373,12 @@ else } var builder = new System.Text.StringBuilder(); - foreach (var group in items.GroupBy(item => item.Status)) + foreach (var group in items + .GroupBy(item => GroupLabel(item.Status)) + .OrderBy(group => GroupSort(group.Key))) { builder.Append("

    ").Append(Encode(group.Key)).Append("

      "); - foreach (var item in group) + foreach (var item in group.OrderBy(item => item.MatchedName ?? item.InputName, StringComparer.OrdinalIgnoreCase)) { builder.Append("
    • "); builder.Append(IconForStatus(item.Status)).Append(' '); @@ -413,10 +415,33 @@ else { StoryImportResolutionStatuses.Matched => "✓", StoryImportResolutionStatuses.Ambiguous => "⚠", + StoryImportResolutionStatuses.PossibleDuplicate => "⚠", StoryImportResolutionStatuses.NewCharacter or StoryImportResolutionStatuses.NewLocation or StoryImportResolutionStatuses.NewAsset => "+", _ => "•" }; + private static string GroupLabel(string status) + => status switch + { + StoryImportResolutionStatuses.NewCharacter or StoryImportResolutionStatuses.NewLocation or StoryImportResolutionStatuses.NewAsset => "New", + StoryImportResolutionStatuses.GenericReference => "Generic", + StoryImportResolutionStatuses.UnresolvedPersonReference => "Unresolved", + _ => status + }; + + private static int GroupSort(string group) + => group switch + { + "Matched" => 0, + "New" => 1, + "Possible Duplicate" => 2, + "Ambiguous" => 3, + "Ignored" => 4, + "Generic" => 5, + "Unresolved" => 6, + _ => 99 + }; + private static string Encode(string? value) => System.Net.WebUtility.HtmlEncode(value ?? string.Empty);