Phase 20T – Entity Resolution Refinement

PlotLine/Models/StoryIntelligenceImportPreviewModels.cs

@@ -6,6 +6,7 @@ public static class StoryImportResolutionStatuses
     public const string NewCharacter = "New Character";
     public const string NewLocation = "New Location";
     public const string NewAsset = "New Asset";
+    public const string PossibleDuplicate = "Possible Duplicate";
     public const string Ambiguous = "Ambiguous";
     public const string UnresolvedPersonReference = "Unresolved Person Reference";
     public const string GenericReference = "Generic Reference";
@@ -59,6 +60,7 @@ public sealed class StoryImportRelationshipPreview
     public string CharacterA { get; init; } = string.Empty;
     public string CharacterB { get; init; } = string.Empty;
     public string Signal { get; init; } = string.Empty;
+    public IReadOnlyList<int> SupportingSceneNumbers { get; init; } = [];
     public decimal? Confidence { get; init; }
     public int? ConfidencePercent => Confidence.HasValue ? (int)Math.Round(Confidence.Value * 100m) : null;
 }
@@ -76,7 +78,7 @@ public sealed class StoryImportKnowledgePreview
 
 public sealed class StoryImportTimelinePreview
 {
-    public string Action { get; init; } = "Would Create Scene Timeline Entry";
+    public string Action { get; init; } = "Timeline Clue";
     public int? SceneNumber { get; init; }
     public string SceneDate { get; init; } = string.Empty;
     public decimal? Confidence { get; init; }

PlotLine/Services/SceneImportResolver.cs

@@ -23,16 +23,24 @@ public sealed class SceneImportResolver(
 
     private static readonly HashSet<string> GenericPeople = CreateSet(
         "the receptionist", "receptionist", "old man", "young woman", "police officer", "doctor", "nurse", "driver",
-        "man", "woman", "boy", "girl", "child", "person", "guard", "waiter", "waitress", "clerk", "shopkeeper");
+        "man", "woman", "women", "boy", "girl", "child", "children", "person", "people", "guard", "security guard",
+        "cashier", "elderly attendant", "attendant", "driver of car behind", "shop assistant", "crowd", "family",
+        "group of women", "small group", "women (small group)", "waiter", "waitress", "clerk", "shopkeeper");
 
     private static readonly HashSet<string> IgnoredPovNames = CreateSet("narrator", "unknown");
 
     private static readonly HashSet<string> GenericLocations = CreateSet(
-        "car park", "road", "house", "kitchen", "stairwell", "stair", "doorway", "corridor", "street", "room",
+        "car park", "house", "kitchen", "stairwell", "stair", "doorway", "corridor", "room",
-        "bedroom", "bathroom", "office", "car", "church", "hall", "table", "chair");
+        "bedroom", "bathroom", "office", "reception", "sixth floor", "floor", "car", "hall", "table", "chair");
 
-    private static readonly HashSet<string> GenericAssets = CreateSet(
+    private static readonly HashSet<string> EnvironmentalAssets = CreateSet(
-        "the key", "key", "the letter", "letter", "the car", "car", "phone", "table", "chair", "door", "window");
+        "pink towel", "mug of tea", "magazine", "fire door", "old oak door", "iron handle", "reception desk",
+        "stained-glass windows", "stained glass windows", "chair", "desk", "window", "windows", "cup", "floor",
+        "wall", "door", "table", "towel", "mug", "handle");
+
+    private static readonly HashSet<string> StoryAssetNames = CreateSet(
+        "car", "vehicle", "passport", "letter", "key", "photograph", "photo", "bank book", "bank account",
+        "necklace", "weapon", "folder", "document", "diary", "telephone number", "phone number", "scrap of paper", "wallet");
 
     public async Task<StoryIntelligenceImportPreview> ResolveAsync(StoryIntelligenceSavedRun run, IReadOnlyList<StoryIntelligenceSavedSceneResult> sceneResults)
     {
@@ -59,7 +67,7 @@ public sealed class SceneImportResolver(
         var characterResolutions = ResolveCharacters(scenes, characterIndex).ToList();
         var locationResolutions = ResolveLocations(scenes, locationIndex).ToList();
         var assetResolutions = ResolveAssets(scenes, assetIndex).ToList();
-        var relationshipPreview = ResolveRelationships(scenes, existingRelationships).ToList();
+        var relationshipPreview = ResolveRelationships(scenes, existingRelationships, characterResolutions).ToList();
         var knowledgePreview = ResolveKnowledge(scenes).ToList();
         var timelinePreview = ResolveTimeline(scenes).ToList();
         var warnings = BuildWarnings(characterResolutions, locationResolutions, assetResolutions).ToList();
@@ -99,7 +107,7 @@ public sealed class SceneImportResolver(
                 continue;
             }
 
-            if (GenericPeople.Contains(name.ToLowerInvariant()))
+            if (IsGenericPerson(name))
             {
                 yield return LogResolution(new StoryImportEntityResolution
                 {
@@ -107,7 +115,7 @@ public sealed class SceneImportResolver(
                     EntityType = "Character",
                     Status = StoryImportResolutionStatuses.UnresolvedPersonReference,
                     Confidence = item.Confidence,
-                    Notes = "Generic person reference; would not create a Character record."
+                    Notes = "Rejected: generic unnamed role; would not create a Character record."
                 });
                 continue;
             }
@@ -118,12 +126,12 @@ public sealed class SceneImportResolver(
 
     private IEnumerable<StoryImportEntityResolution> ResolveLocations(IEnumerable<SceneIntelligenceScene> scenes, EntityIndex index)
     {
-        foreach (var item in scenes
+        foreach (var group in scenes
             .SelectMany(scene => scene.Locations ?? [])
             .Where(location => !string.IsNullOrWhiteSpace(location.Name))
-            .GroupBy(location => Clean(location.Name), StringComparer.OrdinalIgnoreCase)
+            .GroupBy(location => NormaliseLocationName(location.Name), StringComparer.OrdinalIgnoreCase))
-            .Select(group => group.First()))
         {
+            var item = group.First();
             var name = Clean(item.Name);
             if (IsGenericLocation(item))
             {
@@ -133,12 +141,22 @@ public sealed class SceneImportResolver(
                     EntityType = "Location",
                     Status = StoryImportResolutionStatuses.GenericReference,
                     Confidence = item.Confidence,
-                    Notes = "Generic location reference; would not create a Location record."
+                    Notes = "Internal/generic location note; would not create a separate Location record."
                 });
                 continue;
             }
 
-            yield return LogResolution(MatchEntity("Location", name, item.Confidence, index, StoryImportResolutionStatuses.NewLocation));
+            var resolution = MatchEntity("Location", group.Key, item.Confidence, index, StoryImportResolutionStatuses.NewLocation);
+            var variants = group.Select(location => Clean(location.Name)).Distinct(StringComparer.OrdinalIgnoreCase).ToList();
+            if (!string.Equals(group.Key, name, StringComparison.OrdinalIgnoreCase) || variants.Count > 1)
+            {
+                resolution = resolution.Status == StoryImportResolutionStatuses.NewLocation && variants.Count > 1
+                    ? WithStatusAndNotes(resolution, StoryImportResolutionStatuses.PossibleDuplicate, $"Possible duplicate location variant(s): {string.Join(", ", variants)}.")
+                    : WithNotes(resolution, $"Merged location variant(s): {string.Join(", ", variants)}.");
+                logger.LogInformation("Merged Location {InputName} -> {NormalisedName}. Reason={Reason}", name, group.Key, "Location normalisation.");
+            }
+
+            yield return LogResolution(resolution);
         }
     }
 
@@ -151,15 +169,15 @@ public sealed class SceneImportResolver(
             .Select(group => group.First()))
         {
             var name = Clean(item.Name);
-            if (GenericAssets.Contains(name.ToLowerInvariant()))
+            if (IsRejectedAsset(item))
             {
                 yield return LogResolution(new StoryImportEntityResolution
                 {
                     InputName = name,
                     EntityType = "Asset",
-                    Status = StoryImportResolutionStatuses.GenericReference,
+                    Status = StoryImportResolutionStatuses.Ignored,
                     Confidence = item.Confidence,
-                    Notes = "Generic object reference; would not create an Asset record unless later confirmed as meaningful."
+                    Notes = "Rejected: environmental object or low-value prop; would not create an Asset record."
                 });
                 continue;
             }
@@ -168,26 +186,45 @@ public sealed class SceneImportResolver(
         }
     }
 
-    private static IEnumerable<StoryImportRelationshipPreview> ResolveRelationships(IEnumerable<SceneIntelligenceScene> scenes, IReadOnlyList<CharacterRelationship> existingRelationships)
+    private static IEnumerable<StoryImportRelationshipPreview> ResolveRelationships(
+        IEnumerable<SceneIntelligenceScene> scenes,
+        IReadOnlyList<CharacterRelationship> existingRelationships,
+        IReadOnlyList<StoryImportEntityResolution> characterResolutions)
     {
-        foreach (var relationship in scenes.SelectMany(scene => scene.Relationships ?? []))
+        var resolvedCharacters = characterResolutions
-        {
+            .Where(item => item.Status == StoryImportResolutionStatuses.Matched)
-            if (string.IsNullOrWhiteSpace(relationship.CharacterA) || string.IsNullOrWhiteSpace(relationship.CharacterB))
+            .GroupBy(item => Clean(item.InputName), StringComparer.OrdinalIgnoreCase)
-            {
+            .ToDictionary(group => group.Key, group => group.First().MatchedName ?? group.First().InputName, StringComparer.OrdinalIgnoreCase);
-                continue;
-            }
 
+        var grouped = scenes
+            .SelectMany(scene => (scene.Relationships ?? []).Select(relationship => new
+            {
+                Relationship = relationship,
+                SceneNumber = scene.SceneReference?.SceneNumber.HasValue == true ? Convert.ToInt32(scene.SceneReference.SceneNumber.Value) : (int?)null
+            }))
+            .Where(item => !string.IsNullOrWhiteSpace(item.Relationship.CharacterA)
+                && !string.IsNullOrWhiteSpace(item.Relationship.CharacterB)
+                && resolvedCharacters.ContainsKey(Clean(item.Relationship.CharacterA))
+                && resolvedCharacters.ContainsKey(Clean(item.Relationship.CharacterB)))
+            .GroupBy(item => RelationshipKey(resolvedCharacters[Clean(item.Relationship.CharacterA!)], resolvedCharacters[Clean(item.Relationship.CharacterB!)], item.Relationship.RelationshipSignal));
+
+        foreach (var group in grouped)
+        {
+            var first = group.First().Relationship;
+            var characterA = resolvedCharacters[Clean(first.CharacterA)];
+            var characterB = resolvedCharacters[Clean(first.CharacterB)];
             var exists = existingRelationships.Any(existing =>
-                NamesMatch(existing.CharacterAName, relationship.CharacterA) && NamesMatch(existing.CharacterBName, relationship.CharacterB)
+                NamesMatch(existing.CharacterAName, characterA) && NamesMatch(existing.CharacterBName, characterB)
-                || NamesMatch(existing.CharacterAName, relationship.CharacterB) && NamesMatch(existing.CharacterBName, relationship.CharacterA));
+                || NamesMatch(existing.CharacterAName, characterB) && NamesMatch(existing.CharacterBName, characterA));
 
             yield return new StoryImportRelationshipPreview
             {
                 Action = exists ? "Would Update Relationship" : "Would Create Relationship",
-                CharacterA = relationship.CharacterA,
+                CharacterA = characterA,
-                CharacterB = relationship.CharacterB,
+                CharacterB = characterB,
-                Signal = relationship.RelationshipSignal ?? "relationship signal",
+                Signal = first.RelationshipSignal ?? "relationship signal",
-                Confidence = relationship.Confidence
+                Confidence = group.Max(item => item.Relationship.Confidence),
+                SupportingSceneNumbers = group.Select(item => item.SceneNumber).Where(number => number.HasValue).Select(number => number!.Value).Distinct().Order().ToList()
             };
         }
     }
@@ -239,6 +276,22 @@ public sealed class SceneImportResolver(
         };
     }
 
+    private static StoryImportEntityResolution WithNotes(StoryImportEntityResolution resolution, string notes)
+        => WithStatusAndNotes(resolution, resolution.Status, notes);
+
+    private static StoryImportEntityResolution WithStatusAndNotes(StoryImportEntityResolution resolution, string status, string notes)
+        => new()
+        {
+            InputName = resolution.InputName,
+            Status = status,
+            EntityType = resolution.EntityType,
+            MatchedName = resolution.MatchedName,
+            MatchedID = resolution.MatchedID,
+            Confidence = resolution.Confidence,
+            Candidates = resolution.Candidates,
+            Notes = notes
+        };
+
     private async Task<EntityIndex> BuildCharacterIndexAsync(int projectId)
     {
         var rows = new List<EntityIndexRow>();
@@ -265,9 +318,20 @@ public sealed class SceneImportResolver(
         foreach (var location in await locations.ListByProjectAsync(projectId))
         {
             rows.Add(new EntityIndexRow(location.LocationID, location.LocationName, location.LocationName, "Name"));
+            var normalised = NormaliseLocationName(location.LocationName);
+            if (!string.Equals(normalised, location.LocationName, StringComparison.OrdinalIgnoreCase))
+            {
+                rows.Add(new EntityIndexRow(location.LocationID, location.LocationName, normalised, "Normalised name"));
+            }
+
             foreach (var alias in await locations.ListAliasesAsync(location.LocationID))
             {
                 rows.Add(new EntityIndexRow(location.LocationID, location.LocationName, alias.Alias, "Alias"));
+                var normalisedAlias = NormaliseLocationName(alias.Alias);
+                if (!string.Equals(normalisedAlias, alias.Alias, StringComparison.OrdinalIgnoreCase))
+                {
+                    rows.Add(new EntityIndexRow(location.LocationID, location.LocationName, normalisedAlias, "Normalised alias"));
+                }
             }
         }
 
@@ -292,13 +356,14 @@ public sealed class SceneImportResolver(
     private StoryImportEntityResolution LogResolution(StoryImportEntityResolution resolution)
     {
         logger.LogInformation(
-            "Story Intelligence Import Preview resolved {EntityType}. Input={InputName} Status={Status} Resolved={ResolvedName} Confidence={ConfidencePercent} Candidates={Candidates}",
+            "Story Intelligence Import Preview resolved {EntityType}. Input={InputName} Status={Status} Resolved={ResolvedName} Confidence={ConfidencePercent} Candidates={Candidates} Notes={Notes}",
             resolution.EntityType,
             resolution.InputName,
             resolution.Status,
             resolution.MatchedName,
             resolution.ConfidencePercent,
-            string.Join(", ", resolution.Candidates.Select(candidate => candidate.Name)));
+            string.Join(", ", resolution.Candidates.Select(candidate => candidate.Name)),
+            resolution.Notes);
         return resolution;
     }
 
@@ -344,12 +409,78 @@ public sealed class SceneImportResolver(
         => string.Equals(location.LocationType, "generic", StringComparison.OrdinalIgnoreCase)
             || GenericLocations.Contains(Clean(location.Name).ToLowerInvariant());
 
+    private static bool IsGenericPerson(string name)
+    {
+        var clean = Clean(name).ToLowerInvariant();
+        return GenericPeople.Contains(clean)
+            || clean.StartsWith("the ", StringComparison.OrdinalIgnoreCase) && GenericPeople.Contains(clean[4..])
+            || clean.Contains(" group", StringComparison.OrdinalIgnoreCase)
+            || clean.Contains("crowd", StringComparison.OrdinalIgnoreCase);
+    }
+
+    private static bool IsRejectedAsset(SceneIntelligenceAsset asset)
+    {
+        var name = Clean(asset.Name).ToLowerInvariant();
+        if (EnvironmentalAssets.Contains(name))
+        {
+            return true;
+        }
+
+        if (StoryAssetNames.Contains(RemoveLeadingArticle(name)))
+        {
+            return false;
+        }
+
+        var signalText = $"{asset.Status} {asset.OwnerOrHolder} {asset.Notes} {asset.AssetType}".ToLowerInvariant();
+        return !ContainsStoryAssetSignal(signalText);
+    }
+
+    private static bool ContainsStoryAssetSignal(string value)
+        => value.Contains("sought", StringComparison.OrdinalIgnoreCase)
+            || value.Contains("lost", StringComparison.OrdinalIgnoreCase)
+            || value.Contains("hidden", StringComparison.OrdinalIgnoreCase)
+            || value.Contains("stolen", StringComparison.OrdinalIgnoreCase)
+            || value.Contains("exchanged", StringComparison.OrdinalIgnoreCase)
+            || value.Contains("missing", StringComparison.OrdinalIgnoreCase)
+            || value.Contains("important", StringComparison.OrdinalIgnoreCase)
+            || value.Contains("document", StringComparison.OrdinalIgnoreCase)
+            || value.Contains("evidence", StringComparison.OrdinalIgnoreCase);
+
+    private static string NormaliseLocationName(string? value)
+    {
+        var clean = RemoveLeadingArticle(Clean(value));
+        var lower = clean.ToLowerInvariant();
+        var suffixes = new[]
+        {
+            " sixth floor reception", " sixth-floor reception", " reception", " sixth floor", " office", " corridor", " stairwell"
+        };
+
+        foreach (var suffix in suffixes)
+        {
+            if (lower.EndsWith(suffix, StringComparison.OrdinalIgnoreCase) && clean.Length > suffix.Length)
+            {
+                return clean[..^suffix.Length].Trim();
+            }
+        }
+
+        return clean;
+    }
+
     private static string Clean(string? value)
         => string.Join(' ', (value ?? string.Empty).Trim().Split((char[]?)null, StringSplitOptions.RemoveEmptyEntries));
 
+    private static string RemoveLeadingArticle(string value)
+        => value.StartsWith("the ", StringComparison.OrdinalIgnoreCase) ? value[4..].Trim() : value;
+
     private static bool NamesMatch(string? first, string? second)
         => string.Equals(Clean(first), Clean(second), StringComparison.OrdinalIgnoreCase);
 
+    private static string RelationshipKey(string characterA, string characterB, string? signal)
+    {
+        var names = new[] { Clean(characterA), Clean(characterB) }.Order(StringComparer.OrdinalIgnoreCase).ToArray();
+        return $"{names[0]}|{names[1]}|{Clean(signal).ToLowerInvariant()}";
+    }
+
     private static HashSet<string> CreateSet(params string[] values)
         => new(values, StringComparer.OrdinalIgnoreCase);
 
@@ -362,9 +493,11 @@ public sealed class SceneImportResolver(
         public IReadOnlyList<StoryImportResolutionCandidate> Find(string value)
         {
             var clean = Clean(value);
+            var withoutArticle = RemoveLeadingArticle(clean);
             var matches = rows
                 .Where(row => string.Equals(Clean(row.MatchValue), clean, StringComparison.Ordinal)
-                    || string.Equals(Clean(row.MatchValue), clean, StringComparison.OrdinalIgnoreCase))
+                    || string.Equals(Clean(row.MatchValue), clean, StringComparison.OrdinalIgnoreCase)
+                    || string.Equals(RemoveLeadingArticle(Clean(row.MatchValue)), withoutArticle, StringComparison.OrdinalIgnoreCase))
                 .GroupBy(row => row.EntityID)
                 .Select(group =>
                 {

PlotLine/Views/Admin/StoryIntelligenceRunDetails.cshtml

@@ -269,8 +269,8 @@ else
                 </ul>
             }
 
-            <h3>Timeline</h3>
+            <h3>Timeline Clues</h3>
-            <p>@preview.Timeline.Count.ToString("N0") scene timeline entr@(preview.Timeline.Count == 1 ? "y" : "ies") would be created.</p>
+            <p>@preview.Timeline.Count.ToString("N0") timeline clue@(preview.Timeline.Count == 1 ? string.Empty : "s") detected.</p>
             @if (preview.Timeline.Count > 0)
             {
                 <ul>
@@ -373,10 +373,12 @@ else
         }
 
         var builder = new System.Text.StringBuilder();
-        foreach (var group in items.GroupBy(item => item.Status))
+        foreach (var group in items
+            .GroupBy(item => GroupLabel(item.Status))
+            .OrderBy(group => GroupSort(group.Key)))
         {
             builder.Append("<h4>").Append(Encode(group.Key)).Append("</h4><ul>");
-            foreach (var item in group)
+            foreach (var item in group.OrderBy(item => item.MatchedName ?? item.InputName, StringComparer.OrdinalIgnoreCase))
             {
                 builder.Append("<li>");
                 builder.Append(IconForStatus(item.Status)).Append(' ');
@@ -413,10 +415,33 @@ else
         {
             StoryImportResolutionStatuses.Matched => "✓",
             StoryImportResolutionStatuses.Ambiguous => "⚠",
+            StoryImportResolutionStatuses.PossibleDuplicate => "⚠",
             StoryImportResolutionStatuses.NewCharacter or StoryImportResolutionStatuses.NewLocation or StoryImportResolutionStatuses.NewAsset => "+",
             _ => "•"
         };
 
+    private static string GroupLabel(string status)
+        => status switch
+        {
+            StoryImportResolutionStatuses.NewCharacter or StoryImportResolutionStatuses.NewLocation or StoryImportResolutionStatuses.NewAsset => "New",
+            StoryImportResolutionStatuses.GenericReference => "Generic",
+            StoryImportResolutionStatuses.UnresolvedPersonReference => "Unresolved",
+            _ => status
+        };
+
+    private static int GroupSort(string group)
+        => group switch
+        {
+            "Matched" => 0,
+            "New" => 1,
+            "Possible Duplicate" => 2,
+            "Ambiguous" => 3,
+            "Ignored" => 4,
+            "Generic" => 5,
+            "Unresolved" => 6,
+            _ => 99
+        };
+
     private static string Encode(string? value)
         => System.Net.WebUtility.HtmlEncode(value ?? string.Empty);