nick/PlotDirector
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Phase 20T – Entity Resolution Refinement

Browse Source
This commit is contained in:
Nick Beckley 2026-07-05 19:45:23 +01:00
parent 1dbf49ca4c
commit 6325fbfec5
3 changed files with 197 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
PlotLine/Models/StoryIntelligenceImportPreviewModels.cs
View File

@ -6,6 +6,7 @@ public static class StoryImportResolutionStatuses
public const string NewCharacter = "New Character";
public const string NewLocation = "New Location";
public const string NewAsset = "New Asset";
public const string PossibleDuplicate = "Possible Duplicate";
public const string Ambiguous = "Ambiguous";
public const string UnresolvedPersonReference = "Unresolved Person Reference";
public const string GenericReference = "Generic Reference";
@ -59,6 +60,7 @@ public sealed class StoryImportRelationshipPreview
public string CharacterA { get; init; } = string.Empty;
public string CharacterB { get; init; } = string.Empty;
public string Signal { get; init; } = string.Empty;
public IReadOnlyList<int> SupportingSceneNumbers { get; init; } = [];
public decimal? Confidence { get; init; }
public int? ConfidencePercent => Confidence.HasValue ? (int)Math.Round(Confidence.Value * 100m) : null;
}
@ -76,7 +78,7 @@ public sealed class StoryImportKnowledgePreview
public sealed class StoryImportTimelinePreview
{
public string Action { get; init; } = "Would Create Scene Timeline Entry";
public string Action { get; init; } = "Timeline Clue";
public int? SceneNumber { get; init; }
public string SceneDate { get; init; } = string.Empty;
public decimal? Confidence { get; init; }

197
PlotLine/Services/SceneImportResolver.cs
View File

@ -23,16 +23,24 @@ public sealed class SceneImportResolver(
private static readonly HashSet<string> GenericPeople = CreateSet(
"the receptionist", "receptionist", "old man", "young woman", "police officer", "doctor", "nurse", "driver",
"man", "woman", "boy", "girl", "child", "person", "guard", "waiter", "waitress", "clerk", "shopkeeper");
"man", "woman", "women", "boy", "girl", "child", "children", "person", "people", "guard", "security guard",
"cashier", "elderly attendant", "attendant", "driver of car behind", "shop assistant", "crowd", "family",
"group of women", "small group", "women (small group)", "waiter", "waitress", "clerk", "shopkeeper");
private static readonly HashSet<string> IgnoredPovNames = CreateSet("narrator", "unknown");
private static readonly HashSet<string> GenericLocations = CreateSet(
"car park", "road", "house", "kitchen", "stairwell", "stair", "doorway", "corridor", "street", "room",
"bedroom", "bathroom", "office", "car", "church", "hall", "table", "chair");
"car park", "house", "kitchen", "stairwell", "stair", "doorway", "corridor", "room",
"bedroom", "bathroom", "office", "reception", "sixth floor", "floor", "car", "hall", "table", "chair");
private static readonly HashSet<string> GenericAssets = CreateSet(
"the key", "key", "the letter", "letter", "the car", "car", "phone", "table", "chair", "door", "window");
private static readonly HashSet<string> EnvironmentalAssets = CreateSet(
"pink towel", "mug of tea", "magazine", "fire door", "old oak door", "iron handle", "reception desk",
"stained-glass windows", "stained glass windows", "chair", "desk", "window", "windows", "cup", "floor",
"wall", "door", "table", "towel", "mug", "handle");
private static readonly HashSet<string> StoryAssetNames = CreateSet(
"car", "vehicle", "passport", "letter", "key", "photograph", "photo", "bank book", "bank account",
"necklace", "weapon", "folder", "document", "diary", "telephone number", "phone number", "scrap of paper", "wallet");
public async Task<StoryIntelligenceImportPreview> ResolveAsync(StoryIntelligenceSavedRun run, IReadOnlyList<StoryIntelligenceSavedSceneResult> sceneResults)
{
@ -59,7 +67,7 @@ public sealed class SceneImportResolver(
var characterResolutions = ResolveCharacters(scenes, characterIndex).ToList();
var locationResolutions = ResolveLocations(scenes, locationIndex).ToList();
var assetResolutions = ResolveAssets(scenes, assetIndex).ToList();
var relationshipPreview = ResolveRelationships(scenes, existingRelationships).ToList();
var relationshipPreview = ResolveRelationships(scenes, existingRelationships, characterResolutions).ToList();
var knowledgePreview = ResolveKnowledge(scenes).ToList();
var timelinePreview = ResolveTimeline(scenes).ToList();
var warnings = BuildWarnings(characterResolutions, locationResolutions, assetResolutions).ToList();
@ -99,7 +107,7 @@ public sealed class SceneImportResolver(
continue;
}
if (GenericPeople.Contains(name.ToLowerInvariant()))
if (IsGenericPerson(name))
{
yield return LogResolution(new StoryImportEntityResolution
{
@ -107,7 +115,7 @@ public sealed class SceneImportResolver(
EntityType = "Character",
Status = StoryImportResolutionStatuses.UnresolvedPersonReference,
Confidence = item.Confidence,
Notes = "Generic person reference; would not create a Character record."
Notes = "Rejected: generic unnamed role; would not create a Character record."
});
continue;
}
@ -118,12 +126,12 @@ public sealed class SceneImportResolver(
private IEnumerable<StoryImportEntityResolution> ResolveLocations(IEnumerable<SceneIntelligenceScene> scenes, EntityIndex index)
{
foreach (var item in scenes
foreach (var group in scenes
.SelectMany(scene => scene.Locations ?? [])
.Where(location => !string.IsNullOrWhiteSpace(location.Name))
.GroupBy(location => Clean(location.Name), StringComparer.OrdinalIgnoreCase)
.Select(group => group.First()))
.GroupBy(location => NormaliseLocationName(location.Name), StringComparer.OrdinalIgnoreCase))
{
var item = group.First();
var name = Clean(item.Name);
if (IsGenericLocation(item))
{
@ -133,12 +141,22 @@ public sealed class SceneImportResolver(
EntityType = "Location",
Status = StoryImportResolutionStatuses.GenericReference,
Confidence = item.Confidence,
Notes = "Generic location reference; would not create a Location record."
Notes = "Internal/generic location note; would not create a separate Location record."
});
continue;
}
yield return LogResolution(MatchEntity("Location", name, item.Confidence, index, StoryImportResolutionStatuses.NewLocation));
var resolution = MatchEntity("Location", group.Key, item.Confidence, index, StoryImportResolutionStatuses.NewLocation);
var variants = group.Select(location => Clean(location.Name)).Distinct(StringComparer.OrdinalIgnoreCase).ToList();
if (!string.Equals(group.Key, name, StringComparison.OrdinalIgnoreCase) || variants.Count > 1)
{
resolution = resolution.Status == StoryImportResolutionStatuses.NewLocation && variants.Count > 1
? WithStatusAndNotes(resolution, StoryImportResolutionStatuses.PossibleDuplicate, $"Possible duplicate location variant(s): {string.Join(", ", variants)}.")
: WithNotes(resolution, $"Merged location variant(s): {string.Join(", ", variants)}.");
logger.LogInformation("Merged Location {InputName} -> {NormalisedName}. Reason={Reason}", name, group.Key, "Location normalisation.");
}
yield return LogResolution(resolution);
}
}
@ -151,15 +169,15 @@ public sealed class SceneImportResolver(
.Select(group => group.First()))
{
var name = Clean(item.Name);
if (GenericAssets.Contains(name.ToLowerInvariant()))
if (IsRejectedAsset(item))
{
yield return LogResolution(new StoryImportEntityResolution
{
InputName = name,
EntityType = "Asset",
Status = StoryImportResolutionStatuses.GenericReference,
Status = StoryImportResolutionStatuses.Ignored,
Confidence = item.Confidence,
Notes = "Generic object reference; would not create an Asset record unless later confirmed as meaningful."
Notes = "Rejected: environmental object or low-value prop; would not create an Asset record."
});
continue;
}
@ -168,26 +186,45 @@ public sealed class SceneImportResolver(
}
}
private static IEnumerable<StoryImportRelationshipPreview> ResolveRelationships(IEnumerable<SceneIntelligenceScene> scenes, IReadOnlyList<CharacterRelationship> existingRelationships)
private static IEnumerable<StoryImportRelationshipPreview> ResolveRelationships(
IEnumerable<SceneIntelligenceScene> scenes,
IReadOnlyList<CharacterRelationship> existingRelationships,
IReadOnlyList<StoryImportEntityResolution> characterResolutions)
{
foreach (var relationship in scenes.SelectMany(scene => scene.Relationships ?? []))
{
if (string.IsNullOrWhiteSpace(relationship.CharacterA) || string.IsNullOrWhiteSpace(relationship.CharacterB))
{
continue;
}
var resolvedCharacters = characterResolutions
.Where(item => item.Status == StoryImportResolutionStatuses.Matched)
.GroupBy(item => Clean(item.InputName), StringComparer.OrdinalIgnoreCase)
.ToDictionary(group => group.Key, group => group.First().MatchedName ?? group.First().InputName, StringComparer.OrdinalIgnoreCase);
var grouped = scenes
.SelectMany(scene => (scene.Relationships ?? []).Select(relationship => new
{
Relationship = relationship,
SceneNumber = scene.SceneReference?.SceneNumber.HasValue == true ? Convert.ToInt32(scene.SceneReference.SceneNumber.Value) : (int?)null
}))
.Where(item => !string.IsNullOrWhiteSpace(item.Relationship.CharacterA)
&& !string.IsNullOrWhiteSpace(item.Relationship.CharacterB)
&& resolvedCharacters.ContainsKey(Clean(item.Relationship.CharacterA))
&& resolvedCharacters.ContainsKey(Clean(item.Relationship.CharacterB)))
.GroupBy(item => RelationshipKey(resolvedCharacters[Clean(item.Relationship.CharacterA!)], resolvedCharacters[Clean(item.Relationship.CharacterB!)], item.Relationship.RelationshipSignal));
foreach (var group in grouped)
{
var first = group.First().Relationship;
var characterA = resolvedCharacters[Clean(first.CharacterA)];
var characterB = resolvedCharacters[Clean(first.CharacterB)];
var exists = existingRelationships.Any(existing =>
NamesMatch(existing.CharacterAName, relationship.CharacterA) && NamesMatch(existing.CharacterBName, relationship.CharacterB)
|| NamesMatch(existing.CharacterAName, relationship.CharacterB) && NamesMatch(existing.CharacterBName, relationship.CharacterA));
NamesMatch(existing.CharacterAName, characterA) && NamesMatch(existing.CharacterBName, characterB)
|| NamesMatch(existing.CharacterAName, characterB) && NamesMatch(existing.CharacterBName, characterA));
yield return new StoryImportRelationshipPreview
{
Action = exists ? "Would Update Relationship" : "Would Create Relationship",
CharacterA = relationship.CharacterA,
CharacterB = relationship.CharacterB,
Signal = relationship.RelationshipSignal ?? "relationship signal",
Confidence = relationship.Confidence
CharacterA = characterA,
CharacterB = characterB,
Signal = first.RelationshipSignal ?? "relationship signal",
Confidence = group.Max(item => item.Relationship.Confidence),
SupportingSceneNumbers = group.Select(item => item.SceneNumber).Where(number => number.HasValue).Select(number => number!.Value).Distinct().Order().ToList()
};
}
}
@ -239,6 +276,22 @@ public sealed class SceneImportResolver(
};
}
private static StoryImportEntityResolution WithNotes(StoryImportEntityResolution resolution, string notes)
=> WithStatusAndNotes(resolution, resolution.Status, notes);
private static StoryImportEntityResolution WithStatusAndNotes(StoryImportEntityResolution resolution, string status, string notes)
=> new()
{
InputName = resolution.InputName,
Status = status,
EntityType = resolution.EntityType,
MatchedName = resolution.MatchedName,
MatchedID = resolution.MatchedID,
Confidence = resolution.Confidence,
Candidates = resolution.Candidates,
Notes = notes
};
private async Task<EntityIndex> BuildCharacterIndexAsync(int projectId)
{
var rows = new List<EntityIndexRow>();
@ -265,9 +318,20 @@ public sealed class SceneImportResolver(
foreach (var location in await locations.ListByProjectAsync(projectId))
{
rows.Add(new EntityIndexRow(location.LocationID, location.LocationName, location.LocationName, "Name"));
var normalised = NormaliseLocationName(location.LocationName);
if (!string.Equals(normalised, location.LocationName, StringComparison.OrdinalIgnoreCase))
{
rows.Add(new EntityIndexRow(location.LocationID, location.LocationName, normalised, "Normalised name"));
}
foreach (var alias in await locations.ListAliasesAsync(location.LocationID))
{
rows.Add(new EntityIndexRow(location.LocationID, location.LocationName, alias.Alias, "Alias"));
var normalisedAlias = NormaliseLocationName(alias.Alias);
if (!string.Equals(normalisedAlias, alias.Alias, StringComparison.OrdinalIgnoreCase))
{
rows.Add(new EntityIndexRow(location.LocationID, location.LocationName, normalisedAlias, "Normalised alias"));
}
}
}
@ -292,13 +356,14 @@ public sealed class SceneImportResolver(
private StoryImportEntityResolution LogResolution(StoryImportEntityResolution resolution)
{
logger.LogInformation(
"Story Intelligence Import Preview resolved {EntityType}. Input={InputName} Status={Status} Resolved={ResolvedName} Confidence={ConfidencePercent} Candidates={Candidates}",
"Story Intelligence Import Preview resolved {EntityType}. Input={InputName} Status={Status} Resolved={ResolvedName} Confidence={ConfidencePercent} Candidates={Candidates} Notes={Notes}",
resolution.EntityType,
resolution.InputName,
resolution.Status,
resolution.MatchedName,
resolution.ConfidencePercent,
string.Join(", ", resolution.Candidates.Select(candidate => candidate.Name)));
string.Join(", ", resolution.Candidates.Select(candidate => candidate.Name)),
resolution.Notes);
return resolution;
}
@ -344,12 +409,78 @@ public sealed class SceneImportResolver(
=> string.Equals(location.LocationType, "generic", StringComparison.OrdinalIgnoreCase)
|| GenericLocations.Contains(Clean(location.Name).ToLowerInvariant());
private static bool IsGenericPerson(string name)
{
var clean = Clean(name).ToLowerInvariant();
return GenericPeople.Contains(clean)
|| clean.StartsWith("the ", StringComparison.OrdinalIgnoreCase) && GenericPeople.Contains(clean[4..])
|| clean.Contains(" group", StringComparison.OrdinalIgnoreCase)
|| clean.Contains("crowd", StringComparison.OrdinalIgnoreCase);
}
private static bool IsRejectedAsset(SceneIntelligenceAsset asset)
{
var name = Clean(asset.Name).ToLowerInvariant();
if (EnvironmentalAssets.Contains(name))
{
return true;
}
if (StoryAssetNames.Contains(RemoveLeadingArticle(name)))
{
return false;
}
var signalText = $"{asset.Status} {asset.OwnerOrHolder} {asset.Notes} {asset.AssetType}".ToLowerInvariant();
return !ContainsStoryAssetSignal(signalText);
}
private static bool ContainsStoryAssetSignal(string value)
=> value.Contains("sought", StringComparison.OrdinalIgnoreCase)
|| value.Contains("lost", StringComparison.OrdinalIgnoreCase)
|| value.Contains("hidden", StringComparison.OrdinalIgnoreCase)
|| value.Contains("stolen", StringComparison.OrdinalIgnoreCase)
|| value.Contains("exchanged", StringComparison.OrdinalIgnoreCase)
|| value.Contains("missing", StringComparison.OrdinalIgnoreCase)
|| value.Contains("important", StringComparison.OrdinalIgnoreCase)
|| value.Contains("document", StringComparison.OrdinalIgnoreCase)
|| value.Contains("evidence", StringComparison.OrdinalIgnoreCase);
private static string NormaliseLocationName(string? value)
{
var clean = RemoveLeadingArticle(Clean(value));
var lower = clean.ToLowerInvariant();
var suffixes = new[]
{
" sixth floor reception", " sixth-floor reception", " reception", " sixth floor", " office", " corridor", " stairwell"
};
foreach (var suffix in suffixes)
{
if (lower.EndsWith(suffix, StringComparison.OrdinalIgnoreCase) && clean.Length > suffix.Length)
{
return clean[..^suffix.Length].Trim();
}
}
return clean;
}
private static string Clean(string? value)
=> string.Join(' ', (value ?? string.Empty).Trim().Split((char[]?)null, StringSplitOptions.RemoveEmptyEntries));
private static string RemoveLeadingArticle(string value)
=> value.StartsWith("the ", StringComparison.OrdinalIgnoreCase) ? value[4..].Trim() : value;
private static bool NamesMatch(string? first, string? second)
=> string.Equals(Clean(first), Clean(second), StringComparison.OrdinalIgnoreCase);
private static string RelationshipKey(string characterA, string characterB, string? signal)
{
var names = new[] { Clean(characterA), Clean(characterB) }.Order(StringComparer.OrdinalIgnoreCase).ToArray();
return $"{names[0]}|{names[1]}|{Clean(signal).ToLowerInvariant()}";
}
private static HashSet<string> CreateSet(params string[] values)
=> new(values, StringComparer.OrdinalIgnoreCase);
@ -362,9 +493,11 @@ public sealed class SceneImportResolver(
public IReadOnlyList<StoryImportResolutionCandidate> Find(string value)
{
var clean = Clean(value);
var withoutArticle = RemoveLeadingArticle(clean);
var matches = rows
.Where(row => string.Equals(Clean(row.MatchValue), clean, StringComparison.Ordinal)
|| string.Equals(Clean(row.MatchValue), clean, StringComparison.OrdinalIgnoreCase))
|| string.Equals(Clean(row.MatchValue), clean, StringComparison.OrdinalIgnoreCase)
|| string.Equals(RemoveLeadingArticle(Clean(row.MatchValue)), withoutArticle, StringComparison.OrdinalIgnoreCase))
.GroupBy(row => row.EntityID)
.Select(group =>
{

33
PlotLine/Views/Admin/StoryIntelligenceRunDetails.cshtml
View File

@ -269,8 +269,8 @@ else
</ul>
}
<h3>Timeline</h3>
<p>@preview.Timeline.Count.ToString("N0") scene timeline entr@(preview.Timeline.Count == 1 ? "y" : "ies") would be created.</p>
<h3>Timeline Clues</h3>
<p>@preview.Timeline.Count.ToString("N0") timeline clue@(preview.Timeline.Count == 1 ? string.Empty : "s") detected.</p>
@if (preview.Timeline.Count > 0)
{
<ul>
@ -373,10 +373,12 @@ else
}
var builder = new System.Text.StringBuilder();
foreach (var group in items.GroupBy(item => item.Status))
foreach (var group in items
.GroupBy(item => GroupLabel(item.Status))
.OrderBy(group => GroupSort(group.Key)))
{
builder.Append("<h4>").Append(Encode(group.Key)).Append("</h4><ul>");
foreach (var item in group)
foreach (var item in group.OrderBy(item => item.MatchedName ?? item.InputName, StringComparer.OrdinalIgnoreCase))
{
builder.Append("<li>");
builder.Append(IconForStatus(item.Status)).Append(' ');
@ -413,10 +415,33 @@ else
{
StoryImportResolutionStatuses.Matched => "✓",
StoryImportResolutionStatuses.Ambiguous => "⚠",
StoryImportResolutionStatuses.PossibleDuplicate => "⚠",
StoryImportResolutionStatuses.NewCharacter or StoryImportResolutionStatuses.NewLocation or StoryImportResolutionStatuses.NewAsset => "+",
_ => "•"
};
private static string GroupLabel(string status)
=> status switch
{
StoryImportResolutionStatuses.NewCharacter or StoryImportResolutionStatuses.NewLocation or StoryImportResolutionStatuses.NewAsset => "New",
StoryImportResolutionStatuses.GenericReference => "Generic",
StoryImportResolutionStatuses.UnresolvedPersonReference => "Unresolved",
_ => status
};
private static int GroupSort(string group)
=> group switch
{
"Matched" => 0,
"New" => 1,
"Possible Duplicate" => 2,
"Ambiguous" => 3,
"Ignored" => 4,
"Generic" => 5,
"Unresolved" => 6,
_ => 99
};
private static string Encode(string? value)
=> System.Net.WebUtility.HtmlEncode(value ?? string.Empty);