Phase 20AM – Character Review Alias Handling and Candidate Filtering

This commit is contained in:
Nick Beckley 2026-07-09 19:54:35 +01:00
parent 4fee5846ab
commit f4897e5d5f
6 changed files with 317 additions and 9 deletions

View File

@ -10,7 +10,9 @@ var tests = new (string Name, Action Test)[]
("Truncated JSON is rejected", RejectsTruncatedJson),
("Repaired JSON deserialises into SceneIntelligenceScene", RepairedJsonDeserialises),
("Chapter Structure confidence 0. repairs and deserialises", ChapterStructureConfidenceRepairs),
("Shared parser reports raw chapter output on unrecoverable JSON", SharedParserReportsRawOutput)
("Shared parser reports raw chapter output on unrecoverable JSON", SharedParserReportsRawOutput),
("Character filtering rejects generic groups", CharacterFilteringRejectsGenericGroups),
("Character filtering preserves titled names", CharacterFilteringPreservesTitledNames)
};
foreach (var test in tests)
@ -106,6 +108,42 @@ static void SharedParserReportsRawOutput()
Assert(result.RawJson == raw, "Raw malformed output was not retained.");
}
static void CharacterFilteringRejectsGenericGroups()
{
Assert(IsGenericGroupReference("four examiners"), "Quantity + generic group should be rejected.");
Assert(IsGenericGroupReference("groups of men"), "Generic group phrase should be rejected.");
Assert(IsGenericGroupReference("neighbours"), "Generic plural group should be rejected.");
Assert(IsGenericGroupReference("passenger (blue Volkswagen Golf GTi)"), "Unnamed passenger descriptor should be rejected.");
Assert(IsGenericGroupReference("tubby lad"), "Unnamed descriptive person should be rejected.");
Assert(IsGenericGroupReference("lady examiner"), "Unnamed role descriptor should be rejected.");
Assert(IsGenericGroupReference("her mother"), "Determiner + family role should be rejected.");
Assert(IsGenericGroupReference("the child"), "Determiner + generic person should be rejected.");
}
static void CharacterFilteringPreservesTitledNames()
{
Assert(!IsGenericGroupReference("Reverend Potter"), "Reverend Potter should not be treated as generic.");
Assert(IsNamedOrTitledPerson("Reverend Potter"), "Reverend Potter should be recognised as a titled person.");
Assert(IsNamedOrTitledPerson("Detective Sergeant Summerhill"), "Detective Sergeant Summerhill should be recognised as a titled person.");
Assert(IsNamedOrTitledPerson("Mrs Patterson"), "Mrs Patterson should be recognised as a titled person.");
Assert(IsNamedOrTitledPerson("Aunt Elen"), "Aunt Elen should be recognised as a titled person.");
}
static bool IsGenericGroupReference(string name)
=> InvokePrivateCharacterFilter("IsGenericGroupReference", name);
static bool IsNamedOrTitledPerson(string name)
=> InvokePrivateCharacterFilter("IsNamedOrTitledPerson", name);
static bool InvokePrivateCharacterFilter(string methodName, string name)
{
var method = typeof(StoryIntelligenceCharacterImportService).GetMethod(
methodName,
System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Static);
Assert(method is not null, $"{methodName} was not found.");
return method!.Invoke(null, [name]) is true;
}
static JsonSerializerOptions JsonOptions()
=> new()
{

View File

@ -370,6 +370,7 @@ public sealed class OnboardingStoryIntelligenceService(
BookID = batch.BookID,
CharactersCreated = batch.LastCharacterImportResult?.CharactersCreated ?? 0,
CharactersLinked = batch.LastCharacterImportResult?.CharactersLinked ?? 0,
CharactersAliased = batch.LastCharacterImportResult?.CharactersAliased ?? 0,
CharactersIgnored = batch.LastCharacterImportResult?.CharactersIgnored ?? 0,
PovLinksResolved = batch.LastCharacterImportResult?.PovLinksResolved ?? 0,
ScenePeoplePanelsUpdated = batch.LastCharacterImportResult?.ScenePeoplePanelsUpdated ?? 0
@ -711,6 +712,7 @@ public sealed class StoryIntelligenceCharacterImportBatchResult
{
public int CharactersCreated { get; init; }
public int CharactersLinked { get; init; }
public int CharactersAliased { get; init; }
public int CharactersIgnored { get; init; }
public int PovLinksResolved { get; init; }
public int ScenePeoplePanelsUpdated { get; init; }

View File

@ -1,4 +1,5 @@
using System.Text.Json;
using System.Text.RegularExpressions;
using PlotLine.Data;
using PlotLine.Models;
using PlotLine.ViewModels;
@ -31,11 +32,38 @@ public sealed class StoryIntelligenceCharacterImportService(
private static readonly HashSet<string> GenericPeople = CreateSet(
"man", "woman", "boy", "girl", "child", "children", "person", "people", "doctor", "nurse",
"police officer", "driver", "receptionist", "attendant", "guard", "security guard", "waiter", "waitress",
"clerk", "shopkeeper", "cashier", "crowd", "family", "group", "small group", "police");
"clerk", "shopkeeper", "cashier", "crowd", "family", "group", "small group", "police", "officer",
"officers", "staff", "customers", "boys", "girls", "men", "women", "neighbour", "neighbours",
"passenger", "passengers", "examiner", "examiners", "instructor", "instructors", "candidate", "candidates",
"social worker", "social workers", "responders");
private static readonly HashSet<string> GenericGroupHeads = CreateSet(
"man", "men", "woman", "women", "boy", "boys", "girl", "girls", "child", "children", "person", "people",
"group", "crowd", "family", "neighbour", "neighbours", "passenger", "passengers", "examiner", "examiners",
"driver", "drivers", "instructor", "instructors", "candidate", "candidates", "officer", "officers", "staff", "customers",
"responders", "worker", "workers", "lad", "lads");
private static readonly HashSet<string> QuantityWords = CreateSet(
"one", "two", "three", "four", "five", "six", "seven", "eight", "nine", "ten", "several", "some",
"many", "few", "couple", "various", "other", "groups");
private static readonly HashSet<string> DeterminerWords = CreateSet(
"the", "a", "an", "my", "your", "his", "her", "our", "their");
private static readonly HashSet<string> FamilyRoleNames = CreateSet(
"aunt", "uncle", "mother", "father", "mum", "mom", "mam", "dad", "brother", "sister", "grandmother",
"grandfather", "grandma", "grandad", "granddad");
private static readonly HashSet<string> CharacterTitlePrefixes = CreateSet(
"mr", "mrs", "miss", "ms", "dr", "reverend", "rev", "detective", "sergeant", "ds", "aunt", "uncle");
private static readonly HashSet<string> NonCharacterNames = CreateSet(
"christmas", "house", "street", "door", "window", "car", "church", "building", "lobby", "room",
"hallway", "kitchen", "garage", "pavement", "road", "desk", "table", "phone", "letter", "note");
"hallway", "kitchen", "garage", "pavement", "road", "desk", "table", "phone", "letter", "note",
"social services", "under-flyover / link road");
private static readonly Regex ParentheticalSuffix = new(@"\s*\([^)]*\)\s*$", RegexOptions.Compiled);
private static readonly Regex LeadingQuantity = new(@"^\s*(\d+|one|two|three|four|five|six|seven|eight|nine|ten|several|some|many|few|couple|various|other)\s+", RegexOptions.IgnoreCase | RegexOptions.Compiled);
public async Task<StoryIntelligenceCharacterReviewViewModel> BuildReviewAsync(OnboardingStoryIntelligenceBatch batch)
{
@ -86,9 +114,12 @@ public sealed class StoryIntelligenceCharacterImportService(
var created = 0;
var linkedExisting = 0;
var ignored = 0;
var aliased = 0;
var linkedAppearances = 0;
var povLinks = 0;
var resolvedCharacters = new Dictionary<string, int>(StringComparer.OrdinalIgnoreCase);
var resolvedCandidateIds = new Dictionary<string, int>(StringComparer.OrdinalIgnoreCase);
var candidateByKey = data.Candidates.ToDictionary(candidate => candidate.Key, StringComparer.OrdinalIgnoreCase);
foreach (var candidate in data.Candidates)
{
@ -104,6 +135,11 @@ public sealed class StoryIntelligenceCharacterImportService(
continue;
}
if (string.Equals(choice.Action, StoryIntelligenceCharacterImportActions.Alias, StringComparison.OrdinalIgnoreCase))
{
continue;
}
var requestedName = Clean(choice.ImportName);
var importName = string.IsNullOrWhiteSpace(requestedName) ? candidate.DisplayName : requestedName;
var forceCreateSeparate = string.Equals(choice.Action, StoryIntelligenceCharacterImportActions.CreateSeparate, StringComparison.OrdinalIgnoreCase);
@ -151,9 +187,60 @@ public sealed class StoryIntelligenceCharacterImportService(
linkedAppearances++;
}
resolvedCandidateIds[candidate.Key] = characterId.Value;
AddDecision(batch, candidate.Key, choice.Action, importName, characterId.Value, true);
}
foreach (var candidate in data.Candidates)
{
if (!choices.TryGetValue(candidate.Key, out var choice)
|| !string.Equals(choice.Action, StoryIntelligenceCharacterImportActions.Alias, StringComparison.OrdinalIgnoreCase))
{
continue;
}
var targetKey = Clean(choice.AliasTargetKey);
if (string.IsNullOrWhiteSpace(targetKey)
|| string.Equals(targetKey, candidate.Key, StringComparison.OrdinalIgnoreCase)
|| !choices.TryGetValue(targetKey, out var targetChoice)
|| string.Equals(targetChoice.Action, StoryIntelligenceCharacterImportActions.Ignore, StringComparison.OrdinalIgnoreCase)
|| string.Equals(targetChoice.Action, StoryIntelligenceCharacterImportActions.Alias, StringComparison.OrdinalIgnoreCase)
|| !candidateByKey.TryGetValue(targetKey, out var targetCandidate)
|| !resolvedCandidateIds.TryGetValue(targetKey, out var targetCharacterId))
{
logger.LogWarning(
"Story Intelligence character alias candidate {CandidateKey} could not be imported because target {TargetKey} was not a resolved create/link target.",
candidate.Key,
targetKey);
continue;
}
var targetName = targetCandidate.ExistingCharacterName ?? targetChoice.ImportName ?? targetCandidate.DisplayName;
await TryAddAliasAsync(targetCharacterId, candidate.DisplayName, targetName);
AddResolvedName(resolvedCharacters, candidate.DisplayName, targetCharacterId);
foreach (var alias in candidate.Aliases)
{
await TryAddAliasAsync(targetCharacterId, alias, targetName);
AddResolvedName(resolvedCharacters, alias, targetCharacterId);
}
foreach (var appearance in candidate.PresentAppearances.Concat(candidate.MentionedOnlyAppearances).Where(appearance => !appearance.AlreadyLinked))
{
await characters.SaveSceneCharacterAsync(new SceneCharacter
{
SceneID = appearance.SceneID,
CharacterID = targetCharacterId,
RoleInSceneTypeID = MatchRole(roleTypes, appearance.RoleInScene),
PresenceTypeID = MatchPresence(presenceTypes, appearance.MentionedOnly),
AppearanceNotes = BuildAliasAppearanceNote(candidate.DisplayName, targetCandidate.DisplayName, appearance.Notes)
});
linkedAppearances++;
}
aliased++;
AddDecision(batch, candidate.Key, StoryIntelligenceCharacterImportActions.Alias, candidate.DisplayName, targetCharacterId, true);
}
foreach (var scene in data.SceneAnalyses)
{
var povName = Clean(scene.Parsed.PointOfView?.CharacterName);
@ -172,16 +259,18 @@ public sealed class StoryIntelligenceCharacterImportService(
CharactersCreated = created,
CharactersLinked = linkedExisting,
CharactersIgnored = ignored,
CharactersAliased = aliased,
PovLinksResolved = povLinks,
ScenePeoplePanelsUpdated = linkedAppearances
};
await pipelineState.RecordCharacterImportAsync(batch.ProjectID, batch.BookID);
logger.LogInformation(
"Imported Story Intelligence characters for batch {BatchID}. Created={Created} LinkedExisting={LinkedExisting} Ignored={Ignored} LinkedAppearances={LinkedAppearances} PovLinks={PovLinks}",
"Imported Story Intelligence characters for batch {BatchID}. Created={Created} LinkedExisting={LinkedExisting} Aliased={Aliased} Ignored={Ignored} LinkedAppearances={LinkedAppearances} PovLinks={PovLinks}",
batch.BatchID,
created,
linkedExisting,
aliased,
ignored,
linkedAppearances,
povLinks);
@ -190,7 +279,7 @@ public sealed class StoryIntelligenceCharacterImportService(
{
Success = true,
ScenesCreated = linkedAppearances,
Message = $"Characters imported. {created:N0} created, {linkedExisting:N0} linked, {ignored:N0} ignored, {linkedAppearances:N0} scene appearance(s) updated, {povLinks:N0} POV link(s) resolved."
Message = $"Characters imported. {created:N0} created, {linkedExisting:N0} linked, {aliased:N0} alias decision(s) applied, {ignored:N0} ignored, {linkedAppearances:N0} scene appearance(s) updated, {povLinks:N0} POV link(s) resolved."
};
}
@ -258,6 +347,7 @@ public sealed class StoryIntelligenceCharacterImportService(
var candidates = groups.Values
.Where(candidate => candidate.PresentAppearances.Count > 0 || candidate.MentionedOnlyAppearances.Count > 0)
.Where(candidate => candidate.PresentAppearances.Concat(candidate.MentionedOnlyAppearances).Any(appearance => !appearance.AlreadyLinked))
.Where(IsVisibleCharacterCandidate)
.OrderByDescending(candidate => candidate.PresentAppearances.Select(appearance => appearance.SceneID).Distinct().Count())
.ThenByDescending(candidate => candidate.MentionedOnlyAppearances.Select(appearance => appearance.SceneID).Distinct().Count())
.ThenBy(candidate => candidate.DisplayName)
@ -515,6 +605,12 @@ public sealed class StoryIntelligenceCharacterImportService(
}
}
private static string? BuildAliasAppearanceNote(string aliasName, string targetName, string? notes)
{
var prefix = $"Imported as alias '{aliasName}' of {targetName}.";
return string.IsNullOrWhiteSpace(notes) ? prefix : $"{prefix} {notes}";
}
private static string? BuildFirstAppearance(Scene scene, SceneIntelligenceScene parsed, SceneIntelligenceCharacter character)
{
var note = Clean(character.Notes);
@ -551,6 +647,30 @@ public sealed class StoryIntelligenceCharacterImportService(
private static bool IsCharacterEntity(string? entityType)
=> string.Equals(Clean(entityType), "Character", StringComparison.OrdinalIgnoreCase);
private static bool IsVisibleCharacterCandidate(CharacterCandidate candidate)
{
var name = CleanCharacterName(candidate.DisplayName);
if (!IsCharacterNameCandidate(name))
{
return false;
}
if (IsNamedOrTitledPerson(name))
{
return true;
}
if (IsFamilyRoleOnly(name))
{
return candidate.PresentAppearances.Concat(candidate.MentionedOnlyAppearances)
.Select(appearance => appearance.SceneID)
.Distinct()
.Count() >= 2;
}
return !IsGenericGroupReference(name);
}
private static bool IsCharacterNameCandidate(string? name)
{
var clean = CleanCharacterName(name);
@ -570,6 +690,80 @@ public sealed class StoryIntelligenceCharacterImportService(
return true;
}
private static bool IsGenericGroupReference(string? name)
{
var clean = CleanCharacterName(name);
if (string.IsNullOrWhiteSpace(clean))
{
return true;
}
var withoutParentheses = Clean(ParentheticalSuffix.Replace(clean, string.Empty));
var normalised = Normalise(withoutParentheses);
if (GenericPeople.Contains(normalised) || NonCharacterNames.Contains(normalised))
{
return true;
}
var words = WordsForFiltering(withoutParentheses);
if (words.Count == 0)
{
return true;
}
if (LeadingQuantity.IsMatch(withoutParentheses)
&& words.Any(word => GenericGroupHeads.Contains(word)))
{
return true;
}
if (words.Count > 1
&& words.Any(word => GenericGroupHeads.Contains(word))
&& !HasSpecificPossessiveFamilyReference(words))
{
return true;
}
if (words.Count > 1
&& DeterminerWords.Contains(words[0])
&& words.Skip(1).Any(word => FamilyRoleNames.Contains(word) || GenericGroupHeads.Contains(word)))
{
return true;
}
if (words.Count > 1
&& QuantityWords.Contains(words[0])
&& words.Skip(1).Any(word => GenericGroupHeads.Contains(word)))
{
return true;
}
return words.All(word => GenericGroupHeads.Contains(word) || QuantityWords.Contains(word));
}
private static bool IsNamedOrTitledPerson(string name)
{
var words = WordsForFiltering(name);
if (words.Count == 0)
{
return false;
}
return CharacterTitlePrefixes.Contains(words[0]) && words.Count > 1;
}
private static bool IsFamilyRoleOnly(string name)
{
var words = WordsForFiltering(name);
return words.Count == 1 && FamilyRoleNames.Contains(words[0]);
}
private static bool HasSpecificPossessiveFamilyReference(IReadOnlyList<string> words)
=> words.Count == 2
&& FamilyRoleNames.Contains(words[1])
&& words[0].EndsWith("s", StringComparison.OrdinalIgnoreCase)
&& !DeterminerWords.Contains(words[0]);
private static string ResolveCandidateKey(
IReadOnlyDictionary<string, CharacterCandidate> groups,
CharacterIndex existingIndex,
@ -636,6 +830,15 @@ public sealed class StoryIntelligenceCharacterImportService(
return words;
}
private static List<string> WordsForFiltering(string? name)
=> CleanCharacterName(name)
.Replace("/", " ", StringComparison.Ordinal)
.Replace("-", " ", StringComparison.Ordinal)
.Split(' ', StringSplitOptions.RemoveEmptyEntries)
.Select(word => word.Trim('.', ',', ';', ':', '!', '?', '"', '\'', '(', ')').ToLowerInvariant())
.Where(word => !string.IsNullOrWhiteSpace(word))
.ToList();
private static bool IsMergeableTitle(string value)
=> value.Equals("instructor", StringComparison.OrdinalIgnoreCase)
|| value.Equals("detective", StringComparison.OrdinalIgnoreCase)

View File

@ -204,6 +204,7 @@ public sealed class StoryIntelligenceCharacterImportChoiceForm
public string Key { get; set; } = string.Empty;
public string Action { get; set; } = StoryIntelligenceCharacterImportActions.Approve;
public string? ImportName { get; set; }
public string? AliasTargetKey { get; set; }
}
public static class StoryIntelligenceCharacterImportActions
@ -211,6 +212,7 @@ public static class StoryIntelligenceCharacterImportActions
public const string Approve = "Approve";
public const string Ignore = "Ignore";
public const string CreateSeparate = "CreateSeparate";
public const string Alias = "Alias";
}
public sealed class StoryIntelligenceCompletionViewModel
@ -235,6 +237,7 @@ public sealed class StoryIntelligenceCharacterImportResultViewModel
public int BookID { get; init; }
public int CharactersCreated { get; init; }
public int CharactersLinked { get; init; }
public int CharactersAliased { get; init; }
public int CharactersIgnored { get; init; }
public int PovLinksResolved { get; init; }
public int ScenePeoplePanelsUpdated { get; init; }

View File

@ -24,6 +24,10 @@
<span>Characters linked</span>
<strong>@Model.CharactersLinked.ToString("N0")</strong>
</div>
<div>
<span>Aliases added</span>
<strong>@Model.CharactersAliased.ToString("N0")</strong>
</div>
<div>
<span>Characters ignored</span>
<strong>@Model.CharactersIgnored.ToString("N0")</strong>

View File

@ -69,7 +69,7 @@
<input type="hidden" name="Characters[@i].Key" value="@candidate.Key" />
<div class="story-character-card__body">
<div class="story-character-card__body" data-character-key="@candidate.Key" data-character-name="@candidate.CharacterName">
<dl>
<div><dt>First appearance</dt><dd>@Display(candidate.ExampleFirstAppearance)</dd></div>
<div><dt>Possible aliases</dt><dd>@(candidate.PossibleAliases.Count == 0 ? "None detected" : string.Join(", ", candidate.PossibleAliases))</dd></div>
@ -101,10 +101,27 @@
<input type="radio" name="Characters[@i].Action" value="@StoryIntelligenceCharacterImportActions.Ignore" data-character-action />
Ignore
</label>
<label>
<input type="radio" name="Characters[@i].Action" value="@StoryIntelligenceCharacterImportActions.Alias" data-character-action />
Alias of another character
</label>
</fieldset>
<label class="form-label" for="character-import-name-@i">Import name</label>
<input id="character-import-name-@i" class="form-control" name="Characters[@i].ImportName" value="@candidate.ImportName" />
<div data-character-import-name-panel>
<label class="form-label" for="character-import-name-@i">Import name</label>
<input id="character-import-name-@i" class="form-control" name="Characters[@i].ImportName" value="@candidate.ImportName" />
</div>
<div data-character-alias-panel hidden>
<label class="form-label" for="character-alias-target-@i">Alias target</label>
<select id="character-alias-target-@i" class="form-select" name="Characters[@i].AliasTargetKey" data-character-alias-target>
<option value="">Choose character...</option>
@foreach (var target in Model.CharacterReview.Candidates.Where(target => !string.Equals(target.Key, candidate.Key, StringComparison.OrdinalIgnoreCase)))
{
<option value="@target.Key">@target.CharacterName@(target.IsExistingMatch ? $" -> {target.ExistingCharacterName}" : string.Empty)</option>
}
</select>
</div>
</div>
</details>
}
@ -129,7 +146,44 @@
const cards = () => Array.from(form.querySelectorAll("[data-character-card]"));
const setAction = (card, action) => {
const input = card.querySelector(`[data-character-action][value="${action}"]`);
if (input) input.checked = true;
if (input) {
input.checked = true;
updateCard(card);
}
};
const actionFor = (card) => card.querySelector("[data-character-action]:checked")?.value || "@StoryIntelligenceCharacterImportActions.Approve";
const isCanonicalTarget = (card) => {
const action = actionFor(card);
return action !== "@StoryIntelligenceCharacterImportActions.Ignore" && action !== "@StoryIntelligenceCharacterImportActions.Alias";
};
const updateAliasTargets = () => {
const allCards = cards();
for (const card of allCards) {
const body = card.querySelector("[data-character-key]");
const ownKey = body?.getAttribute("data-character-key") || "";
const select = card.querySelector("[data-character-alias-target]");
if (!select) continue;
const previous = select.value;
for (const option of Array.from(select.options)) {
if (!option.value) continue;
const targetCard = allCards.find(item => item.querySelector("[data-character-key]")?.getAttribute("data-character-key") === option.value);
option.disabled = option.value === ownKey || !targetCard || !isCanonicalTarget(targetCard);
}
if (select.selectedOptions[0]?.disabled) {
select.value = "";
} else {
select.value = previous;
}
}
};
const updateCard = (card) => {
const action = actionFor(card);
const namePanel = card.querySelector("[data-character-import-name-panel]");
const aliasPanel = card.querySelector("[data-character-alias-panel]");
if (namePanel) namePanel.hidden = action === "@StoryIntelligenceCharacterImportActions.Ignore" || action === "@StoryIntelligenceCharacterImportActions.Alias";
if (aliasPanel) aliasPanel.hidden = action !== "@StoryIntelligenceCharacterImportActions.Alias";
updateAliasTargets();
};
document.querySelectorAll("[data-character-bulk]").forEach(button => {
button.addEventListener("click", () => {
@ -142,6 +196,10 @@
if (action === "collapse") cards().forEach(card => card.open = false);
});
});
form.querySelectorAll("[data-character-action]").forEach(input => {
input.addEventListener("change", () => updateCard(input.closest("[data-character-card]")));
});
cards().forEach(updateCard);
form.addEventListener("submit", () => {
cards()
.filter(card => !card.querySelector("[data-character-selected]")?.checked)