using System.Text.Json; using PlotLine.Data; using PlotLine.Models; using PlotLine.ViewModels; namespace PlotLine.Services; public interface IStoryIntelligenceCharacterImportService { Task BuildReviewAsync(OnboardingStoryIntelligenceBatch batch); Task ImportAsync(OnboardingStoryIntelligenceBatch batch, StoryIntelligenceCharacterImportForm form); } public sealed class StoryIntelligenceCharacterImportService( IStoryIntelligenceResultRepository runs, ICharacterRepository characters, ISceneRepository scenes, IStoryIntelligencePipelineStateService pipelineState, ILogger logger) : IStoryIntelligenceCharacterImportService { private static readonly JsonSerializerOptions JsonOptions = new() { PropertyNamingPolicy = JsonNamingPolicy.CamelCase, PropertyNameCaseInsensitive = true }; private static readonly HashSet IgnoredNames = CreateSet("narrator", "unknown"); private static readonly HashSet GenericPeople = CreateSet( "man", "woman", "boy", "girl", "child", "children", "person", "people", "doctor", "nurse", "police officer", "driver", "receptionist", "attendant", "guard", "security guard", "waiter", "waitress", "clerk", "shopkeeper", "cashier", "crowd", "family", "group", "small group"); public async Task BuildReviewAsync(OnboardingStoryIntelligenceBatch batch) { var data = await BuildCandidateDataAsync(batch); var decidedKeys = batch.CharacterDecisions.Select(decision => decision.Key).ToHashSet(StringComparer.OrdinalIgnoreCase); var visibleCandidates = data.Candidates.Where(candidate => !decidedKeys.Contains(candidate.Key)).ToList(); return new StoryIntelligenceCharacterReviewViewModel { HasCommittedScenes = data.HasCommittedScenes, CanImport = visibleCandidates.Count > 0, AlreadyLinkedCount = data.AlreadyLinkedCount, IsComplete = data.HasCommittedScenes && visibleCandidates.Count == 0, Candidates = visibleCandidates.Select(candidate => new StoryIntelligenceCharacterReviewCandidateViewModel { Key = candidate.Key, CharacterName = candidate.DisplayName, ImportName = candidate.DisplayName, AppearsInScenes = candidate.PresentAppearances.Concat(candidate.MentionedOnlyAppearances).Select(appearance => appearance.SceneID).Distinct().Count(), Confidence = ConfidenceBand(candidate.Confidence), PossibleAliases = candidate.Aliases.OrderBy(value => value).ToList(), ExampleFirstAppearance = candidate.FirstAppearanceNote, ExistingCharacterID = candidate.ExistingCharacterID, ExistingCharacterName = candidate.ExistingCharacterName }).ToList() }; } public async Task ImportAsync(OnboardingStoryIntelligenceBatch batch, StoryIntelligenceCharacterImportForm form) { var data = await BuildCandidateDataAsync(batch); if (!data.HasCommittedScenes) { return new StoryIntelligenceImportCommitResult { Success = false, Message = "Create scenes before importing characters." }; } var choices = form.Characters .Where(choice => !string.IsNullOrWhiteSpace(choice.Key)) .ToDictionary(choice => choice.Key, choice => choice, StringComparer.OrdinalIgnoreCase); if (choices.Count == 0) { return new StoryIntelligenceImportCommitResult { Success = false, Message = "Choose at least one character to create, link or ignore." }; } var existingIndex = await BuildCharacterIndexAsync(batch.ProjectID); var lookupData = await characters.GetLookupsAsync(batch.ProjectID); var roleTypes = lookupData.RoleTypes; var presenceTypes = lookupData.PresenceTypes; var created = 0; var linkedExisting = 0; var ignored = 0; var linkedAppearances = 0; var povLinks = 0; var resolvedCharacters = new Dictionary(StringComparer.OrdinalIgnoreCase); foreach (var candidate in data.Candidates) { if (!choices.TryGetValue(candidate.Key, out var choice)) { continue; } if (string.Equals(choice.Action, StoryIntelligenceCharacterImportActions.Ignore, StringComparison.OrdinalIgnoreCase)) { ignored++; AddDecision(batch, candidate.Key, StoryIntelligenceCharacterImportActions.Ignore, candidate.DisplayName, null, false); continue; } var requestedName = Clean(choice.ImportName); var importName = string.IsNullOrWhiteSpace(requestedName) ? candidate.DisplayName : requestedName; var forceCreateSeparate = string.Equals(choice.Action, StoryIntelligenceCharacterImportActions.CreateSeparate, StringComparison.OrdinalIgnoreCase); var matchedExistingId = forceCreateSeparate ? null : candidate.ExistingCharacterID ?? existingIndex.Find(importName)?.CharacterID ?? existingIndex.Find(candidate.DisplayName)?.CharacterID; var characterId = matchedExistingId; if (!characterId.HasValue) { characterId = await characters.SaveCharacterAsync(new Character { ProjectID = batch.ProjectID, CharacterName = importName, ShortName = FirstName(importName), DefaultDescription = candidate.Description }); created++; } else { linkedExisting++; } AddResolvedName(resolvedCharacters, candidate.DisplayName, characterId.Value); AddResolvedName(resolvedCharacters, importName, characterId.Value); foreach (var alias in candidate.Aliases) { AddResolvedName(resolvedCharacters, alias, characterId.Value); await TryAddAliasAsync(characterId.Value, alias, importName); } foreach (var appearance in candidate.PresentAppearances) { await characters.SaveSceneCharacterAsync(new SceneCharacter { SceneID = appearance.SceneID, CharacterID = characterId.Value, RoleInSceneTypeID = MatchRole(roleTypes, appearance.RoleInScene), PresenceTypeID = MatchPresence(presenceTypes, appearance.MentionedOnly), AppearanceNotes = appearance.Notes }); linkedAppearances++; } AddDecision(batch, candidate.Key, choice.Action, importName, characterId.Value, true); } foreach (var scene in data.SceneAnalyses) { var povName = Clean(scene.Parsed.PointOfView?.CharacterName); if (string.IsNullOrWhiteSpace(povName) || !resolvedCharacters.TryGetValue(povName, out var characterId)) { continue; } await scenes.UpdatePovCharacterAsync(scene.SceneID, characterId); povLinks++; } batch.CharacterStageComplete = true; batch.LastCharacterImportResult = new StoryIntelligenceCharacterImportBatchResult { CharactersCreated = created, CharactersLinked = linkedExisting, CharactersIgnored = ignored, PovLinksResolved = povLinks, ScenePeoplePanelsUpdated = linkedAppearances }; await pipelineState.RecordCharacterImportAsync(batch.ProjectID, batch.BookID); logger.LogInformation( "Imported Story Intelligence characters for batch {BatchID}. Created={Created} LinkedExisting={LinkedExisting} Ignored={Ignored} LinkedAppearances={LinkedAppearances} PovLinks={PovLinks}", batch.BatchID, created, linkedExisting, ignored, linkedAppearances, povLinks); return new StoryIntelligenceImportCommitResult { Success = true, ScenesCreated = linkedAppearances, Message = $"Characters imported. {created:N0} created, {linkedExisting:N0} linked, {ignored:N0} ignored, {linkedAppearances:N0} scene appearance(s) updated, {povLinks:N0} POV link(s) resolved." }; } private static void AddDecision(OnboardingStoryIntelligenceBatch batch, string key, string action, string? characterName, int? characterId, bool createdOrLinked) { batch.CharacterDecisions.RemoveAll(decision => string.Equals(decision.Key, key, StringComparison.OrdinalIgnoreCase)); batch.CharacterDecisions.Add(new OnboardingStoryIntelligenceCharacterDecision { Key = key, Action = action, CharacterName = characterName, CharacterID = characterId, CreatedOrLinked = createdOrLinked }); } private async Task BuildCandidateDataAsync(OnboardingStoryIntelligenceBatch batch) { var existingIndex = await BuildCharacterIndexAsync(batch.ProjectID); var groups = new Dictionary(StringComparer.OrdinalIgnoreCase); var sceneAnalyses = new List(); var alreadyLinked = 0; var hasCommittedScenes = false; foreach (var item in batch.Items) { var commit = await runs.GetImportCommitAsync(item.RunID); if (commit is null || !string.Equals(commit.Status, StoryIntelligenceImportCommitStatuses.Completed, StringComparison.OrdinalIgnoreCase)) { continue; } hasCommittedScenes = true; var importedScenes = (await scenes.ListByChapterAsync(item.ChapterID)) .Where(scene => scene.ImportRunID == item.RunID) .ToList(); var importedByRange = importedScenes .Where(scene => scene.SourceStartParagraph.HasValue && scene.SourceEndParagraph.HasValue) .ToDictionary(scene => $"{scene.SourceStartParagraph}-{scene.SourceEndParagraph}", scene => scene, StringComparer.OrdinalIgnoreCase); var importedByNumber = importedScenes.ToDictionary(scene => Convert.ToInt32(scene.SceneNumber), scene => scene); var sceneResults = await runs.ListSceneResultsAsync(item.RunID); foreach (var sceneResult in sceneResults) { var parsed = TryReadScene(sceneResult); if (parsed is null) { continue; } var importedScene = ResolveImportedScene(sceneResult, importedByRange, importedByNumber); if (importedScene is null) { continue; } sceneAnalyses.Add(new ImportedSceneAnalysis(importedScene.SceneID, parsed)); var existingAppearances = await characters.ListSceneCharactersAsync(importedScene.SceneID); alreadyLinked += existingAppearances.Count; foreach (var character in parsed.Characters ?? []) { AddMention(groups, existingIndex, importedScene, parsed, character, existingAppearances); } } } var candidates = groups.Values .Where(candidate => candidate.PresentAppearances.Count > 0 || candidate.MentionedOnlyAppearances.Count > 0) .Where(candidate => candidate.PresentAppearances.Any(appearance => !appearance.AlreadyLinked)) .OrderByDescending(candidate => candidate.PresentAppearances.Select(appearance => appearance.SceneID).Distinct().Count()) .ThenBy(candidate => candidate.DisplayName) .ToList(); return new CharacterCandidateData(hasCommittedScenes, alreadyLinked, candidates, sceneAnalyses); } private static void AddMention( Dictionary groups, CharacterIndex existingIndex, Scene importedScene, SceneIntelligenceScene parsed, SceneIntelligenceCharacter character, IReadOnlyList existingAppearances) { var name = Clean(character.Name); if (string.IsNullOrWhiteSpace(name) || IgnoredNames.Contains(name) || IsGenericPerson(name)) { return; } var key = Normalise(name); if (!groups.TryGetValue(key, out var candidate)) { var match = existingIndex.Find(name); candidate = new CharacterCandidate(key, name, match?.CharacterID, match?.CharacterName); groups[key] = candidate; } foreach (var alias in character.Aliases ?? []) { var cleanAlias = Clean(alias); if (!string.IsNullOrWhiteSpace(cleanAlias) && !string.Equals(cleanAlias, candidate.DisplayName, StringComparison.OrdinalIgnoreCase)) { candidate.Aliases.Add(cleanAlias); } } var notes = Clean(character.Notes); if (string.IsNullOrWhiteSpace(notes) && character.Actions?.Count > 0) { notes = Clean(string.Join("; ", character.Actions)); } var appearance = new CharacterAppearanceImport( importedScene.SceneID, importedScene.SceneNumber, importedScene.SceneTitle, character.RoleInScene, character.MentionedOnly == true, notes, character.Confidence, candidate.ExistingCharacterID.HasValue && existingAppearances.Any(appearance => appearance.CharacterID == candidate.ExistingCharacterID.Value)); if (character.MentionedOnly == true) { candidate.MentionedOnlyAppearances.Add(appearance); } else { candidate.PresentAppearances.Add(appearance); } candidate.Confidence = Max(candidate.Confidence, character.Confidence); candidate.Description ??= notes; candidate.FirstAppearanceNote ??= BuildFirstAppearance(importedScene, parsed, character); } private async Task BuildCharacterIndexAsync(int projectId) { var index = new CharacterIndex(); foreach (var character in await characters.ListCharactersAsync(projectId)) { index.Add(character.CharacterName, character.CharacterID, character.CharacterName); index.Add(character.ShortName, character.CharacterID, character.CharacterName); foreach (var alias in await characters.ListAliasesAsync(character.CharacterID)) { index.Add(alias.Alias, character.CharacterID, character.CharacterName); } } return index; } private async Task TryAddAliasAsync(int characterId, string alias, string primaryName) { var cleanAlias = Clean(alias); if (string.IsNullOrWhiteSpace(cleanAlias) || string.Equals(cleanAlias, primaryName, StringComparison.OrdinalIgnoreCase)) { return; } var existingAliases = await characters.ListAliasesAsync(characterId); if (existingAliases.Any(item => string.Equals(Clean(item.Alias), cleanAlias, StringComparison.OrdinalIgnoreCase))) { return; } await characters.AddAliasAsync(characterId, cleanAlias, null); } private static Scene? ResolveImportedScene( StoryIntelligenceSavedSceneResult sceneResult, IReadOnlyDictionary byRange, IReadOnlyDictionary byNumber) { if (sceneResult.StartParagraph.HasValue && sceneResult.EndParagraph.HasValue && byRange.TryGetValue($"{sceneResult.StartParagraph}-{sceneResult.EndParagraph}", out var rangeMatch)) { return rangeMatch; } return byNumber.TryGetValue(sceneResult.TemporarySceneNumber, out var numberMatch) ? numberMatch : null; } private static SceneIntelligenceScene? TryReadScene(StoryIntelligenceSavedSceneResult result) { if (string.IsNullOrWhiteSpace(result.ParsedJson)) { return null; } try { var direct = JsonSerializer.Deserialize(result.ParsedJson, JsonOptions); if (!string.IsNullOrWhiteSpace(direct?.SchemaVersion)) { return direct; } using var document = JsonDocument.Parse(result.ParsedJson); if (document.RootElement.TryGetProperty("parsedScene", out var parsedScene)) { return parsedScene.Deserialize(JsonOptions); } } catch (JsonException) { } return null; } private static int? MatchRole(IReadOnlyList roleTypes, string? role) { var value = Clean(role); if (string.IsNullOrWhiteSpace(value)) { return null; } return roleTypes.FirstOrDefault(type => value.Contains(type.TypeName, StringComparison.OrdinalIgnoreCase) || type.TypeName.Contains(value, StringComparison.OrdinalIgnoreCase))?.CharacterRoleInSceneTypeID; } private static int? MatchPresence(IReadOnlyList presenceTypes, bool mentionedOnly) { var preferred = mentionedOnly ? "Mentioned" : "Present"; return presenceTypes.FirstOrDefault(type => type.TypeName.Contains(preferred, StringComparison.OrdinalIgnoreCase))?.PresenceTypeID; } private static void AddResolvedName(Dictionary map, string name, int characterId) { var clean = Clean(name); if (!string.IsNullOrWhiteSpace(clean)) { map[clean] = characterId; } } private static string? BuildFirstAppearance(Scene scene, SceneIntelligenceScene parsed, SceneIntelligenceCharacter character) { var note = Clean(character.Notes); if (!string.IsNullOrWhiteSpace(note)) { return $"Scene {scene.SceneNumber:g}: {note}"; } var summary = Clean(parsed.Summary?.Short); return string.IsNullOrWhiteSpace(summary) ? $"Scene {scene.SceneNumber:g}: {scene.SceneTitle}" : $"Scene {scene.SceneNumber:g}: {summary}"; } private static decimal? Max(decimal? current, decimal? next) => current.HasValue && next.HasValue ? Math.Max(current.Value, next.Value) : current ?? next; private static string ConfidenceBand(decimal? confidence) => confidence switch { >= 0.75m => "High", >= 0.45m => "Medium", null => "Unknown", _ => "Low" }; private static string? FirstName(string name) { var parts = Clean(name).Split(' ', StringSplitOptions.RemoveEmptyEntries); return parts.Length > 1 ? parts[0] : null; } private static bool IsGenericPerson(string name) => GenericPeople.Contains(name) || GenericPeople.Contains(Normalise(name)); private static string Normalise(string? value) => Clean(value).ToLowerInvariant(); private static string Clean(string? value) => string.Join(' ', (value ?? string.Empty).Trim().Split((char[]?)null, StringSplitOptions.RemoveEmptyEntries)); private static HashSet CreateSet(params string[] values) => values.Select(Normalise).ToHashSet(StringComparer.OrdinalIgnoreCase); private sealed class CharacterIndex { private readonly Dictionary byName = new(StringComparer.OrdinalIgnoreCase); public void Add(string? name, int characterId, string characterName) { var key = Normalise(name); if (!string.IsNullOrWhiteSpace(key)) { byName.TryAdd(key, new CharacterMatch(characterId, characterName)); } } public CharacterMatch? Find(string name) => byName.TryGetValue(Normalise(name), out var match) ? match : null; } private sealed record CharacterMatch(int CharacterID, string CharacterName); private sealed class CharacterCandidate(string key, string displayName, int? existingCharacterId, string? existingCharacterName) { public string Key { get; } = key; public string DisplayName { get; } = displayName; public int? ExistingCharacterID { get; } = existingCharacterId; public string? ExistingCharacterName { get; } = existingCharacterName; public HashSet Aliases { get; } = new(StringComparer.OrdinalIgnoreCase); public List PresentAppearances { get; } = []; public List MentionedOnlyAppearances { get; } = []; public decimal? Confidence { get; set; } public string? Description { get; set; } public string? FirstAppearanceNote { get; set; } } private sealed record CharacterAppearanceImport( int SceneID, decimal SceneNumber, string SceneTitle, string? RoleInScene, bool MentionedOnly, string? Notes, decimal? Confidence, bool AlreadyLinked); private sealed record ImportedSceneAnalysis(int SceneID, SceneIntelligenceScene Parsed); private sealed record CharacterCandidateData( bool HasCommittedScenes, int AlreadyLinkedCount, IReadOnlyList Candidates, IReadOnlyList SceneAnalyses); }