using System.Text.Json; using System.Text.RegularExpressions; using PlotLine.Data; using PlotLine.Models; using PlotLine.ViewModels; namespace PlotLine.Services; public interface IStoryIntelligenceLocationImportService { Task BuildReviewAsync(OnboardingStoryIntelligenceBatch batch); Task ImportAsync(OnboardingStoryIntelligenceBatch batch, StoryIntelligenceLocationImportForm form); } public sealed class StoryIntelligenceLocationImportService( IStoryIntelligenceResultRepository runs, ILocationRepository locations, ISceneRepository scenes, IStoryIntelligencePipelineStateService pipelineState, ILogger logger) : IStoryIntelligenceLocationImportService { private static readonly JsonSerializerOptions JsonOptions = new() { PropertyNamingPolicy = JsonNamingPolicy.CamelCase, PropertyNameCaseInsensitive = true }; private static readonly HashSet GenericLocations = CreateSet( "room", "place", "area", "building", "street", "road", "door", "doorway", "wall", "floor", "pavement", "car", "car park", "parking lot", "stairs", "stairwell", "landing", "kitchen", "bathroom", "hallway", "corridor", "lobby", "reception", "desk", "pew", "altar", "outside", "inside", "nearby", "somewhere", "home", "house", "flat"); private static readonly HashSet InternalLocationWords = CreateSet( "room", "kitchen", "bathroom", "hallway", "corridor", "lobby", "reception", "stairwell", "stairs", "landing", "doorway", "pew", "altar", "floor"); private static readonly Regex AddressPattern = new(@"^\d+\s+\p{L}", RegexOptions.IgnoreCase | RegexOptions.Compiled); private static readonly Regex NamedRoadPattern = new(@"\b(street|road|lane|avenue|drive|way|close|crescent|square|place)\b", RegexOptions.IgnoreCase | RegexOptions.Compiled); public async Task BuildReviewAsync(OnboardingStoryIntelligenceBatch batch) { var data = await BuildCandidateDataAsync(batch); var decidedKeys = batch.LocationDecisions.Select(decision => decision.Key).ToHashSet(StringComparer.OrdinalIgnoreCase); var visibleCandidates = data.Candidates.Where(candidate => !decidedKeys.Contains(candidate.Key)).ToList(); return new StoryIntelligenceLocationReviewViewModel { HasCommittedScenes = data.HasCommittedScenes, CanImport = visibleCandidates.Count > 0, AlreadyLinkedCount = data.AlreadyLinkedCount, IsComplete = data.HasCommittedScenes && visibleCandidates.Count == 0, Candidates = visibleCandidates.Select(candidate => new StoryIntelligenceLocationReviewCandidateViewModel { Key = candidate.Key, LocationName = candidate.DisplayName, ImportName = candidate.DisplayName, Category = candidate.Category, AppearsInScenes = candidate.Appearances.Select(appearance => appearance.SceneID).Distinct().Count(), Confidence = ConfidenceBand(candidate.Confidence), ParentLocationHint = candidate.ParentLocationHint, ExampleFirstAppearance = candidate.FirstAppearanceNote, ExampleContext = candidate.Description, ExistingLocationID = candidate.ExistingLocationID, ExistingLocationName = candidate.ExistingLocationName }).ToList() }; } public async Task ImportAsync(OnboardingStoryIntelligenceBatch batch, StoryIntelligenceLocationImportForm form) { var data = await BuildCandidateDataAsync(batch); if (!data.HasCommittedScenes) { return new StoryIntelligenceImportCommitResult { Success = false, Message = "Create scenes before importing locations." }; } var choices = form.Locations .Where(choice => !string.IsNullOrWhiteSpace(choice.Key)) .ToDictionary(choice => choice.Key, choice => choice, StringComparer.OrdinalIgnoreCase); if (choices.Count == 0) { return new StoryIntelligenceImportCommitResult { Success = false, Message = "Choose at least one location to create, link or ignore." }; } var existingIndex = await BuildLocationIndexAsync(batch.ProjectID); var lookupData = await locations.GetLookupsAsync(); var created = 0; var linkedExisting = 0; var ignored = 0; var aliasesAdded = 0; var sceneLocationsUpdated = 0; var resolvedNames = new Dictionary(StringComparer.OrdinalIgnoreCase); var resolvedCandidateIds = new Dictionary(StringComparer.OrdinalIgnoreCase); var candidateByKey = data.Candidates.ToDictionary(candidate => candidate.Key, StringComparer.OrdinalIgnoreCase); foreach (var candidate in data.Candidates) { if (!choices.TryGetValue(candidate.Key, out var choice)) { continue; } if (string.Equals(choice.Action, StoryIntelligenceLocationImportActions.Ignore, StringComparison.OrdinalIgnoreCase)) { ignored++; AddDecision(batch, candidate.Key, StoryIntelligenceLocationImportActions.Ignore, candidate.DisplayName, null, false); continue; } if (string.Equals(choice.Action, StoryIntelligenceLocationImportActions.Alias, StringComparison.OrdinalIgnoreCase)) { continue; } var requestedName = Clean(choice.ImportName); var importName = string.IsNullOrWhiteSpace(requestedName) ? candidate.DisplayName : requestedName; var forceCreateSeparate = string.Equals(choice.Action, StoryIntelligenceLocationImportActions.CreateSeparate, StringComparison.OrdinalIgnoreCase); var matchedExistingId = forceCreateSeparate ? null : candidate.ExistingLocationID ?? existingIndex.Find(importName)?.LocationID ?? existingIndex.Find(candidate.DisplayName)?.LocationID; var locationId = matchedExistingId; if (!locationId.HasValue) { var parentId = ResolveParentLocationId(candidate.ParentLocationHint, existingIndex); locationId = await locations.SaveAsync(new LocationItem { ProjectID = batch.ProjectID, ParentLocationID = parentId, LocationName = importName, LocationTypeID = MatchLocationType(lookupData.LocationTypes, candidate), Description = BuildLocationDescription(candidate), DetectionPriority = 50 }); created++; } else { linkedExisting++; } AddResolvedName(resolvedNames, candidate.DisplayName, locationId.Value); AddResolvedName(resolvedNames, importName, locationId.Value); resolvedCandidateIds[candidate.Key] = locationId.Value; AddDecision(batch, candidate.Key, choice.Action, importName, locationId.Value, true); } foreach (var candidate in data.Candidates) { if (!choices.TryGetValue(candidate.Key, out var choice) || !string.Equals(choice.Action, StoryIntelligenceLocationImportActions.Alias, StringComparison.OrdinalIgnoreCase)) { continue; } var targetKey = Clean(choice.AliasTargetKey); if (string.IsNullOrWhiteSpace(targetKey) || string.Equals(targetKey, candidate.Key, StringComparison.OrdinalIgnoreCase) || !choices.TryGetValue(targetKey, out var targetChoice) || string.Equals(targetChoice.Action, StoryIntelligenceLocationImportActions.Ignore, StringComparison.OrdinalIgnoreCase) || string.Equals(targetChoice.Action, StoryIntelligenceLocationImportActions.Alias, StringComparison.OrdinalIgnoreCase) || !candidateByKey.TryGetValue(targetKey, out var targetCandidate) || !resolvedCandidateIds.TryGetValue(targetKey, out var targetLocationId)) { logger.LogWarning( "Story Intelligence location alias candidate {CandidateKey} could not be imported because target {TargetKey} was not a resolved create/link target.", candidate.Key, targetKey); continue; } var targetName = targetCandidate.ExistingLocationName ?? targetChoice.ImportName ?? targetCandidate.DisplayName; if (await TryAddAliasAsync(targetLocationId, candidate.DisplayName, targetName)) { aliasesAdded++; } AddResolvedName(resolvedNames, candidate.DisplayName, targetLocationId); AddDecision(batch, candidate.Key, StoryIntelligenceLocationImportActions.Alias, candidate.DisplayName, targetLocationId, true); } foreach (var candidate in data.Candidates) { if (!choices.TryGetValue(candidate.Key, out var choice) || string.Equals(choice.Action, StoryIntelligenceLocationImportActions.Ignore, StringComparison.OrdinalIgnoreCase)) { continue; } var locationId = string.Equals(choice.Action, StoryIntelligenceLocationImportActions.Alias, StringComparison.OrdinalIgnoreCase) ? resolvedCandidateIds.GetValueOrDefault(Clean(choice.AliasTargetKey)) : resolvedCandidateIds.GetValueOrDefault(candidate.Key); if (locationId <= 0) { continue; } foreach (var appearance in candidate.Appearances.Where(appearance => appearance.PresentInScene && !appearance.AlreadyLinked)) { var scene = await scenes.GetAsync(appearance.SceneID); if (scene is null || scene.PrimaryLocationID.HasValue) { continue; } scene.PrimaryLocationID = locationId; await scenes.SaveAsync(scene); sceneLocationsUpdated++; } } batch.LocationStageComplete = true; batch.LastLocationImportResult = new StoryIntelligenceLocationImportBatchResult { LocationsCreated = created, LocationsLinked = linkedExisting, LocationAliasesAdded = aliasesAdded, LocationsIgnored = ignored, SceneLocationsUpdated = sceneLocationsUpdated }; await pipelineState.RecordLocationImportAsync(batch.ProjectID, batch.BookID); logger.LogInformation( "Imported Story Intelligence locations for batch {BatchID}. Created={Created} LinkedExisting={LinkedExisting} Aliases={Aliases} Ignored={Ignored} SceneLocationsUpdated={SceneLocationsUpdated}", batch.BatchID, created, linkedExisting, aliasesAdded, ignored, sceneLocationsUpdated); return new StoryIntelligenceImportCommitResult { Success = true, ScenesCreated = sceneLocationsUpdated, Message = $"Locations imported. {created:N0} created, {linkedExisting:N0} linked, {aliasesAdded:N0} alias/variant(s) added, {ignored:N0} ignored, {sceneLocationsUpdated:N0} scene location(s) updated." }; } private async Task BuildCandidateDataAsync(OnboardingStoryIntelligenceBatch batch) { var existingIndex = await BuildLocationIndexAsync(batch.ProjectID); var groups = new Dictionary(StringComparer.OrdinalIgnoreCase); var alreadyLinked = 0; var hasCommittedScenes = false; foreach (var item in batch.Items) { var commit = await runs.GetImportCommitAsync(item.RunID); if (commit is null || !string.Equals(commit.Status, StoryIntelligenceImportCommitStatuses.Completed, StringComparison.OrdinalIgnoreCase)) { continue; } hasCommittedScenes = true; var importedScenes = (await scenes.ListByChapterAsync(item.ChapterID)) .Where(scene => scene.ImportRunID == item.RunID) .ToList(); var importedByRange = importedScenes .Where(scene => scene.SourceStartParagraph.HasValue && scene.SourceEndParagraph.HasValue) .ToDictionary(scene => $"{scene.SourceStartParagraph}-{scene.SourceEndParagraph}", scene => scene, StringComparer.OrdinalIgnoreCase); var importedByNumber = importedScenes.ToDictionary(scene => Convert.ToInt32(scene.SceneNumber), scene => scene); var sceneResults = await runs.ListSceneResultsAsync(item.RunID); foreach (var sceneResult in sceneResults) { var parsed = TryReadScene(sceneResult); if (parsed is null) { continue; } var importedScene = ResolveImportedScene(sceneResult, importedByRange, importedByNumber); if (importedScene is null) { continue; } if (importedScene.PrimaryLocationID.HasValue) { alreadyLinked++; } AddSceneLocations(groups, existingIndex, importedScene, parsed); } } var decidedKeys = batch.LocationDecisions.Select(decision => decision.Key).ToHashSet(StringComparer.OrdinalIgnoreCase); var candidates = groups.Values .Where(candidate => candidate.Appearances.Any(appearance => !appearance.AlreadyLinked)) .Where(candidate => IsVisibleLocationCandidate(candidate)) .OrderByDescending(candidate => candidate.Appearances.Select(appearance => appearance.SceneID).Distinct().Count()) .ThenBy(candidate => candidate.DisplayName) .ToList(); return new LocationCandidateData(hasCommittedScenes, alreadyLinked, candidates, decidedKeys); } private static void AddSceneLocations( Dictionary groups, LocationIndex existingIndex, Scene importedScene, SceneIntelligenceScene parsed) { var setting = parsed.Setting; var settingName = FirstConfigured(setting?.LocationName, setting?.GenericRoomType); AddLocation(groups, existingIndex, importedScene, parsed, settingName, setting?.LocationType, setting?.GenericRoomType, setting?.ParentLocationHint, true, false, setting?.Confidence, "Scene setting"); foreach (var location in parsed.Locations ?? []) { AddLocation(groups, existingIndex, importedScene, parsed, location.Name, location.LocationType, location.GenericRoomType, location.ParentLocationHint, location.PresentInScene == true, location.MentionedOnly == true, location.Confidence, location.Notes); } foreach (var observation in parsed.Observations ?? []) { if (IsLocationEntity(observation.SubjectEntityType)) { AddLocation(groups, existingIndex, importedScene, parsed, observation.SubjectName, null, null, null, IsPresentPredicate(observation.Predicate), !IsPresentPredicate(observation.Predicate), observation.Confidence, observation.Description); } if (IsLocationEntity(observation.ObjectEntityType)) { AddLocation(groups, existingIndex, importedScene, parsed, observation.ObjectName, null, null, null, IsPresentPredicate(observation.Predicate), !IsPresentPredicate(observation.Predicate), observation.Confidence, observation.Description); } } } private static void AddLocation( Dictionary groups, LocationIndex existingIndex, Scene importedScene, SceneIntelligenceScene parsed, string? name, string? locationType, string? genericRoomType, string? parentLocationHint, bool presentInScene, bool mentionedOnly, decimal? confidence, string? notes) { var cleanName = CleanLocationName(name); if (!IsLocationNameCandidate(cleanName)) { return; } var key = ResolveCandidateKey(groups, existingIndex, cleanName); if (!groups.TryGetValue(key, out var candidate)) { var match = existingIndex.Find(cleanName); candidate = new LocationCandidate(key, cleanName, match?.LocationID, match?.LocationName) { Category = Categorise(cleanName, locationType, genericRoomType), ParentLocationHint = Clean(parentLocationHint) }; groups[key] = candidate; } else if (!string.Equals(candidate.DisplayName, cleanName, StringComparison.OrdinalIgnoreCase)) { candidate.Aliases.Add(cleanName); } candidate.Confidence = Max(candidate.Confidence, confidence); candidate.Description ??= Clean(notes); candidate.FirstAppearanceNote ??= BuildFirstAppearance(importedScene, parsed, cleanName); candidate.ParentLocationHint ??= Clean(parentLocationHint); candidate.Appearances.Add(new LocationAppearanceImport( importedScene.SceneID, importedScene.SceneNumber, importedScene.SceneTitle, presentInScene && !mentionedOnly, mentionedOnly, notes, confidence, importedScene.PrimaryLocationID.HasValue && candidate.ExistingLocationID == importedScene.PrimaryLocationID)); } private async Task BuildLocationIndexAsync(int projectId) { var index = new LocationIndex(); foreach (var location in await locations.ListByProjectAsync(projectId)) { index.Add(location.LocationName, location.LocationID, location.LocationName); index.Add(location.LocationPath, location.LocationID, location.LocationName); foreach (var alias in await locations.ListAliasesAsync(location.LocationID)) { index.Add(alias.Alias, location.LocationID, location.LocationName); } } return index; } private async Task TryAddAliasAsync(int locationId, string alias, string primaryName) { var cleanAlias = Clean(alias); if (string.IsNullOrWhiteSpace(cleanAlias) || string.Equals(cleanAlias, primaryName, StringComparison.OrdinalIgnoreCase)) { return false; } var existingAliases = await locations.ListAliasesAsync(locationId); if (existingAliases.Any(item => string.Equals(Clean(item.Alias), cleanAlias, StringComparison.OrdinalIgnoreCase))) { return false; } await locations.AddAliasAsync(locationId, cleanAlias, null); return true; } private static Scene? ResolveImportedScene( StoryIntelligenceSavedSceneResult sceneResult, IReadOnlyDictionary byRange, IReadOnlyDictionary byNumber) { if (sceneResult.StartParagraph.HasValue && sceneResult.EndParagraph.HasValue && byRange.TryGetValue($"{sceneResult.StartParagraph}-{sceneResult.EndParagraph}", out var rangeMatch)) { return rangeMatch; } return byNumber.TryGetValue(sceneResult.TemporarySceneNumber, out var numberMatch) ? numberMatch : null; } private static SceneIntelligenceScene? TryReadScene(StoryIntelligenceSavedSceneResult result) { if (string.IsNullOrWhiteSpace(result.ParsedJson)) { return null; } try { var direct = JsonSerializer.Deserialize(result.ParsedJson, JsonOptions); if (!string.IsNullOrWhiteSpace(direct?.SchemaVersion)) { return direct; } using var document = JsonDocument.Parse(result.ParsedJson); if (document.RootElement.TryGetProperty("parsedScene", out var parsedScene)) { return parsedScene.Deserialize(JsonOptions); } } catch (JsonException) { } return null; } private static int? MatchLocationType(IReadOnlyList types, LocationCandidate candidate) => types.FirstOrDefault(type => candidate.Category.Contains(type.TypeName, StringComparison.OrdinalIgnoreCase) || type.TypeName.Contains(candidate.Category, StringComparison.OrdinalIgnoreCase))?.LocationTypeID; private static int? ResolveParentLocationId(string? parentHint, LocationIndex index) => string.IsNullOrWhiteSpace(parentHint) ? null : index.Find(parentHint)?.LocationID; private static string BuildLocationDescription(LocationCandidate candidate) { var lines = new List(); if (!string.IsNullOrWhiteSpace(candidate.ParentLocationHint)) { lines.Add($"Parent/location hint from Story Intelligence: {candidate.ParentLocationHint}"); } if (!string.IsNullOrWhiteSpace(candidate.Description)) { lines.Add(candidate.Description); } return string.Join(Environment.NewLine, lines); } private static bool IsVisibleLocationCandidate(LocationCandidate candidate) { if (IsNamedLocation(candidate.DisplayName)) { return true; } if (!string.IsNullOrWhiteSpace(candidate.ParentLocationHint)) { return candidate.Appearances.Count(appearance => appearance.PresentInScene) > 0; } if (IsGenericLocation(candidate.DisplayName)) { return candidate.Appearances.Select(appearance => appearance.SceneID).Distinct().Count() >= 2; } return true; } private static bool IsLocationNameCandidate(string? name) { var clean = CleanLocationName(name); return !string.IsNullOrWhiteSpace(clean) && clean.Length > 1 && clean.Any(char.IsLetter); } private static bool IsGenericLocation(string name) => GenericLocations.Contains(Normalise(name)) || InternalLocationWords.Contains(Normalise(name)); private static bool IsNamedLocation(string name) => AddressPattern.IsMatch(name) || IsNamedRoad(name) || name.Any(char.IsUpper) || name.Contains('\'') || name.Contains('’'); private static bool IsNamedRoad(string name) => NamedRoadPattern.IsMatch(name) && !GenericLocations.Contains(Normalise(name)); private static string Categorise(string name, string? locationType, string? genericRoomType) { if (AddressPattern.IsMatch(name)) { return "Address"; } if (IsNamedRoad(name)) { return "Road / street"; } if (Normalise(locationType).Contains("generic", StringComparison.OrdinalIgnoreCase) || !string.IsNullOrWhiteSpace(genericRoomType)) { return "Room / interior area"; } if (Normalise(name).Contains("car", StringComparison.OrdinalIgnoreCase)) { return "Vehicle-as-location"; } return IsNamedLocation(name) ? "Named location" : "Ambiguous location"; } private static bool IsLocationEntity(string? entityType) => string.Equals(Clean(entityType), "Location", StringComparison.OrdinalIgnoreCase); private static bool IsPresentPredicate(string? predicate) { var value = Clean(predicate); return value.Contains("isPresentAt", StringComparison.OrdinalIgnoreCase) || value.Contains("travelsTo", StringComparison.OrdinalIgnoreCase) || value.Contains("occursAt", StringComparison.OrdinalIgnoreCase); } private static string ResolveCandidateKey( IReadOnlyDictionary groups, LocationIndex existingIndex, string name) { var existingMatch = existingIndex.Find(name); if (existingMatch is not null) { var existingKey = CanonicalLocationKey(existingMatch.LocationName); if (groups.ContainsKey(existingKey)) { return existingKey; } return existingKey; } var canonical = CanonicalLocationKey(name); return groups.ContainsKey(canonical) ? canonical : canonical; } private static void AddResolvedName(Dictionary map, string name, int locationId) { var clean = Clean(name); if (!string.IsNullOrWhiteSpace(clean)) { map[clean] = locationId; } } private static void AddDecision(OnboardingStoryIntelligenceBatch batch, string key, string action, string? locationName, int? locationId, bool createdOrLinked) { batch.LocationDecisions.RemoveAll(decision => string.Equals(decision.Key, key, StringComparison.OrdinalIgnoreCase)); batch.LocationDecisions.Add(new OnboardingStoryIntelligenceLocationDecision { Key = key, Action = action, LocationName = locationName, LocationID = locationId, CreatedOrLinked = createdOrLinked }); } private static string? BuildFirstAppearance(Scene scene, SceneIntelligenceScene parsed, string locationName) { var summary = Clean(parsed.Summary?.Short); return string.IsNullOrWhiteSpace(summary) ? $"Scene {scene.SceneNumber:g}: {locationName}" : $"Scene {scene.SceneNumber:g}: {summary}"; } private static string FirstConfigured(params string?[] values) => values.FirstOrDefault(value => !string.IsNullOrWhiteSpace(value))?.Trim() ?? string.Empty; private static decimal? Max(decimal? current, decimal? next) => current.HasValue && next.HasValue ? Math.Max(current.Value, next.Value) : current ?? next; private static string ConfidenceBand(decimal? confidence) => confidence switch { >= 0.75m => "High", >= 0.45m => "Medium", null => "Unknown", _ => "Low" }; private static string CanonicalLocationKey(string name) => Normalise(CleanLocationName(name) .Replace("the ", string.Empty, StringComparison.OrdinalIgnoreCase)); private static string CleanLocationName(string? value) { var clean = Clean(value).Trim(' ', '.', ',', ';', ':', '!', '?', '"', '\''); if (clean.EndsWith("'s", StringComparison.OrdinalIgnoreCase) || clean.EndsWith("’s", StringComparison.OrdinalIgnoreCase)) { return clean; } return clean; } private static string Normalise(string? value) => Clean(value).ToLowerInvariant(); private static string Clean(string? value) => string.Join(' ', (value ?? string.Empty).Trim().Split((char[]?)null, StringSplitOptions.RemoveEmptyEntries)); private static HashSet CreateSet(params string[] values) => values.Select(Normalise).ToHashSet(StringComparer.OrdinalIgnoreCase); private sealed class LocationIndex { private readonly Dictionary byName = new(StringComparer.OrdinalIgnoreCase); public void Add(string? name, int locationId, string locationName) { var key = CanonicalLocationKey(name ?? string.Empty); if (!string.IsNullOrWhiteSpace(key)) { byName.TryAdd(key, new LocationMatch(locationId, locationName)); } } public LocationMatch? Find(string name) => byName.TryGetValue(CanonicalLocationKey(name), out var match) ? match : null; } private sealed record LocationMatch(int LocationID, string LocationName); private sealed class LocationCandidate(string key, string displayName, int? existingLocationId, string? existingLocationName) { public string Key { get; } = key; public string DisplayName { get; } = displayName; public int? ExistingLocationID { get; } = existingLocationId; public string? ExistingLocationName { get; } = existingLocationName; public string Category { get; set; } = "Ambiguous location"; public string? ParentLocationHint { get; set; } public HashSet Aliases { get; } = new(StringComparer.OrdinalIgnoreCase); public List Appearances { get; } = []; public decimal? Confidence { get; set; } public string? Description { get; set; } public string? FirstAppearanceNote { get; set; } } private sealed record LocationAppearanceImport( int SceneID, decimal SceneNumber, string SceneTitle, bool PresentInScene, bool MentionedOnly, string? Notes, decimal? Confidence, bool AlreadyLinked); private sealed record LocationCandidateData( bool HasCommittedScenes, int AlreadyLinkedCount, IReadOnlyList Candidates, IReadOnlySet DecidedKeys); }