using System.Text.Json; using PlotLine.Data; using PlotLine.Models; namespace PlotLine.Services; public interface IStoryIntelligenceMaterialisationService { Task MaterialiseReadyBookAsync(int bookId, int userId, CancellationToken cancellationToken = default); } public sealed class StoryIntelligenceMaterialisationService( IStoryIntelligenceResultRepository results, IStoryIntelligencePipelineRepository pipelines, ISceneMetricTypeRepository metricTypes, ISceneRepository scenes, IChapterRepository chapters, IStoryIntelligenceNarrativeReviewService narrative, ILogger logger) : IStoryIntelligenceMaterialisationService { private static readonly JsonSerializerOptions JsonOptions = new() { PropertyNamingPolicy = JsonNamingPolicy.CamelCase, PropertyNameCaseInsensitive = true }; public async Task MaterialiseReadyBookAsync(int bookId, int userId, CancellationToken cancellationToken = default) { var pipeline = await pipelines.GetByBookForUserAsync(bookId, userId); if (pipeline is null) { return StoryIntelligenceMaterialisationResult.NotReady(bookId, "No Story Intelligence pipeline exists for this book and user."); } var allRuns = (await results.ListRunsByBookForUserAsync(bookId, userId)) .Where(run => run.ProjectID == pipeline.ProjectID && run.BookID == bookId) .OrderBy(run => run.CreatedUtc) .ThenBy(run => run.StoryIntelligenceRunID) .ToList(); if (allRuns.Count == 0) { return StoryIntelligenceMaterialisationResult.NotReady(bookId, "No persisted Story Intelligence runs exist for this book and user."); } var lastRun = pipeline.LastRunID.HasValue ? allRuns.FirstOrDefault(run => run.StoryIntelligenceRunID == pipeline.LastRunID.Value) : null; lastRun ??= allRuns.Last(); var windowStart = lastRun.CreatedUtc.AddMinutes(-30); var scopedRuns = allRuns .Where(run => run.CreatedUtc >= windowStart) .OrderBy(run => run.ChapterNumber ?? decimal.MaxValue) .ThenBy(run => run.StoryIntelligenceRunID) .ToList(); if (scopedRuns.Any(run => run.Status is StoryIntelligenceRunStatuses.Pending or StoryIntelligenceRunStatuses.Running)) { return StoryIntelligenceMaterialisationResult.NotReady(bookId, "Story Intelligence still has active chapter runs."); } if (scopedRuns.Any(run => run.Status is StoryIntelligenceRunStatuses.Failed or StoryIntelligenceRunStatuses.Cancelled)) { return StoryIntelligenceMaterialisationResult.NotReady(bookId, "Story Intelligence has failed or cancelled chapter runs."); } var canonicalScenes = new List(); foreach (var chapter in await chapters.ListByBookAsync(bookId)) { canonicalScenes.AddRange(await scenes.ListByChapterAsync(chapter.ChapterID)); } var expectedScenes = canonicalScenes.Count(scene => !scene.IsArchived); var completedScenes = scopedRuns.Sum(run => run.CompletedScenes ?? 0); var failedScenes = scopedRuns.Sum(run => run.FailedScenes ?? 0); if (expectedScenes == 0 || completedScenes + failedScenes < expectedScenes) { return StoryIntelligenceMaterialisationResult.NotReady( bookId, $"Scene results are not complete. Completed/failed {completedScenes + failedScenes:N0} of {expectedScenes:N0} canonical scene(s)."); } var configuredMetrics = (await metricTypes.ListForManagementAsync(pipeline.ProjectID)) .Where(metric => metric.IsActive && metric.IsEnabledForProject) .OrderBy(metric => metric.SortOrder) .ThenBy(metric => metric.MetricName) .ToList(); var metricMap = BuildMetricMap(configuredMetrics); var metricValuesSaved = 0; var sceneResultsRead = 0; var sceneResultsParsed = 0; var batchItems = new List(); foreach (var run in scopedRuns.Where(run => run.Status is StoryIntelligenceRunStatuses.Completed or StoryIntelligenceRunStatuses.CompletedWithWarnings)) { cancellationToken.ThrowIfCancellationRequested(); if (run.ChapterID.HasValue) { batchItems.Add(new OnboardingStoryIntelligenceBatchItem { TemporaryChapterKey = $"book-{bookId}-chapter-{run.ChapterID.Value}", ChapterNumber = Convert.ToInt32(run.ChapterNumber ?? 0), ChapterTitle = string.Empty, ChapterID = run.ChapterID.Value, RunID = run.StoryIntelligenceRunID }); } foreach (var sceneResult in await results.ListSceneResultsAsync(run.StoryIntelligenceRunID)) { cancellationToken.ThrowIfCancellationRequested(); sceneResultsRead++; if (!sceneResult.SceneID.HasValue || sceneResult.ValidationErrorsCount > 0) { continue; } var parsed = TryReadScene(sceneResult); if (parsed is null) { continue; } sceneResultsParsed++; var metrics = BuildMetrics(parsed, metricMap); if (metrics.Count == 0) { continue; } await scenes.SaveMetricValuesAsync(sceneResult.SceneID.Value, metrics); metricValuesSaved += metrics.Count; } } var narrativeCandidatesBefore = (await narrative.ListCandidatesByBookAsync(bookId)).Count; var narrativeCandidatesAfter = narrativeCandidatesBefore; if (batchItems.Count > 0) { var batch = new OnboardingStoryIntelligenceBatch { UserID = userId, OnboardingID = 0, PreviewID = Guid.Empty, ProjectID = pipeline.ProjectID, BookID = bookId, ProjectName = pipeline.ProjectName, BookTitle = pipeline.BookDisplayTitle, Items = batchItems }; await narrative.EnsureCandidatesAsync(batch); narrativeCandidatesAfter = (await narrative.ListCandidatesByBookAsync(bookId)).Count; } logger.LogInformation( "Story Intelligence deterministic materialisation completed for BookID={BookID}. PipelineID={PipelineID} Runs={RunCount} SceneResultsRead={SceneResultsRead} SceneResultsParsed={SceneResultsParsed} MetricsSaved={MetricsSaved} NarrativeCandidatesBefore={NarrativeCandidatesBefore} NarrativeCandidatesAfter={NarrativeCandidatesAfter}.", bookId, pipeline.StoryIntelligenceBookPipelineID, scopedRuns.Count, sceneResultsRead, sceneResultsParsed, metricValuesSaved, narrativeCandidatesBefore, narrativeCandidatesAfter); return new StoryIntelligenceMaterialisationResult( bookId, pipeline.StoryIntelligenceBookPipelineID, IsReady: true, Reason: null, RunCount: scopedRuns.Count, ExpectedScenes: expectedScenes, SceneResultsRead: sceneResultsRead, SceneResultsParsed: sceneResultsParsed, MetricValuesSaved: metricValuesSaved, NarrativeCandidatesBefore: narrativeCandidatesBefore, NarrativeCandidatesAfter: narrativeCandidatesAfter); } public static IReadOnlyList BuildMetrics(SceneIntelligenceScene parsed, IReadOnlyDictionary metricMap) { var metrics = ReadMetrics(parsed); if (metrics.Count == 0) { return []; } var values = new List(); var importedMetricTypeIds = new HashSet(); foreach (var (name, metric) in metrics) { if (!metric.Score.HasValue || !metricMap.TryGetValue(name, out var metricType)) { continue; } if (!importedMetricTypeIds.Add(metricType.MetricTypeID)) { continue; } values.Add(new SceneMetricValue { MetricTypeID = metricType.MetricTypeID, MetricName = metricType.MetricName, Value = Math.Clamp(metric.Score.Value, metricType.MinValue, metricType.MaxValue), Notes = metric.Confidence.HasValue ? $"Story Intelligence confidence {metric.Confidence.Value:0.##}." : "Imported from Story Intelligence." }); } return values; } public static IReadOnlyDictionary BuildMetricMap(IReadOnlyList metricTypes) { var byName = metricTypes.ToDictionary(metric => Clean(metric.MetricName), metric => metric, StringComparer.OrdinalIgnoreCase); var map = new Dictionary(StringComparer.OrdinalIgnoreCase); foreach (var metric in metricTypes) { map[$"metric_{metric.MetricTypeID}"] = metric; var cleanedName = Clean(metric.MetricName); if (!string.IsNullOrWhiteSpace(cleanedName)) { map.TryAdd(cleanedName, metric); } } foreach (var (schemaName, aliases) in MetricAliases) { var metric = aliases.Select(alias => byName.TryGetValue(Clean(alias), out var match) ? match : null).FirstOrDefault(match => match is not null); if (metric is not null) { map[schemaName] = metric; } } return map; } private static IReadOnlyDictionary ReadMetrics(SceneIntelligenceScene parsed) { if (parsed.MetricValues is { Count: > 0 }) { return parsed.MetricValues .Select(metric => new { Key = metric.Key?.Trim(), metric.Score, metric.Confidence }) .Where(metric => !string.IsNullOrWhiteSpace(metric.Key)) .ToDictionary( metric => metric.Key!, metric => new SceneIntelligenceMetric { Score = metric.Score, Confidence = metric.Confidence }, StringComparer.OrdinalIgnoreCase); } return parsed.Metrics is { Count: > 0 } ? parsed.Metrics : new Dictionary(StringComparer.OrdinalIgnoreCase); } private static SceneIntelligenceScene? TryReadScene(StoryIntelligenceSavedSceneResult result) { foreach (var json in new[] { result.ParsedJson, result.OutputTextJson }) { if (string.IsNullOrWhiteSpace(json)) { continue; } try { var direct = JsonSerializer.Deserialize(json, JsonOptions); if (direct?.Summary is not null || direct?.MetricValues?.Count > 0 || direct?.Metrics?.Count > 0) { return direct; } using var document = JsonDocument.Parse(json); if (document.RootElement.TryGetProperty("parsedScene", out var parsedScene)) { return parsedScene.Deserialize(JsonOptions); } } catch (JsonException) { // Try the next persisted JSON form. } } return null; } private static string Clean(string? value) => string.IsNullOrWhiteSpace(value) ? string.Empty : new string(value.Where(char.IsLetterOrDigit).Select(char.ToLowerInvariant).ToArray()); private static readonly IReadOnlyDictionary MetricAliases = new Dictionary(StringComparer.OrdinalIgnoreCase) { ["overallIntensity"] = ["Overall Intensity", "Intensity"], ["tension"] = ["Tension"], ["emotionalWeight"] = ["Emotional Weight", "Emotion", "Emotional Resonance"], ["emotion"] = ["Emotion", "Emotional Weight", "Emotional Resonance"], ["action"] = ["Action"], ["conflict"] = ["Conflict", "Relationship Conflict", "Tension"], ["darkness"] = ["Darkness"], ["romance"] = ["Romance"], ["sexualCharge"] = ["Sexual Charge"], ["violence"] = ["Violence"], ["hopeLightness"] = ["Hope / Lightness", "Hope", "Lightness"], ["comedy"] = ["Comedy", "Humour", "Humor"], ["humour"] = ["Humour", "Humor", "Comedy", "Comic Relief"], ["humor"] = ["Humour", "Humor", "Comedy", "Comic Relief"], ["mystery"] = ["Mystery"], ["revelation"] = ["Revelation", "Reveal", "Mystery"], ["pacing"] = ["Pacing", "PacingIntensity", "Pacing Intensity", "Overall Intensity"], ["pacingIntensity"] = ["Pacing", "PacingIntensity", "Pacing Intensity", "Overall Intensity"] }; } public sealed record StoryIntelligenceMaterialisationResult( int BookID, int? StoryIntelligenceBookPipelineID, bool IsReady, string? Reason, int RunCount, int ExpectedScenes, int SceneResultsRead, int SceneResultsParsed, int MetricValuesSaved, int NarrativeCandidatesBefore, int NarrativeCandidatesAfter) { public static StoryIntelligenceMaterialisationResult NotReady(int bookId, string reason) => new(bookId, null, false, reason, 0, 0, 0, 0, 0, 0, 0); }