PlotDirector/PlotLine/Services/StoryIntelligenceMaterialisationService.cs

using System.Text.Json;
using PlotLine.Data;
using PlotLine.Models;

namespace PlotLine.Services;

public interface IStoryIntelligenceMaterialisationService
{
    Task<StoryIntelligenceMaterialisationResult> MaterialiseReadyBookAsync(int bookId, int userId, CancellationToken cancellationToken = default);
}

public sealed class StoryIntelligenceMaterialisationService(
    IStoryIntelligenceResultRepository results,
    IStoryIntelligencePipelineRepository pipelines,
    ISceneMetricTypeRepository metricTypes,
    ISceneRepository scenes,
    IChapterRepository chapters,
    IStoryIntelligenceNarrativeReviewService narrative,
    ILogger<StoryIntelligenceMaterialisationService> logger) : IStoryIntelligenceMaterialisationService
{
    private static readonly JsonSerializerOptions JsonOptions = new()
    {
        PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
        PropertyNameCaseInsensitive = true
    };

    public async Task<StoryIntelligenceMaterialisationResult> MaterialiseReadyBookAsync(int bookId, int userId, CancellationToken cancellationToken = default)
    {
        var pipeline = await pipelines.GetByBookForUserAsync(bookId, userId);
        if (pipeline is null)
        {
            return StoryIntelligenceMaterialisationResult.NotReady(bookId, "No Story Intelligence pipeline exists for this book and user.");
        }

        var allRuns = (await results.ListRunsByBookForUserAsync(bookId, userId))
            .Where(run => run.ProjectID == pipeline.ProjectID && run.BookID == bookId)
            .OrderBy(run => run.CreatedUtc)
            .ThenBy(run => run.StoryIntelligenceRunID)
            .ToList();
        if (allRuns.Count == 0)
        {
            return StoryIntelligenceMaterialisationResult.NotReady(bookId, "No persisted Story Intelligence runs exist for this book and user.");
        }

        var lastRun = pipeline.LastRunID.HasValue
            ? allRuns.FirstOrDefault(run => run.StoryIntelligenceRunID == pipeline.LastRunID.Value)
            : null;
        lastRun ??= allRuns.Last();
        var windowStart = lastRun.CreatedUtc.AddMinutes(-30);
        var scopedRuns = allRuns
            .Where(run => run.CreatedUtc >= windowStart)
            .OrderBy(run => run.ChapterNumber ?? decimal.MaxValue)
            .ThenBy(run => run.StoryIntelligenceRunID)
            .ToList();

        if (scopedRuns.Any(run => run.Status is StoryIntelligenceRunStatuses.Pending or StoryIntelligenceRunStatuses.Running))
        {
            return StoryIntelligenceMaterialisationResult.NotReady(bookId, "Story Intelligence still has active chapter runs.");
        }

        if (scopedRuns.Any(run => run.Status is StoryIntelligenceRunStatuses.Failed or StoryIntelligenceRunStatuses.Cancelled))
        {
            return StoryIntelligenceMaterialisationResult.NotReady(bookId, "Story Intelligence has failed or cancelled chapter runs.");
        }

        var canonicalScenes = new List<Scene>();
        foreach (var chapter in await chapters.ListByBookAsync(bookId))
        {
            canonicalScenes.AddRange(await scenes.ListByChapterAsync(chapter.ChapterID));
        }

        var expectedScenes = canonicalScenes.Count(scene => !scene.IsArchived);
        var completedScenes = scopedRuns.Sum(run => run.CompletedScenes ?? 0);
        var failedScenes = scopedRuns.Sum(run => run.FailedScenes ?? 0);
        if (expectedScenes == 0 || completedScenes + failedScenes < expectedScenes)
        {
            return StoryIntelligenceMaterialisationResult.NotReady(
                bookId,
                $"Scene results are not complete. Completed/failed {completedScenes + failedScenes:N0} of {expectedScenes:N0} canonical scene(s).");
        }

        var configuredMetrics = (await metricTypes.ListForManagementAsync(pipeline.ProjectID))
            .Where(metric => metric.IsActive && metric.IsEnabledForProject)
            .OrderBy(metric => metric.SortOrder)
            .ThenBy(metric => metric.MetricName)
            .ToList();
        var metricMap = BuildMetricMap(configuredMetrics);

        var metricValuesSaved = 0;
        var sceneResultsRead = 0;
        var sceneResultsParsed = 0;
        var batchItems = new List<OnboardingStoryIntelligenceBatchItem>();

        foreach (var run in scopedRuns.Where(run => run.Status is StoryIntelligenceRunStatuses.Completed or StoryIntelligenceRunStatuses.CompletedWithWarnings))
        {
            cancellationToken.ThrowIfCancellationRequested();
            if (run.ChapterID.HasValue)
            {
                batchItems.Add(new OnboardingStoryIntelligenceBatchItem
                {
                    TemporaryChapterKey = $"book-{bookId}-chapter-{run.ChapterID.Value}",
                    ChapterNumber = Convert.ToInt32(run.ChapterNumber ?? 0),
                    ChapterTitle = string.Empty,
                    ChapterID = run.ChapterID.Value,
                    RunID = run.StoryIntelligenceRunID
                });
            }

            foreach (var sceneResult in await results.ListSceneResultsAsync(run.StoryIntelligenceRunID))
            {
                cancellationToken.ThrowIfCancellationRequested();
                sceneResultsRead++;
                if (!sceneResult.SceneID.HasValue || sceneResult.ValidationErrorsCount > 0)
                {
                    continue;
                }

                var parsed = TryReadScene(sceneResult);
                if (parsed is null)
                {
                    continue;
                }

                sceneResultsParsed++;
                var metrics = BuildMetrics(parsed, metricMap);
                if (metrics.Count == 0)
                {
                    continue;
                }

                await scenes.SaveMetricValuesAsync(sceneResult.SceneID.Value, metrics);
                metricValuesSaved += metrics.Count;
            }
        }

        var narrativeCandidatesBefore = (await narrative.ListCandidatesByBookAsync(bookId)).Count;
        var narrativeCandidatesAfter = narrativeCandidatesBefore;
        if (batchItems.Count > 0)
        {
            var batch = new OnboardingStoryIntelligenceBatch
            {
                UserID = userId,
                OnboardingID = 0,
                PreviewID = Guid.Empty,
                ProjectID = pipeline.ProjectID,
                BookID = bookId,
                ProjectName = pipeline.ProjectName,
                BookTitle = pipeline.BookDisplayTitle,
                Items = batchItems
            };
            await narrative.EnsureCandidatesAsync(batch);
            narrativeCandidatesAfter = (await narrative.ListCandidatesByBookAsync(bookId)).Count;
        }

        logger.LogInformation(
            "Story Intelligence deterministic materialisation completed for BookID={BookID}. PipelineID={PipelineID} Runs={RunCount} SceneResultsRead={SceneResultsRead} SceneResultsParsed={SceneResultsParsed} MetricsSaved={MetricsSaved} NarrativeCandidatesBefore={NarrativeCandidatesBefore} NarrativeCandidatesAfter={NarrativeCandidatesAfter}.",
            bookId,
            pipeline.StoryIntelligenceBookPipelineID,
            scopedRuns.Count,
            sceneResultsRead,
            sceneResultsParsed,
            metricValuesSaved,
            narrativeCandidatesBefore,
            narrativeCandidatesAfter);

        return new StoryIntelligenceMaterialisationResult(
            bookId,
            pipeline.StoryIntelligenceBookPipelineID,
            IsReady: true,
            Reason: null,
            RunCount: scopedRuns.Count,
            ExpectedScenes: expectedScenes,
            SceneResultsRead: sceneResultsRead,
            SceneResultsParsed: sceneResultsParsed,
            MetricValuesSaved: metricValuesSaved,
            NarrativeCandidatesBefore: narrativeCandidatesBefore,
            NarrativeCandidatesAfter: narrativeCandidatesAfter);
    }

    public static IReadOnlyList<SceneMetricValue> BuildMetrics(SceneIntelligenceScene parsed, IReadOnlyDictionary<string, SceneMetricType> metricMap)
    {
        var metrics = ReadMetrics(parsed);
        if (metrics.Count == 0)
        {
            return [];
        }

        var values = new List<SceneMetricValue>();
        var importedMetricTypeIds = new HashSet<int>();
        foreach (var (name, metric) in metrics)
        {
            if (!metric.Score.HasValue || !metricMap.TryGetValue(name, out var metricType))
            {
                continue;
            }

            if (!importedMetricTypeIds.Add(metricType.MetricTypeID))
            {
                continue;
            }

            values.Add(new SceneMetricValue
            {
                MetricTypeID = metricType.MetricTypeID,
                MetricName = metricType.MetricName,
                Value = Math.Clamp(metric.Score.Value, metricType.MinValue, metricType.MaxValue),
                Notes = metric.Confidence.HasValue ? $"Story Intelligence confidence {metric.Confidence.Value:0.##}." : "Imported from Story Intelligence."
            });
        }

        return values;
    }

    public static IReadOnlyDictionary<string, SceneMetricType> BuildMetricMap(IReadOnlyList<SceneMetricType> metricTypes)
    {
        var byName = metricTypes.ToDictionary(metric => Clean(metric.MetricName), metric => metric, StringComparer.OrdinalIgnoreCase);
        var map = new Dictionary<string, SceneMetricType>(StringComparer.OrdinalIgnoreCase);
        foreach (var metric in metricTypes)
        {
            map[$"metric_{metric.MetricTypeID}"] = metric;
            var cleanedName = Clean(metric.MetricName);
            if (!string.IsNullOrWhiteSpace(cleanedName))
            {
                map.TryAdd(cleanedName, metric);
            }
        }

        foreach (var (schemaName, aliases) in MetricAliases)
        {
            var metric = aliases.Select(alias => byName.TryGetValue(Clean(alias), out var match) ? match : null).FirstOrDefault(match => match is not null);
            if (metric is not null)
            {
                map[schemaName] = metric;
            }
        }

        return map;
    }

    private static IReadOnlyDictionary<string, SceneIntelligenceMetric> ReadMetrics(SceneIntelligenceScene parsed)
    {
        if (parsed.MetricValues is { Count: > 0 })
        {
            return parsed.MetricValues
                .Select(metric => new { Key = metric.Key?.Trim(), metric.Score, metric.Confidence })
                .Where(metric => !string.IsNullOrWhiteSpace(metric.Key))
                .ToDictionary(
                    metric => metric.Key!,
                    metric => new SceneIntelligenceMetric { Score = metric.Score, Confidence = metric.Confidence },
                    StringComparer.OrdinalIgnoreCase);
        }

        return parsed.Metrics is { Count: > 0 }
            ? parsed.Metrics
            : new Dictionary<string, SceneIntelligenceMetric>(StringComparer.OrdinalIgnoreCase);
    }

    private static SceneIntelligenceScene? TryReadScene(StoryIntelligenceSavedSceneResult result)
    {
        foreach (var json in new[] { result.ParsedJson, result.OutputTextJson })
        {
            if (string.IsNullOrWhiteSpace(json))
            {
                continue;
            }

            try
            {
                var direct = JsonSerializer.Deserialize<SceneIntelligenceScene>(json, JsonOptions);
                if (direct?.Summary is not null || direct?.MetricValues?.Count > 0 || direct?.Metrics?.Count > 0)
                {
                    return direct;
                }

                using var document = JsonDocument.Parse(json);
                if (document.RootElement.TryGetProperty("parsedScene", out var parsedScene))
                {
                    return parsedScene.Deserialize<SceneIntelligenceScene>(JsonOptions);
                }
            }
            catch (JsonException)
            {
                // Try the next persisted JSON form.
            }
        }

        return null;
    }

    private static string Clean(string? value)
        => string.IsNullOrWhiteSpace(value)
            ? string.Empty
            : new string(value.Where(char.IsLetterOrDigit).Select(char.ToLowerInvariant).ToArray());

    private static readonly IReadOnlyDictionary<string, string[]> MetricAliases = new Dictionary<string, string[]>(StringComparer.OrdinalIgnoreCase)
    {
        ["overallIntensity"] = ["Overall Intensity", "Intensity"],
        ["tension"] = ["Tension"],
        ["emotionalWeight"] = ["Emotional Weight", "Emotion", "Emotional Resonance"],
        ["emotion"] = ["Emotion", "Emotional Weight", "Emotional Resonance"],
        ["action"] = ["Action"],
        ["conflict"] = ["Conflict", "Relationship Conflict", "Tension"],
        ["darkness"] = ["Darkness"],
        ["romance"] = ["Romance"],
        ["sexualCharge"] = ["Sexual Charge"],
        ["violence"] = ["Violence"],
        ["hopeLightness"] = ["Hope / Lightness", "Hope", "Lightness"],
        ["comedy"] = ["Comedy", "Humour", "Humor"],
        ["humour"] = ["Humour", "Humor", "Comedy", "Comic Relief"],
        ["humor"] = ["Humour", "Humor", "Comedy", "Comic Relief"],
        ["mystery"] = ["Mystery"],
        ["revelation"] = ["Revelation", "Reveal", "Mystery"],
        ["pacing"] = ["Pacing", "PacingIntensity", "Pacing Intensity", "Overall Intensity"],
        ["pacingIntensity"] = ["Pacing", "PacingIntensity", "Pacing Intensity", "Overall Intensity"]
    };
}

public sealed record StoryIntelligenceMaterialisationResult(
    int BookID,
    int? StoryIntelligenceBookPipelineID,
    bool IsReady,
    string? Reason,
    int RunCount,
    int ExpectedScenes,
    int SceneResultsRead,
    int SceneResultsParsed,
    int MetricValuesSaved,
    int NarrativeCandidatesBefore,
    int NarrativeCandidatesAfter)
{
    public static StoryIntelligenceMaterialisationResult NotReady(int bookId, string reason)
        => new(bookId, null, false, reason, 0, 0, 0, 0, 0, 0, 0);
}