using System.Diagnostics;
using System.Text.Json;
using Microsoft.Extensions.Options;
using PlotLine.Data;
using PlotLine.Models;

namespace PlotLine.Services;

public interface IPersistedStoryIntelligenceRunner
{
    Task ProcessNextAsync(CancellationToken cancellationToken);
}

public sealed class PersistedStoryIntelligenceRunner(
    IStoryIntelligenceResultRepository repository,
    IStoryIntelligencePipelineRepository pipelines,
    IStoryPromptRepository prompts,
    IStoryPromptVersionService versions,
    IStoryPromptBuilder scenePromptBuilder,
    IStoryIntelligenceClient client,
    IChapterStructureValidator chapterValidator,
    IStorySceneValidator sceneValidator,
    IOptions<StoryIntelligenceOptions> options,
    IOptions<StoryIntelligencePricingOptions> pricingOptions,
    IStoryIntelligenceProgressNotifier notifier,
    IStoryMemoryService storyMemory,
    ICharacterIntelligenceService characterIntelligence,
    IStoryIntelligenceEntitlementService entitlement,
    IStoryIntelligenceCompletionNotificationService completionNotifications,
    ILogger<PersistedStoryIntelligenceRunner> logger) : IPersistedStoryIntelligenceRunner
{
    private const string ChapterPromptFile = "Chapter-Structure-Prompt-V2.md";
    private const string ScenePromptFile = "Scene-Prompt-V2.md";
    private static readonly JsonSerializerOptions JsonOptions = new()
    {
        PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
        PropertyNameCaseInsensitive = true,
        WriteIndented = true
    };

    private readonly StoryIntelligenceOptions settings = options.Value;
    private readonly StoryIntelligencePricingOptions pricing = pricingOptions.Value;

    public async Task ProcessNextAsync(CancellationToken cancellationToken)
    {
        var run = await repository.ClaimNextPendingAsync();
        if (run is null)
        {
            return;
        }

        await ProcessAsync(run, cancellationToken);
    }

    private async Task ProcessAsync(StoryIntelligenceQueuedRun run, CancellationToken cancellationToken)
    {
        var stopwatch = Stopwatch.StartNew();
        var totals = new TokenTotals();
        var chapterPromptVersion = versions.GetPromptVersion(ChapterPromptFile);
        var scenePromptVersion = versions.GetPromptVersion(ScenePromptFile);
        var stageModels = ResolveStageModels(run);
        int? chapterResultId = null;
        var failedScenes = 0;
        var completedScenes = 0;
        var hasWarnings = false;
        var currentFailureStage = StoryIntelligenceFailureStages.DocumentRead;

        try
        {
            await PublishAsync(
                run,
                "Import started",
                StoryIntelligenceRunStatuses.Running,
                StoryIntelligenceFailureStages.DocumentRead,
                "Story Intelligence has started reading this chapter.",
                stopwatch.ElapsedMilliseconds);

            if (string.IsNullOrWhiteSpace(run.SourceText))
            {
                throw new StoryIntelligenceRunFailureException(
                    StoryIntelligenceFailureStages.DocumentRead,
                    "Story Intelligence run has no source text.");
            }

            await entitlement.EnsureCanExecuteAsync(run.UserID);

            await ThrowIfCancellationRequestedAsync(run.StoryIntelligenceRunID, stopwatch.ElapsedMilliseconds, cancellationToken);
            var paragraphs = StoryIntelligenceParagraphs.Split(run.SourceText);
            if (paragraphs.Count == 0)
            {
                throw new StoryIntelligenceRunFailureException(
                    StoryIntelligenceFailureStages.DocumentRead,
                    "Story Intelligence run source text does not contain readable paragraphs.");
            }

            var chapterContextJson = BuildChapterContextJson(run);
            var chapterTemplate = await prompts.LoadPromptAsync(ChapterPromptFile, cancellationToken);
            var numberedChapterText = StoryIntelligenceParagraphs.Number(paragraphs);
            var chapterPrompt = BuildChapterPrompt(chapterTemplate, chapterContextJson, numberedChapterText);
            currentFailureStage = StoryIntelligenceFailureStages.ChapterStructure;

            await repository.UpdateProgressAsync(
                run.StoryIntelligenceRunID,
                currentStage: StoryIntelligenceFailureStages.ChapterStructure,
                currentMessage: "Running Chapter Structure analysis.",
                totalDurationMs: stopwatch.ElapsedMilliseconds);
            await PublishAsync(
                run,
                "Chapter started",
                StoryIntelligenceRunStatuses.Running,
                StoryIntelligenceFailureStages.ChapterStructure,
                "PlotDirector is finding the scene boundaries in this chapter.",
                stopwatch.ElapsedMilliseconds);

            var chapterClientResult = await client.ExecutePromptAsync(
                chapterPrompt,
                chapterPromptVersion,
                cancellationToken,
                stageModels.ChapterStructureModel);
            totals.Add(chapterClientResult, pricing);
            AiJsonParseResult<ChapterStructureModel> chapterAttempt;
            try
            {
                chapterAttempt = await ParseAiJsonWithRetryAsync<ChapterStructureModel>(
                    chapterPrompt,
                    chapterPromptVersion,
                    stageModels.ChapterStructureModel,
                    chapterClientResult,
                    totals,
                    "Chapter Structure",
                    cancellationToken);
            }
            catch (AiJsonParseException ex)
            {
                await repository.SaveChapterResultAsync(
                    run.StoryIntelligenceRunID,
                    new StoryIntelligenceChapterResultSaveRequest
                    {
                        ProjectID = run.ProjectID,
                        BookID = run.BookID,
                        ChapterID = run.ChapterID,
                        ChapterNumber = ChapterNumber(run),
                        SourceLabel = SourceLabel(run),
                        PromptVersion = chapterPromptVersion,
                        Model = stageModels.ChapterStructureModel,
                        RawResponseJson = ex.RawResponseJson,
                        OutputTextJson = ex.RawAssistantOutput,
                        ParsedJson = SerialiseFailedAiJsonParse(ex),
                        ValidationErrorsCount = 1,
                        ValidationWarningsCount = ex.RetryAttempted ? 1 : 0,
                        InputTokens = ex.InputTokens,
                        OutputTokens = ex.OutputTokens,
                        TotalTokens = SumTokens(ex.InputTokens, ex.OutputTokens),
                        DurationMs = Convert.ToInt64(ex.Duration.TotalMilliseconds)
                    });

                throw new StoryIntelligenceRunFailureException(
                    StoryIntelligenceFailureStages.Validation,
                    "Chapter Structure JSON could not be parsed after one retry.",
                    ex.ToString());
            }

            var parsedChapter = chapterAttempt.Parsed;
            var chapterNormalisation = ChapterStructureBoundaryNormaliser.Normalise(parsedChapter, paragraphs.Count);
            var chapterForProcessing = chapterNormalisation.ChapterStructure;
            var chapterValidation = chapterValidator.Validate(chapterForProcessing, paragraphs.Count);
            foreach (var warning in chapterAttempt.Warnings)
            {
                chapterValidation.Warnings.Add(new PlotLine.Models.StoryIntelligence.ValidationIssue
                {
                    Severity = "Warning",
                    Path = "$",
                    Message = warning,
                    SuggestedFix = "Review the raw and repaired Chapter Structure JSON."
                });
            }

            ChapterStructureBoundaryNormaliser.AddIssuesTo(chapterValidation, chapterNormalisation);
            hasWarnings = chapterValidation.Warnings.Count > 0;
            chapterResultId = await repository.SaveChapterResultAsync(
                run.StoryIntelligenceRunID,
                new StoryIntelligenceChapterResultSaveRequest
                {
                    ProjectID = run.ProjectID,
                    BookID = run.BookID,
                    ChapterID = run.ChapterID,
                    ChapterNumber = ChapterNumber(run),
                    SourceLabel = SourceLabel(run),
                    PromptVersion = chapterPromptVersion,
                    Model = chapterAttempt.Model,
                    RawResponseJson = chapterAttempt.RawResponseJson,
                    OutputTextJson = chapterAttempt.RawAssistantOutput,
                    ParsedJson = SerialiseChapterParseAudit(chapterForProcessing, chapterNormalisation, chapterAttempt),
                    ValidationErrorsCount = chapterValidation.Errors.Count,
                    ValidationWarningsCount = chapterValidation.Warnings.Count,
                    InputTokens = chapterAttempt.InputTokens,
                    OutputTokens = chapterAttempt.OutputTokens,
                    TotalTokens = SumTokens(chapterAttempt.InputTokens, chapterAttempt.OutputTokens),
                    DurationMs = Convert.ToInt64(chapterAttempt.Duration.TotalMilliseconds)
                });

            if (!chapterValidation.IsValid || chapterForProcessing?.SceneBoundaries is null)
            {
                throw new StoryIntelligenceRunFailureException(
                    StoryIntelligenceFailureStages.ChapterStructure,
                    "Chapter Structure validation failed.",
                    BuildValidationDetail(chapterValidation));
            }

            var sceneBlocks = chapterForProcessing.SceneBoundaries
                .Select(boundary => BuildSceneBlock(run, paragraphs, boundary))
                .ToList();
            failedScenes += sceneBlocks.Count(block => !block.SplitValid);
            currentFailureStage = StoryIntelligenceFailureStages.SceneSplit;

            await repository.UpdateProgressAsync(
                run.StoryIntelligenceRunID,
                currentStage: StoryIntelligenceFailureStages.SceneSplit,
                currentMessage: $"Detected {sceneBlocks.Count:N0} suggested scene(s).",
                totalDetectedScenes: sceneBlocks.Count,
                failedScenes: failedScenes,
                totalInputTokens: totals.InputTokens,
                totalOutputTokens: totals.OutputTokens,
                totalTokens: totals.TotalTokens,
                totalDurationMs: stopwatch.ElapsedMilliseconds,
                estimatedCostUSD: totals.EstimatedCostUSD,
                estimatedCostGBP: totals.EstimatedCostGBP);
            await PublishAsync(
                run,
                "Scene boundaries detected",
                StoryIntelligenceRunStatuses.Running,
                StoryIntelligenceFailureStages.SceneSplit,
                $"Detected {sceneBlocks.Count:N0} suggested scene(s).",
                stopwatch.ElapsedMilliseconds,
                totalDetectedScenes: sceneBlocks.Count,
                completedScenes: completedScenes,
                failedScenes: failedScenes,
                totalTokens: totals.TotalTokens);

            var sceneTemplate = await prompts.LoadPromptAsync(ScenePromptFile, cancellationToken);
            foreach (var block in sceneBlocks)
            {
                await ThrowIfCancellationRequestedAsync(run.StoryIntelligenceRunID, stopwatch.ElapsedMilliseconds, cancellationToken);
                if (!block.SplitValid)
                {
                    await PersistFailedSceneAsync(run, chapterResultId, block, scenePromptVersion, "Scene boundary range is invalid for the submitted chapter paragraphs.");
                    await PublishAsync(
                        run,
                        "Scene analysis failed",
                        StoryIntelligenceRunStatuses.Running,
                        StoryIntelligenceFailureStages.SceneIntelligence,
                        $"Suggested scene {block.TemporarySceneNumber:N0} could not be analysed because its paragraph range was invalid.",
                        stopwatch.ElapsedMilliseconds,
                        totalDetectedScenes: sceneBlocks.Count,
                        completedScenes: completedScenes,
                        failedScenes: failedScenes,
                        totalTokens: totals.TotalTokens);
                    continue;
                }

                await repository.UpdateProgressAsync(
                    run.StoryIntelligenceRunID,
                    currentStage: StoryIntelligenceFailureStages.SceneIntelligence,
                    currentMessage: $"Running Scene Intelligence for suggested scene {block.TemporarySceneNumber:N0}.",
                    completedScenes: completedScenes,
                    failedScenes: failedScenes,
                    totalDurationMs: stopwatch.ElapsedMilliseconds);
                await PublishAsync(
                    run,
                    "Scene analysis started",
                    StoryIntelligenceRunStatuses.Running,
                    StoryIntelligenceFailureStages.SceneIntelligence,
                    $"Reading scene {block.TemporarySceneNumber:N0}.",
                    stopwatch.ElapsedMilliseconds,
                    totalDetectedScenes: sceneBlocks.Count,
                    completedScenes: completedScenes,
                    failedScenes: failedScenes,
                    totalTokens: totals.TotalTokens,
                    currentSceneNumber: block.TemporarySceneNumber);

                try
                {
                    currentFailureStage = StoryIntelligenceFailureStages.SceneIntelligence;
                    var completedScenePrompt = scenePromptBuilder.BuildPrompt(sceneTemplate, block.SceneContextJson, block.SceneText);
                    var sceneClientResult = await client.ExecutePromptAsync(
                        completedScenePrompt,
                        scenePromptVersion,
                        cancellationToken,
                        stageModels.SceneIntelligenceModel,
                        SceneIntelligenceMaxOutputTokens());
                    totals.Add(sceneClientResult, pricing);
                    var sceneAttempt = await ParseAiJsonWithRetryAsync<SceneIntelligenceScene>(
                        completedScenePrompt,
                        scenePromptVersion,
                        stageModels.SceneIntelligenceModel,
                        sceneClientResult,
                        totals,
                        "Scene Intelligence",
                        cancellationToken);
                    var validation = sceneValidator.Validate(sceneAttempt.Parsed);
                    foreach (var warning in sceneAttempt.Warnings)
                    {
                        validation.Warnings.Add(new PlotLine.Models.StoryIntelligence.ValidationIssue
                        {
                            Severity = "Warning",
                            Path = "$",
                            Message = warning,
                            SuggestedFix = "Review the raw and repaired Scene Intelligence JSON."
                        });
                    }

                    hasWarnings = hasWarnings || validation.Warnings.Count > 0;

                    var sceneResultId = await repository.SaveSceneResultAsync(
                        run.StoryIntelligenceRunID,
                        chapterResultId,
                        new StoryIntelligenceSceneResultSaveRequest
                        {
                            ProjectID = run.ProjectID,
                            BookID = run.BookID,
                            ChapterID = run.ChapterID,
                            SceneID = null,
                            TemporarySceneNumber = block.TemporarySceneNumber,
                            StartParagraph = block.StartParagraph,
                            EndParagraph = block.EndParagraph,
                            SourceLabel = block.SourceLabel,
                            PromptVersion = scenePromptVersion,
                            Model = sceneClientResult.Model,
                            RawResponseJson = sceneAttempt.RawResponseJson,
                            OutputTextJson = sceneAttempt.RawAssistantOutput,
                            ParsedJson = SerialiseSceneParseAudit(sceneAttempt, validation.IsValid),
                            ValidationErrorsCount = validation.Errors.Count,
                            ValidationWarningsCount = validation.Warnings.Count,
                            InputTokens = sceneAttempt.InputTokens,
                            OutputTokens = sceneAttempt.OutputTokens,
                            TotalTokens = SumTokens(sceneAttempt.InputTokens, sceneAttempt.OutputTokens),
                            DurationMs = Convert.ToInt64(sceneAttempt.Duration.TotalMilliseconds)
                        });
                    var importSessionId = run.BookID.HasValue ? (await pipelines.GetByBookAsync(run.BookID.Value))?.StoryIntelligenceBookPipelineID : null;
                    logger.LogInformation(
                        "Story Intelligence importer synchronisation: import session {ImportSessionId}; run {RunId}; current chapter {ChapterNumber}; current scene {SceneNumber}; scene result {SceneResultId}; snapshot version ImportPipeline; change token n/a; status analysed.",
                        importSessionId,
                        run.StoryIntelligenceRunID,
                        run.ChapterNumber,
                        block.TemporarySceneNumber,
                        sceneResultId);
                    if (importSessionId.HasValue)
                    {
                        try
                        {
                            var queueImageDemand = settings.AutomaticImageGenerationDuringImportEnabled
                                && !settings.RequireImageGenerationApproval
                                && settings.MaxImagesPerImport > 0;
                            await storyMemory.ProcessSceneResultAsync(
                                importSessionId.Value,
                                run.UserID,
                                sceneResultId,
                                new StoryMemoryResolveOptions
                                {
                                    QueueDemand = queueImageDemand,
                                    MaxDemandToQueue = settings.MaxImagesPerImport,
                                    MaxCharacterDemandToQueue = settings.MaxCharacterImagesPerImport,
                                    MaxLocationDemandToQueue = settings.MaxLocationImagesPerImport,
                                    MaxAssetDemandToQueue = settings.MaxAssetImagesPerImport
                                },
                                cancellationToken);
                        }
                        catch (Exception ex) when (ex is not OperationCanceledException && !IsFatal(ex))
                        {
                            logger.LogError(ex, "Story Memory processing failed for import session {ImportSessionId}, scene result {SceneResultId}.", importSessionId, sceneResultId);
                        }

                        try
                        {
                            var importSession = await pipelines.GetByIdForUserAsync(importSessionId.Value, run.UserID);
                            if (importSession is not null && sceneAttempt.Parsed is not null)
                            {
                                await characterIntelligence.UpdateProfilesForSceneAsync(
                                    importSession,
                                    sceneResultId,
                                    sceneAttempt.Parsed,
                                    cancellationToken);
                            }
                        }
                        catch (Exception ex) when (ex is not OperationCanceledException && !IsFatal(ex))
                        {
                            logger.LogError(ex, "Character Intelligence profile update failed for import session {ImportSessionId}, scene result {SceneResultId}.", importSessionId, sceneResultId);
                        }
                    }

                    completedScenes++;
                    await PublishAsync(
                        run,
                        "Scene analysis completed",
                        StoryIntelligenceRunStatuses.Running,
                        StoryIntelligenceFailureStages.SceneIntelligence,
                        $"Scene {block.TemporarySceneNumber:N0} analysed.",
                        stopwatch.ElapsedMilliseconds,
                        totalDetectedScenes: sceneBlocks.Count,
                        completedScenes: completedScenes,
                        failedScenes: failedScenes,
                        totalTokens: totals.TotalTokens,
                        currentSceneNumber: block.TemporarySceneNumber,
                        latestSceneSummary: SceneSummaryText(sceneAttempt.Parsed),
                        recentDiscoveries: BuildDiscoveries(sceneAttempt.Parsed));
                    if (!validation.IsValid || validation.Warnings.Count > 0)
                    {
                        failedScenes += validation.Errors.Count > 0 ? 1 : 0;
                    }
                }
                catch (Exception ex) when (ex is not OperationCanceledException && !IsFatal(ex))
                {
                    failedScenes++;
                    await PersistFailedSceneAsync(run, chapterResultId, block, scenePromptVersion, ex.Message, ex as AiJsonParseException);
                    await PublishAsync(
                        run,
                        "Scene analysis failed",
                        StoryIntelligenceRunStatuses.Running,
                        StoryIntelligenceFailureStages.SceneIntelligence,
                        $"Scene {block.TemporarySceneNumber:N0} needs attention.",
                        stopwatch.ElapsedMilliseconds,
                        totalDetectedScenes: sceneBlocks.Count,
                        completedScenes: completedScenes,
                        failedScenes: failedScenes,
                        totalTokens: totals.TotalTokens,
                        currentSceneNumber: block.TemporarySceneNumber,
                        errorMessage: ex.Message);
                    logger.LogWarning(ex, "Story Intelligence scene failed for run {StoryIntelligenceRunID}, scene {TemporarySceneNumber}.", run.StoryIntelligenceRunID, block.TemporarySceneNumber);
                }

                await repository.UpdateProgressAsync(
                    run.StoryIntelligenceRunID,
                    completedScenes: completedScenes,
                    failedScenes: failedScenes,
                    totalInputTokens: totals.InputTokens,
                    totalOutputTokens: totals.OutputTokens,
                    totalTokens: totals.TotalTokens,
                    totalDurationMs: stopwatch.ElapsedMilliseconds,
                    estimatedCostUSD: totals.EstimatedCostUSD,
                    estimatedCostGBP: totals.EstimatedCostGBP);
            }

            var finalStatus = failedScenes > 0 || hasWarnings
                ? StoryIntelligenceRunStatuses.CompletedWithWarnings
                : StoryIntelligenceRunStatuses.Completed;

            await repository.CompleteRunAsync(
                run.StoryIntelligenceRunID,
                finalStatus,
                totals.InputTokens,
                totals.OutputTokens,
                totals.TotalTokens,
                stopwatch.ElapsedMilliseconds,
                totals.EstimatedCostGBP,
                totals.EstimatedCostUSD);
            await PublishAsync(
                run,
                finalStatus == StoryIntelligenceRunStatuses.Completed ? "Import completed" : "Chapter completed",
                finalStatus,
                StoryIntelligenceRunStatuses.Completed,
                finalStatus == StoryIntelligenceRunStatuses.Completed
                    ? "Chapter analysis complete."
                    : "Chapter analysis complete with notes to review.",
                stopwatch.ElapsedMilliseconds,
                totalDetectedScenes: sceneBlocks.Count,
                completedScenes: completedScenes,
                failedScenes: failedScenes,
                totalTokens: totals.TotalTokens);
            await completionNotifications.NotifyIfImportTerminalAsync(run, cancellationToken);
        }
        catch (StoryIntelligenceRunCancelledException)
        {
            await repository.CancelRunAsync(run.StoryIntelligenceRunID, stopwatch.ElapsedMilliseconds);
            await PublishAsync(
                run,
                "Import cancelled",
                StoryIntelligenceRunStatuses.Cancelled,
                currentFailureStage,
                "Analysis was cancelled.",
                stopwatch.ElapsedMilliseconds);
        }
        catch (OperationCanceledException ex) when (!cancellationToken.IsCancellationRequested)
        {
            await repository.FailRunAsync(
                run.StoryIntelligenceRunID,
                currentFailureStage,
                "Story Intelligence request timed out or was cancelled before the run completed.",
                ex.ToString(),
                stopwatch.ElapsedMilliseconds);
            await PublishAsync(
                run,
                "Import failed",
                StoryIntelligenceRunStatuses.Failed,
                currentFailureStage,
                "Analysis could not finish.",
                stopwatch.ElapsedMilliseconds,
                errorMessage: ex.Message);
            await completionNotifications.NotifyIfImportTerminalAsync(run, cancellationToken);
        }
        catch (StoryIntelligenceRunFailureException ex)
        {
            await repository.FailRunAsync(run.StoryIntelligenceRunID, ex.FailureStage, ex.Message, ex.Detail, stopwatch.ElapsedMilliseconds);
            await PublishAsync(
                run,
                "Import failed",
                StoryIntelligenceRunStatuses.Failed,
                ex.FailureStage,
                "Analysis could not finish.",
                stopwatch.ElapsedMilliseconds,
                errorMessage: ex.Message);
            await completionNotifications.NotifyIfImportTerminalAsync(run, cancellationToken);
        }
        catch (Exception ex) when (ex is not OperationCanceledException)
        {
            await repository.FailRunAsync(
                run.StoryIntelligenceRunID,
                ClassifyFailureStage(ex),
                ex.Message,
                ex.ToString(),
                stopwatch.ElapsedMilliseconds);
            await PublishAsync(
                run,
                "Import failed",
                StoryIntelligenceRunStatuses.Failed,
                ClassifyFailureStage(ex),
                "Analysis could not finish.",
                stopwatch.ElapsedMilliseconds,
                errorMessage: ex.Message);
            await completionNotifications.NotifyIfImportTerminalAsync(run, cancellationToken);
        }
    }

    private async Task ThrowIfCancellationRequestedAsync(int runId, long elapsedMs, CancellationToken cancellationToken)
    {
        cancellationToken.ThrowIfCancellationRequested();
        var latest = await repository.GetRunAsync(runId);
        if (latest?.CancellationRequestedUtc.HasValue == true || string.Equals(latest?.Status, StoryIntelligenceRunStatuses.Cancelled, StringComparison.Ordinal))
        {
            await repository.CancelRunAsync(runId, elapsedMs);
            throw new StoryIntelligenceRunCancelledException();
        }
    }

    private Task PublishAsync(
        StoryIntelligenceQueuedRun run,
        string eventType,
        string status,
        string currentStage,
        string currentMessage,
        long elapsedMs,
        int? totalDetectedScenes = null,
        int? completedScenes = null,
        int? failedScenes = null,
        int? totalTokens = null,
        int? currentSceneNumber = null,
        string? latestSceneSummary = null,
        IReadOnlyList<string>? recentDiscoveries = null,
        string? errorMessage = null)
        => notifier.PublishAsync(new StoryIntelligenceRunProgressEvent
        {
            RunID = run.StoryIntelligenceRunID,
            UserID = run.UserID,
            ProjectID = run.ProjectID,
            BookID = run.BookID,
            ChapterID = run.ChapterID,
            ChapterNumber = run.ChapterNumber,
            Status = status,
            CurrentStage = currentStage,
            CurrentMessage = currentMessage,
            EventType = eventType,
            TotalDetectedScenes = totalDetectedScenes,
            CompletedScenes = completedScenes,
            FailedScenes = failedScenes,
            TotalTokens = totalTokens,
            TotalDurationMs = elapsedMs,
            CurrentSceneNumber = currentSceneNumber,
            ElapsedTime = FormatDuration(elapsedMs),
            EstimatedRemaining = EstimateRemaining(elapsedMs, completedScenes ?? 0, totalDetectedScenes ?? 0),
            LatestSceneSummary = latestSceneSummary,
            RecentDiscoveries = recentDiscoveries ?? [],
            ErrorMessage = errorMessage,
            UpdatedUtc = DateTime.UtcNow,
            IsActive = status is StoryIntelligenceRunStatuses.Pending or StoryIntelligenceRunStatuses.Running,
            IsCompleted = status is StoryIntelligenceRunStatuses.Completed or StoryIntelligenceRunStatuses.CompletedWithWarnings
        });

    private static string FormatDuration(long? milliseconds)
    {
        var elapsed = TimeSpan.FromMilliseconds(Math.Max(0, milliseconds ?? 0));
        if (elapsed.TotalMinutes >= 1)
        {
            return $"{Math.Max(1, (int)Math.Round(elapsed.TotalMinutes)):N0} min";
        }

        return $"{Math.Max(0, (int)Math.Round(elapsed.TotalSeconds)):N0} sec";
    }

    private static string? EstimateRemaining(long elapsedMs, int completedScenes, int totalScenes)
    {
        if (totalScenes > 0 && completedScenes >= totalScenes)
        {
            return "Nearly finished";
        }

        if (completedScenes <= 0 || totalScenes <= 0)
        {
            return "Calculating";
        }

        var remainingScenes = Math.Max(0, totalScenes - completedScenes);
        var averageSceneMs = elapsedMs / Math.Max(1, completedScenes);
        return FormatDuration(averageSceneMs * remainingScenes);
    }

    private static IReadOnlyList<string> BuildDiscoveries(SceneIntelligenceScene? scene)
    {
        if (scene is null)
        {
            return [];
        }

        var discoveries = new List<string>();
        discoveries.AddRange((scene.Characters ?? [])
            .Select(character => TrimOptional(character.Name, 120))
            .Where(name => !string.IsNullOrWhiteSpace(name))
            .Distinct(StringComparer.OrdinalIgnoreCase)
            .Take(3)
            .Select(name => $"Possible character found: {name}"));
        discoveries.AddRange((scene.Locations ?? [])
            .Select(location => TrimOptional(location.Name, 120))
            .Where(name => !string.IsNullOrWhiteSpace(name))
            .Distinct(StringComparer.OrdinalIgnoreCase)
            .Take(3)
            .Select(name => $"Possible location found: {name}"));
        discoveries.AddRange((scene.Assets ?? [])
            .Select(asset => TrimOptional(asset.Name, 120))
            .Where(name => !string.IsNullOrWhiteSpace(name))
            .Distinct(StringComparer.OrdinalIgnoreCase)
            .Take(2)
            .Select(name => $"Possible asset found: {name}"));
        discoveries.AddRange((scene.Relationships ?? [])
            .Select(relationship => BuildRelationshipDiscovery(relationship))
            .Where(text => !string.IsNullOrWhiteSpace(text))
            .OfType<string>()
            .Take(2));
        discoveries.AddRange((scene.TimelineClues ?? [])
            .Select(clue => TrimOptional(clue.Clue, 160))
            .Where(text => !string.IsNullOrWhiteSpace(text))
            .Take(2)
            .Select(text => $"Timeline clue found: {text}"));

        return discoveries
            .Distinct(StringComparer.OrdinalIgnoreCase)
            .Take(8)
            .ToList();
    }

    private static string? BuildRelationshipDiscovery(SceneIntelligenceRelationship relationship)
    {
        var characterA = TrimOptional(relationship.CharacterA, 80);
        var characterB = TrimOptional(relationship.CharacterB, 80);
        var signal = TrimOptional(relationship.RelationshipSignal, 120);
        if (string.IsNullOrWhiteSpace(characterA) || string.IsNullOrWhiteSpace(characterB))
        {
            return null;
        }

        return string.IsNullOrWhiteSpace(signal)
            ? $"Relationship signal found: {characterA} and {characterB}"
            : $"Relationship signal found: {characterA} and {characterB} - {signal}";
    }

    private static string? SceneSummaryText(SceneIntelligenceScene? scene)
        => TrimOptional(
            FirstConfigured(scene?.Summary?.Short, scene?.Summary?.Detailed),
            220);

    private static string? TrimOptional(string? value, int maxLength)
    {
        if (string.IsNullOrWhiteSpace(value))
        {
            return null;
        }

        var trimmed = value.Trim();
        return trimmed.Length <= maxLength ? trimmed : trimmed[..maxLength].TrimEnd();
    }

    private static string? FirstConfigured(params string?[] values)
        => values.FirstOrDefault(value => !string.IsNullOrWhiteSpace(value));

    private async Task<AiJsonParseResult<T>> ParseAiJsonWithRetryAsync<T>(
        string completedPrompt,
        string promptVersion,
        string model,
        StoryIntelligenceClientResult initialClientResult,
        TokenTotals totals,
        string label,
        CancellationToken cancellationToken)
    {
        var initialRawOutput = TryExtractOutputText(initialClientResult.RawResponseText, out var initialExtractError);
        var initialParse = StoryIntelligenceAiJsonParser.Parse<T>(initialRawOutput, JsonOptions);
        if (initialParse.Success)
        {
            return AiJsonParseResult<T>.Success(
                initialClientResult.Model,
                initialClientResult.RawResponseText,
                initialRawOutput,
                initialParse.RawJson,
                initialParse.RepairedJson,
                initialParse.Parsed,
                initialClientResult.InputTokens,
                initialClientResult.OutputTokens,
                initialClientResult.Duration,
                retryAttempted: false,
                warnings: initialParse.Warnings);
        }

        var initialError = initialExtractError ?? initialParse.ErrorMessage ?? $"{label} JSON could not be parsed.";
        var retryPrompt = BuildJsonRetryPrompt(completedPrompt, label, initialError, initialRawOutput);
        var retryClientResult = await client.ExecutePromptAsync(
            retryPrompt,
            promptVersion,
            cancellationToken,
            model,
            MaxOutputTokensForLabel(label));
        totals.Add(retryClientResult, pricing);

        var retryRawOutput = TryExtractOutputText(retryClientResult.RawResponseText, out var retryExtractError);
        var retryParse = StoryIntelligenceAiJsonParser.Parse<T>(retryRawOutput, JsonOptions);
        if (retryParse.Success)
        {
            var warnings = new List<string> { $"{label} JSON parse failed once and succeeded after one retry." };
            warnings.AddRange(initialParse.Warnings);
            warnings.AddRange(retryParse.Warnings);
            return AiJsonParseResult<T>.Success(
                retryClientResult.Model,
                CombineRawResponses(initialClientResult.RawResponseText, retryClientResult.RawResponseText),
                CombineAssistantOutputs(initialRawOutput, retryRawOutput),
                retryParse.RawJson,
                retryParse.RepairedJson,
                retryParse.Parsed,
                SumTokens(initialClientResult.InputTokens, retryClientResult.InputTokens),
                SumTokens(initialClientResult.OutputTokens, retryClientResult.OutputTokens),
                initialClientResult.Duration + retryClientResult.Duration,
                retryAttempted: true,
                warnings: warnings,
                initialError: initialError);
        }

        var retryError = retryExtractError ?? retryParse.ErrorMessage ?? $"{label} retry JSON could not be parsed.";
        throw new AiJsonParseException(
            $"{label} JSON could not be parsed after one retry.",
            CombineRawResponses(initialClientResult.RawResponseText, retryClientResult.RawResponseText),
            CombineAssistantOutputs(initialRawOutput, retryRawOutput),
            initialError,
            retryAttempted: true,
            retryError: retryError,
            inputTokens: SumTokens(initialClientResult.InputTokens, retryClientResult.InputTokens),
            outputTokens: SumTokens(initialClientResult.OutputTokens, retryClientResult.OutputTokens),
            duration: initialClientResult.Duration + retryClientResult.Duration);
    }

    private static string TryExtractOutputText(string rawResponseText, out string? error)
    {
        try
        {
            error = null;
            return ExtractOutputText(rawResponseText);
        }
        catch (JsonException ex)
        {
            error = ex.Message;
            return string.Empty;
        }
    }

    private async Task PersistFailedSceneAsync(
        StoryIntelligenceQueuedRun run,
        int? chapterResultId,
        StorySceneTextBlock block,
        string scenePromptVersion,
        string message,
        AiJsonParseException? parseException = null)
    {
        var sceneResultId = await repository.SaveSceneResultAsync(
            run.StoryIntelligenceRunID,
            chapterResultId,
            new StoryIntelligenceSceneResultSaveRequest
            {
                ProjectID = run.ProjectID,
                BookID = run.BookID,
                ChapterID = run.ChapterID,
                SceneID = null,
                TemporarySceneNumber = block.TemporarySceneNumber,
                StartParagraph = block.StartParagraph,
                EndParagraph = block.EndParagraph,
                SourceLabel = block.SourceLabel,
                PromptVersion = scenePromptVersion,
                Model = ResolveStageModels(run).SceneIntelligenceModel,
                RawResponseJson = parseException?.RawResponseJson ?? string.Empty,
                OutputTextJson = parseException?.RawAssistantOutput ?? string.Empty,
                ParsedJson = JsonSerializer.Serialize(new
                {
                    error = message,
                    parseError = parseException?.InitialError,
                    retryAttempted = parseException?.RetryAttempted ?? false,
                    retryError = parseException?.RetryError
                }, JsonOptions),
                ValidationErrorsCount = 1,
                ValidationWarningsCount = parseException?.RetryAttempted == true ? 1 : 0,
                InputTokens = parseException?.InputTokens,
                OutputTokens = parseException?.OutputTokens,
                TotalTokens = SumTokens(parseException?.InputTokens, parseException?.OutputTokens),
                DurationMs = parseException is null ? null : Convert.ToInt64(parseException.Duration.TotalMilliseconds)
            });
        var importSessionId = run.BookID.HasValue ? (await pipelines.GetByBookAsync(run.BookID.Value))?.StoryIntelligenceBookPipelineID : null;
        logger.LogInformation(
            "Story Intelligence importer synchronisation: import session {ImportSessionId}; run {RunId}; current chapter {ChapterNumber}; current scene {SceneNumber}; scene result {SceneResultId}; snapshot version ImportPipeline; change token n/a; status failed.",
            importSessionId,
            run.StoryIntelligenceRunID,
            run.ChapterNumber,
            block.TemporarySceneNumber,
            sceneResultId);
    }

    private StoryIntelligenceStageModels ResolveStageModels(StoryIntelligenceQueuedRun run)
        => StoryIntelligenceOptions.ParseModelSummary(
            run.Model,
            string.IsNullOrWhiteSpace(settings.EffectiveChapterStructureModel) ? settings.EffectiveModel : settings.EffectiveChapterStructureModel,
            string.IsNullOrWhiteSpace(settings.EffectiveSceneIntelligenceModel) ? settings.EffectiveModel : settings.EffectiveSceneIntelligenceModel);

    private int SceneIntelligenceMaxOutputTokens()
        => settings.SceneIntelligenceMaxOutputTokens.GetValueOrDefault(settings.MaxOutputTokens) > 0
            ? settings.SceneIntelligenceMaxOutputTokens.GetValueOrDefault(settings.MaxOutputTokens)
            : settings.MaxOutputTokens;

    private int MaxOutputTokensForLabel(string label)
        => label.Contains("Scene", StringComparison.OrdinalIgnoreCase)
            ? SceneIntelligenceMaxOutputTokens()
            : settings.MaxOutputTokens;

    private static StorySceneTextBlock BuildSceneBlock(StoryIntelligenceQueuedRun run, IReadOnlyList<string> paragraphs, ChapterSceneBoundary boundary)
    {
        var sceneNumber = boundary.SceneNumber ?? 0;
        var start = boundary.StartParagraph ?? 0;
        var end = boundary.EndParagraph ?? 0;
        var sourceLabel = $"{SourceLabel(run)}, suggested scene {sceneNumber}";

        if (sceneNumber <= 0 || start <= 0 || end <= 0 || start > end || end > paragraphs.Count)
        {
            return new StorySceneTextBlock(sceneNumber, start, end, sourceLabel, string.Empty, string.Empty, false);
        }

        var sceneText = string.Join(Environment.NewLine + Environment.NewLine, paragraphs.Skip(start - 1).Take(end - start + 1));
        return new StorySceneTextBlock(
            sceneNumber,
            start,
            end,
            sourceLabel,
            sceneText,
            BuildSceneContextJson(run, boundary),
            true);
    }

    private static string BuildChapterContextJson(StoryIntelligenceQueuedRun run)
        => JsonSerializer.Serialize(new
        {
            projectId = run.ProjectID,
            bookId = run.BookID,
            chapterId = run.ChapterID,
            chapterNumber = ChapterNumber(run),
            sourceLabel = SourceLabel(run),
            knownCharacters = KnownCharacters(run)
        }, JsonOptions);

    private static string BuildSceneContextJson(StoryIntelligenceQueuedRun run, ChapterSceneBoundary boundary)
        => JsonSerializer.Serialize(new
        {
            projectId = run.ProjectID,
            bookId = run.BookID,
            chapterId = run.ChapterID,
            sceneId = (int?)null,
            chapterNumber = ChapterNumber(run),
            sceneNumber = boundary.SceneNumber,
            sourceLabel = $"{SourceLabel(run)}, suggested scene {boundary.SceneNumber}",
            knownCharacters = KnownCharacters(run)
        }, JsonOptions);

    private static IReadOnlyList<StoryIntelligenceKnownCharacterContext> KnownCharacters(StoryIntelligenceQueuedRun run)
    {
        if (string.IsNullOrWhiteSpace(run.KnownCharactersJson))
        {
            return [];
        }

        try
        {
            return JsonSerializer.Deserialize<IReadOnlyList<StoryIntelligenceKnownCharacterContext>>(run.KnownCharactersJson, JsonOptions) ?? [];
        }
        catch (JsonException)
        {
            return [];
        }
    }

    private static string BuildChapterPrompt(string promptTemplate, string chapterContextJson, string chapterText)
        => promptTemplate
            .Replace("{{CHAPTER_CONTEXT_JSON}}", chapterContextJson ?? string.Empty, StringComparison.Ordinal)
            .Replace("{{CHAPTER_TEXT}}", chapterText ?? string.Empty, StringComparison.Ordinal);

    private static string ExtractOutputText(string rawResponseText)
    {
        var response = JsonSerializer.Deserialize<OpenAIResponseEnvelope>(rawResponseText, JsonOptions)
            ?? throw new JsonException("OpenAI response envelope was empty.");
        var outputText = response.Output?
            .SelectMany(item => item.Content ?? [])
            .FirstOrDefault(content => string.Equals(content.Type, "output_text", StringComparison.OrdinalIgnoreCase)
                && !string.IsNullOrWhiteSpace(content.Text))
            ?.Text;

        return string.IsNullOrWhiteSpace(outputText)
            ? throw new JsonException("OpenAI response did not contain output_text content.")
            : outputText.Trim();
    }

    private static string SerializeParsed<T>(T? value)
        => value is null ? string.Empty : JsonSerializer.Serialize(value, JsonOptions);

    private static string SerialiseChapterParseAudit(
        ChapterStructureModel? chapterStructure,
        ChapterStructureBoundaryNormalisationResult normalisation,
        AiJsonParseResult<ChapterStructureModel> parseResult)
    {
        if (!parseResult.RepairApplied && !parseResult.RetryAttempted && normalisation.Issues.Count == 0)
        {
            return SerializeParsed(chapterStructure);
        }

        return JsonSerializer.Serialize(new
        {
            chapterStructure?.SchemaVersion,
            chapterStructure?.ChapterSummary,
            chapterStructure?.SceneBoundaries,
            repairedJson = parseResult.RepairApplied ? parseResult.RepairedJson : null,
            repairApplied = parseResult.RepairApplied,
            retryAttempted = parseResult.RetryAttempted,
            initialParseError = parseResult.InitialError,
            jsonWarnings = parseResult.Warnings,
            normalisation = normalisation.Issues.Select(issue => new
            {
                issue.Severity,
                issue.Path,
                issue.Message,
                issue.SuggestedFix
            })
        }, JsonOptions);
    }

    private static string SerialiseSceneParseAudit(AiJsonParseResult<SceneIntelligenceScene> parseResult, bool validationPassed)
    {
        if (!parseResult.RepairApplied && !parseResult.RetryAttempted)
        {
            return SerializeParsed(parseResult.Parsed);
        }

        return JsonSerializer.Serialize(new
        {
            parsedScene = parseResult.Parsed,
            repairedJson = parseResult.RepairApplied ? parseResult.RepairedJson : null,
            repairApplied = parseResult.RepairApplied,
            retryAttempted = parseResult.RetryAttempted,
            initialParseError = parseResult.InitialError,
            validationPassed,
            warnings = parseResult.Warnings
        }, JsonOptions);
    }

    private static string SerialiseFailedAiJsonParse(AiJsonParseException parseException)
        => JsonSerializer.Serialize(new
        {
            error = parseException.Message,
            parseError = parseException.InitialError,
            retryAttempted = parseException.RetryAttempted,
            retryError = parseException.RetryError
        }, JsonOptions);

    private static string BuildJsonRetryPrompt(string originalPrompt, string label, string parseError, string malformedJson)
        => string.Concat(
            originalPrompt,
            Environment.NewLine,
            Environment.NewLine,
            "## JSON Repair Retry",
            Environment.NewLine,
            $"Your previous response was not valid JSON. Return the same {label} object as valid JSON only. Do not include markdown. Do not truncate. All numeric confidence values must be valid JSON numbers such as 0.9 or 0.0.",
            Environment.NewLine,
            "Parse error:",
            Environment.NewLine,
            parseError,
            Environment.NewLine,
            "Malformed JSON returned previously:",
            Environment.NewLine,
            "```json",
            Environment.NewLine,
            malformedJson,
            Environment.NewLine,
            "```");

    private static string CombineRawResponses(string initialRawResponse, string retryRawResponse)
        => JsonSerializer.Serialize(new
        {
            initialRawResponseJson = initialRawResponse,
            retryRawResponseJson = retryRawResponse
        }, JsonOptions);

    private static string CombineAssistantOutputs(string initialOutput, string retryOutput)
        => JsonSerializer.Serialize(new
        {
            initialAssistantOutput = initialOutput,
            retryAssistantOutput = retryOutput
        }, JsonOptions);

    private static int? SumTokens(int? inputTokens, int? outputTokens)
        => inputTokens.HasValue || outputTokens.HasValue ? (inputTokens ?? 0) + (outputTokens ?? 0) : null;

    private static string SourceLabel(StoryIntelligenceQueuedRun run)
        => string.IsNullOrWhiteSpace(run.SourceLabel) ? "Admin persisted Story Intelligence chapter" : run.SourceLabel.Trim();

    private static decimal ChapterNumber(StoryIntelligenceQueuedRun run)
        => run.ChapterNumber ?? 1;

    private static string BuildValidationDetail(PlotLine.Models.StoryIntelligence.ValidationResult validation)
        => string.Join(
            Environment.NewLine,
            validation.Errors.Concat(validation.Warnings).Select(issue => $"{issue.Severity} {issue.Path}: {issue.Message}"));

    private static bool IsFatal(Exception ex)
        => ex is InvalidOperationException invalid
            && (invalid.Message.Contains("API key", StringComparison.OrdinalIgnoreCase)
                || invalid.Message.Contains("model is not configured", StringComparison.OrdinalIgnoreCase)
                || invalid.Message.Contains("prompt", StringComparison.OrdinalIgnoreCase));

    private static string ClassifyFailureStage(Exception ex)
        => ex is FileNotFoundException
            ? StoryIntelligenceFailureStages.ChapterStructure
            : ex is JsonException
                ? StoryIntelligenceFailureStages.Validation
                : StoryIntelligenceFailureStages.Persistence;

    private sealed class StoryIntelligenceRunFailureException(string failureStage, string message, string? detail = null) : Exception(message)
    {
        public string FailureStage { get; } = failureStage;
        public string? Detail { get; } = detail;
    }

    private sealed class AiJsonParseException(
        string message,
        string rawResponseJson,
        string rawAssistantOutput,
        string initialError,
        bool retryAttempted,
        string? retryError,
        int? inputTokens,
        int? outputTokens,
        TimeSpan duration) : Exception(message)
    {
        public string RawResponseJson { get; } = rawResponseJson;
        public string RawAssistantOutput { get; } = rawAssistantOutput;
        public string InitialError { get; } = initialError;
        public bool RetryAttempted { get; } = retryAttempted;
        public string? RetryError { get; } = retryError;
        public int? InputTokens { get; } = inputTokens;
        public int? OutputTokens { get; } = outputTokens;
        public TimeSpan Duration { get; } = duration;
    }

    private sealed record AiJsonParseResult<T>(
        string Model,
        string RawResponseJson,
        string RawAssistantOutput,
        string RawJson,
        string? RepairedJson,
        T? Parsed,
        int? InputTokens,
        int? OutputTokens,
        TimeSpan Duration,
        bool RetryAttempted,
        IReadOnlyList<string> Warnings,
        string? InitialError)
    {
        public bool RepairApplied => Warnings.Any(warning =>
            warning.Contains("markdown", StringComparison.OrdinalIgnoreCase)
            || warning.Contains("Normalised", StringComparison.OrdinalIgnoreCase)
            || warning.Contains("Escaped", StringComparison.OrdinalIgnoreCase)
            || warning.Contains("Trimmed", StringComparison.OrdinalIgnoreCase));

        public static AiJsonParseResult<T> Success(
            string model,
            string rawResponseJson,
            string rawAssistantOutput,
            string rawJson,
            string? repairedJson,
            T? parsed,
            int? inputTokens,
            int? outputTokens,
            TimeSpan duration,
            bool retryAttempted,
            IReadOnlyList<string> warnings,
            string? initialError = null)
            => new(
                model,
                rawResponseJson,
                rawAssistantOutput,
                rawJson,
                repairedJson,
                parsed,
                inputTokens,
                outputTokens,
                duration,
                retryAttempted,
                warnings,
                initialError);
    }

    private sealed class StoryIntelligenceRunCancelledException : Exception
    {
    }

    private sealed record StorySceneTextBlock(
        int TemporarySceneNumber,
        int StartParagraph,
        int EndParagraph,
        string SourceLabel,
        string SceneText,
        string SceneContextJson,
        bool SplitValid);

    private sealed class TokenTotals
    {
        public int? InputTokens { get; private set; }
        public int? OutputTokens { get; private set; }
        public int? TotalTokens => InputTokens.HasValue || OutputTokens.HasValue ? (InputTokens ?? 0) + (OutputTokens ?? 0) : null;
        public decimal? EstimatedCostUSD { get; private set; }
        public decimal? EstimatedCostGBP { get; private set; }

        public void Add(StoryIntelligenceClientResult result, StoryIntelligencePricingOptions pricing)
        {
            InputTokens = SumNullable(InputTokens, result.InputTokens);
            OutputTokens = SumNullable(OutputTokens, result.OutputTokens);
            AddCost(result, pricing);
        }

        private static int? SumNullable(int? current, int? next)
            => current.HasValue || next.HasValue ? (current ?? 0) + (next ?? 0) : null;

        private void AddCost(StoryIntelligenceClientResult result, StoryIntelligencePricingOptions pricing)
        {
            var modelPricing = pricing.GetPricingForModel(result.Model);
            if (modelPricing?.InputPerMillionUsd is not decimal inputPrice
                || modelPricing.OutputPerMillionUsd is not decimal outputPrice
                || result.InputTokens is not int inputTokens
                || result.OutputTokens is not int outputTokens)
            {
                EstimatedCostUSD = null;
                EstimatedCostGBP = null;
                return;
            }

            var requestUsd = (inputTokens / 1_000_000m * inputPrice)
                + (outputTokens / 1_000_000m * outputPrice);
            EstimatedCostUSD = (EstimatedCostUSD ?? 0m) + requestUsd;
            EstimatedCostGBP = pricing.UsdToGbpRate.HasValue
                ? (EstimatedCostGBP ?? 0m) + (requestUsd * pricing.UsdToGbpRate.Value)
                : null;
        }
    }
}