PlotDirector/PlotLine/Services/StoryIntelligenceFullChapterTestService.cs

using System.Text;
using System.Text.RegularExpressions;
using Microsoft.AspNetCore.Http;
using Microsoft.Extensions.Options;
using PlotLine.Data;
using PlotLine.Models;

namespace PlotLine.Services;

public interface IStoryIntelligenceFullChapterTestService
{
    StoryIntelligenceFullChapterTestViewModel GetDefault();
    Task<StoryIntelligenceFullChapterTestViewModel> AnalyzeAsync(StoryIntelligenceFullChapterTestForm form, IFormFile? textFile);
    Task<int> QueueAsync(int userId, StoryIntelligenceFullChapterTestForm form, IFormFile? textFile);
}

public sealed class StoryIntelligenceFullChapterTestService(
    IStoryIntelligenceResultRepository runs,
    IStoryIntelligenceClient client,
    IOptions<StoryIntelligencePricingOptions> pricingOptions,
    ILogger<StoryIntelligenceFullChapterTestService> logger) : IStoryIntelligenceFullChapterTestService
{
    private const int LargeChapterCharacterThreshold = 40000;
    private const int VeryLargeChapterCharacterThreshold = 60000;
    private const int LowParagraphCountThreshold = 5;
    private const int MaxUploadBytes = 2 * 1024 * 1024;
    private const string PromptVersionsSummary = "Chapter=Chapter-Structure-Prompt-V1; Scene=Scene-Prompt-V1";
    private readonly StoryIntelligencePricingOptions pricing = pricingOptions.Value;

    public StoryIntelligenceFullChapterTestViewModel GetDefault()
        => new();

    public async Task<StoryIntelligenceFullChapterTestViewModel> AnalyzeAsync(StoryIntelligenceFullChapterTestForm form, IFormFile? textFile)
    {
        var source = await ReadSourceAsync(form, textFile);
        if (!string.IsNullOrWhiteSpace(source.ErrorMessage))
        {
            return new StoryIntelligenceFullChapterTestViewModel
            {
                Form = form,
                ErrorMessage = source.ErrorMessage
            };
        }

        return BuildViewModel(form, source, queued: false);
    }

    public async Task<int> QueueAsync(int userId, StoryIntelligenceFullChapterTestForm form, IFormFile? textFile)
    {
        var source = await ReadSourceAsync(form, textFile);
        if (!string.IsNullOrWhiteSpace(source.ErrorMessage))
        {
            throw new InvalidOperationException(source.ErrorMessage);
        }

        if (string.IsNullOrWhiteSpace(source.Text))
        {
            throw new InvalidOperationException("Paste chapter text or upload a .txt chapter file before queueing.");
        }

        var preflight = Analyze(source.Text);
        var clientStatus = client.GetConfigurationStatus();
        var runId = await runs.QueueAdminTextAsync(new StoryIntelligenceRunQueueRequest
        {
            UserID = userId,
            ProjectID = form.ProjectID,
            BookID = form.BookID,
            ChapterID = form.ChapterID,
            SourceType = "AdminFullChapterText",
            SourceLabel = CleanSourceLabel(form.SourceLabel, source.SourceFileName),
            ChapterText = source.Text,
            SourceFileName = source.SourceFileName,
            SourceFileSizeBytes = source.SourceFileSizeBytes,
            SourceWordCount = preflight.WordCount,
            SourceCharacterCount = preflight.CharacterCount,
            SourceParagraphCount = preflight.ParagraphCount,
            SourceChapterCount = 1,
            Model = clientStatus.Model,
            PromptVersionsSummary = PromptVersionsSummary
        });

        logger.LogInformation(
            "Queued full chapter Story Intelligence test run {StoryIntelligenceRunID}. UserID={UserID} ProjectID={ProjectID} BookID={BookID} ChapterID={ChapterID} CharacterCount={CharacterCount} WordCount={WordCount} ParagraphCount={ParagraphCount} EstimatedInputTokens={EstimatedInputTokens} SourceFileName={SourceFileName}",
            runId,
            userId,
            form.ProjectID,
            form.BookID,
            form.ChapterID,
            preflight.CharacterCount,
            preflight.WordCount,
            preflight.ParagraphCount,
            preflight.EstimatedInputTokens,
            source.SourceFileName);

        return runId;
    }

    private StoryIntelligenceFullChapterTestViewModel BuildViewModel(
        StoryIntelligenceFullChapterTestForm form,
        FullChapterSource source,
        bool queued)
        => new()
        {
            Form = new StoryIntelligenceFullChapterTestForm
            {
                SourceLabel = form.SourceLabel,
                ProjectID = form.ProjectID,
                BookID = form.BookID,
                ChapterID = form.ChapterID,
                ChapterText = queued ? string.Empty : source.Text
            },
            Preflight = string.IsNullOrWhiteSpace(source.Text) ? null : Analyze(source.Text),
            SourceFileName = source.SourceFileName,
            SourceFileSizeBytes = source.SourceFileSizeBytes
        };

    private StoryIntelligenceFullChapterPreflight Analyze(string text)
    {
        var characterCount = text.Length;
        var wordCount = CountWords(text);
        var paragraphCount = StoryIntelligenceParagraphs.Split(text).Count;
        var estimatedInputTokens = EstimateTokens(text);
        var estimatedPipelineInputTokens = estimatedInputTokens * 2;
        var warnings = BuildWarnings(text, characterCount, paragraphCount);

        return new StoryIntelligenceFullChapterPreflight
        {
            CharacterCount = characterCount,
            WordCount = wordCount,
            ParagraphCount = paragraphCount,
            EstimatedInputTokens = estimatedInputTokens,
            EstimatedPipelineInputTokens = estimatedPipelineInputTokens,
            EstimatedInputCostUSD = EstimateInputCost(estimatedPipelineInputTokens, pricing.InputPerMillionUsd),
            EstimatedInputCostGBP = EstimateInputCost(estimatedPipelineInputTokens, pricing.InputPerMillionUsd, pricing.UsdToGbpRate),
            Warnings = warnings
        };
    }

    private async Task<FullChapterSource> ReadSourceAsync(StoryIntelligenceFullChapterTestForm form, IFormFile? textFile)
    {
        if (textFile is not null && textFile.Length > 0)
        {
            if (!textFile.FileName.EndsWith(".txt", StringComparison.OrdinalIgnoreCase))
            {
                return FullChapterSource.Error("Only .txt uploads are supported for the full chapter test harness.");
            }

            if (textFile.Length > MaxUploadBytes)
            {
                return FullChapterSource.Error("The uploaded file is too large for this test harness. Use a .txt file up to 2 MB.");
            }

            await using var stream = textFile.OpenReadStream();
            using var reader = new StreamReader(stream, Encoding.UTF8, detectEncodingFromByteOrderMarks: true);
            var text = await reader.ReadToEndAsync();
            return new FullChapterSource(text, Path.GetFileName(textFile.FileName), textFile.Length, null);
        }

        return new FullChapterSource(form.ChapterText?.Trim() ?? string.Empty, null, null, null);
    }

    private static IReadOnlyList<string> BuildWarnings(string text, int characterCount, int paragraphCount)
    {
        var warnings = new List<string>();
        if (characterCount >= VeryLargeChapterCharacterThreshold)
        {
            warnings.Add("This is a very large chapter. Processing may take several minutes and may expose prompt, model, or timeout limits.");
        }
        else if (characterCount >= LargeChapterCharacterThreshold)
        {
            warnings.Add("This is a large chapter. Monitor the persisted run for timeout, validation, and scene split behaviour.");
        }

        if (paragraphCount > 0 && paragraphCount < LowParagraphCountThreshold)
        {
            warnings.Add("The paragraph count is very low. Chapter Structure depends on clear paragraph boundaries, so scene splitting may be poor.");
        }

        if (text.Any(ch => char.IsControl(ch) && ch is not '\r' and not '\n' and not '\t'))
        {
            warnings.Add("The text contains unusual control characters. Remove binary or corrupted content before queueing if possible.");
        }

        if (text.Contains('\uFFFD'))
        {
            warnings.Add("The text contains replacement characters, which may indicate an encoding problem.");
        }

        if (Regex.IsMatch(text, "<(html|body|script|style|table|div|span)\\b", RegexOptions.IgnoreCase))
        {
            warnings.Add("The text appears to contain HTML or markup. Plain manuscript text is safer for this test.");
        }

        var longestLine = text.Split(["\r\n", "\n", "\r"], StringSplitOptions.None).DefaultIfEmpty(string.Empty).Max(line => line.Length);
        if (longestLine > 8000)
        {
            warnings.Add("The text contains an unusually long line. Paragraph breaks may not have been preserved.");
        }

        return warnings;
    }

    private static string CleanSourceLabel(string? sourceLabel, string? fileName)
        => !string.IsNullOrWhiteSpace(sourceLabel)
            ? sourceLabel.Trim()
            : !string.IsNullOrWhiteSpace(fileName)
                ? $"Full chapter upload: {fileName}"
                : "Admin full chapter test";

    private static int CountWords(string value)
        => string.IsNullOrWhiteSpace(value)
            ? 0
            : value.Split((char[]?)null, StringSplitOptions.RemoveEmptyEntries).Length;

    private static int EstimateTokens(string value)
        => Math.Max(1, (int)Math.Ceiling(value.Length / 4m));

    private static decimal? EstimateInputCost(int tokens, decimal? inputPerMillionUsd, decimal? usdToGbp = null)
    {
        if (!inputPerMillionUsd.HasValue)
        {
            return null;
        }

        var usd = tokens / 1_000_000m * inputPerMillionUsd.Value;
        return usdToGbp.HasValue ? usd * usdToGbp.Value : usd;
    }

    private sealed record FullChapterSource(string Text, string? SourceFileName, long? SourceFileSizeBytes, string? ErrorMessage)
    {
        public static FullChapterSource Error(string message) => new(string.Empty, null, null, message);
    }
}