using System.Text; using System.Text.RegularExpressions; using Microsoft.AspNetCore.Http; using Microsoft.Extensions.Options; using PlotLine.Data; using PlotLine.Models; namespace PlotLine.Services; public interface IStoryIntelligenceFullChapterTestService { Task GetDefaultAsync(int? sourceRunId = null); Task AnalyzeAsync(StoryIntelligenceFullChapterTestForm form, IFormFile? textFile); Task QueueAsync(int userId, StoryIntelligenceFullChapterTestForm form, IFormFile? textFile); } public sealed class StoryIntelligenceFullChapterTestService( IStoryIntelligenceResultRepository runs, IOptions options, IOptions pricingOptions, ILogger logger) : IStoryIntelligenceFullChapterTestService { private const int LargeChapterCharacterThreshold = 40000; private const int VeryLargeChapterCharacterThreshold = 60000; private const int LowParagraphCountThreshold = 5; private const int MaxUploadBytes = 2 * 1024 * 1024; private const string PromptVersionsSummary = "Chapter=Chapter-Structure-Prompt-V1; Scene=Scene-Prompt-V1"; private static readonly IReadOnlyList AllowedModelOptions = ["gpt-5", "gpt-5-mini"]; private readonly StoryIntelligenceOptions settings = options.Value; private readonly StoryIntelligencePricingOptions pricing = pricingOptions.Value; public async Task GetDefaultAsync(int? sourceRunId = null) { if (sourceRunId is int runId) { var run = await runs.GetRunAsync(runId); if (run is not null && !string.IsNullOrWhiteSpace(run.SourceText)) { return BuildViewModel( new StoryIntelligenceFullChapterTestForm { SourceLabel = $"{CleanSourceLabel(run.SourceFileName, run.SourceFileName)} rerun", ProjectID = run.ProjectID, BookID = run.BookID, ChapterID = run.ChapterID, ChapterStructureModel = DefaultChapterStructureModel(), SceneIntelligenceModel = DefaultSceneIntelligenceModel(), ChapterText = run.SourceText }, new FullChapterSource(run.SourceText, run.SourceFileName, run.SourceFileSizeBytes, null), queued: false); } } return BuildViewModel(new StoryIntelligenceFullChapterTestForm(), new FullChapterSource(string.Empty, null, null, null), queued: false); } public async Task AnalyzeAsync(StoryIntelligenceFullChapterTestForm form, IFormFile? textFile) { var source = await ReadSourceAsync(form, textFile); if (!string.IsNullOrWhiteSpace(source.ErrorMessage)) { return new StoryIntelligenceFullChapterTestViewModel { Form = NormalizeForm(form, keepText: true), ModelOptions = AllowedModelOptions, ErrorMessage = source.ErrorMessage }; } return BuildViewModel(form, source, queued: false); } public async Task QueueAsync(int userId, StoryIntelligenceFullChapterTestForm form, IFormFile? textFile) { var source = await ReadSourceAsync(form, textFile); if (!string.IsNullOrWhiteSpace(source.ErrorMessage)) { throw new InvalidOperationException(source.ErrorMessage); } if (string.IsNullOrWhiteSpace(source.Text)) { throw new InvalidOperationException("Paste chapter text or upload a .txt chapter file before queueing."); } var modelSelection = NormalizeModelSelection(form); var preflight = Analyze(source.Text, modelSelection); var runId = await runs.QueueAdminTextAsync(new StoryIntelligenceRunQueueRequest { UserID = userId, ProjectID = form.ProjectID, BookID = form.BookID, ChapterID = form.ChapterID, SourceType = "AdminFullChapterText", SourceLabel = CleanSourceLabel(form.SourceLabel, source.SourceFileName), ChapterText = source.Text, SourceFileName = source.SourceFileName, SourceFileSizeBytes = source.SourceFileSizeBytes, SourceWordCount = preflight.WordCount, SourceCharacterCount = preflight.CharacterCount, SourceParagraphCount = preflight.ParagraphCount, SourceChapterCount = 1, Model = StoryIntelligenceOptions.BuildModelSummary(modelSelection.ChapterStructureModel, modelSelection.SceneIntelligenceModel), PromptVersionsSummary = PromptVersionsSummary }); logger.LogInformation( "Queued full chapter Story Intelligence test run {StoryIntelligenceRunID}. UserID={UserID} ProjectID={ProjectID} BookID={BookID} ChapterID={ChapterID} CharacterCount={CharacterCount} WordCount={WordCount} ParagraphCount={ParagraphCount} EstimatedInputTokens={EstimatedInputTokens} ChapterStructureModel={ChapterStructureModel} SceneIntelligenceModel={SceneIntelligenceModel} SourceFileName={SourceFileName}", runId, userId, form.ProjectID, form.BookID, form.ChapterID, preflight.CharacterCount, preflight.WordCount, preflight.ParagraphCount, preflight.EstimatedInputTokens, modelSelection.ChapterStructureModel, modelSelection.SceneIntelligenceModel, source.SourceFileName); return runId; } private StoryIntelligenceFullChapterTestViewModel BuildViewModel( StoryIntelligenceFullChapterTestForm form, FullChapterSource source, bool queued) => new() { Form = new StoryIntelligenceFullChapterTestForm { SourceLabel = form.SourceLabel, ProjectID = form.ProjectID, BookID = form.BookID, ChapterID = form.ChapterID, ChapterStructureModel = NormalizeModel(form.ChapterStructureModel, DefaultChapterStructureModel()), SceneIntelligenceModel = NormalizeModel(form.SceneIntelligenceModel, DefaultSceneIntelligenceModel()), ChapterText = queued ? string.Empty : source.Text }, ModelOptions = AllowedModelOptions, Preflight = string.IsNullOrWhiteSpace(source.Text) ? null : Analyze(source.Text, NormalizeModelSelection(form)), SourceFileName = source.SourceFileName, SourceFileSizeBytes = source.SourceFileSizeBytes }; private StoryIntelligenceFullChapterPreflight Analyze(string text, StoryIntelligenceStageModels modelSelection) { var characterCount = text.Length; var wordCount = CountWords(text); var paragraphCount = StoryIntelligenceParagraphs.Split(text).Count; var estimatedInputTokens = EstimateTokens(text); var estimatedPipelineInputTokens = estimatedInputTokens * 2; var warnings = BuildWarnings(text, characterCount, paragraphCount); return new StoryIntelligenceFullChapterPreflight { CharacterCount = characterCount, WordCount = wordCount, ParagraphCount = paragraphCount, EstimatedInputTokens = estimatedInputTokens, EstimatedPipelineInputTokens = estimatedPipelineInputTokens, EstimatedInputCostUSD = EstimateInputCost( estimatedInputTokens, pricing.GetPricingForModel(modelSelection.ChapterStructureModel)?.InputPerMillionUsd) + EstimateInputCost( estimatedInputTokens, pricing.GetPricingForModel(modelSelection.SceneIntelligenceModel)?.InputPerMillionUsd), EstimatedInputCostGBP = EstimateInputCost( estimatedInputTokens, pricing.GetPricingForModel(modelSelection.ChapterStructureModel)?.InputPerMillionUsd, pricing.UsdToGbpRate) + EstimateInputCost( estimatedInputTokens, pricing.GetPricingForModel(modelSelection.SceneIntelligenceModel)?.InputPerMillionUsd, pricing.UsdToGbpRate), Warnings = warnings }; } private StoryIntelligenceStageModels NormalizeModelSelection(StoryIntelligenceFullChapterTestForm form) => new( NormalizeModel(form.ChapterStructureModel, DefaultChapterStructureModel()), NormalizeModel(form.SceneIntelligenceModel, DefaultSceneIntelligenceModel())); private StoryIntelligenceFullChapterTestForm NormalizeForm(StoryIntelligenceFullChapterTestForm form, bool keepText) => new() { SourceLabel = form.SourceLabel, ProjectID = form.ProjectID, BookID = form.BookID, ChapterID = form.ChapterID, ChapterStructureModel = NormalizeModel(form.ChapterStructureModel, DefaultChapterStructureModel()), SceneIntelligenceModel = NormalizeModel(form.SceneIntelligenceModel, DefaultSceneIntelligenceModel()), ChapterText = keepText ? form.ChapterText : null }; private static string NormalizeModel(string? model, string fallback) { var value = string.IsNullOrWhiteSpace(model) ? fallback : model.Trim(); return AllowedModelOptions.Contains(value, StringComparer.OrdinalIgnoreCase) ? value : fallback; } private string DefaultChapterStructureModel() => string.IsNullOrWhiteSpace(settings.EffectiveChapterStructureModel) ? "gpt-5" : settings.EffectiveChapterStructureModel; private string DefaultSceneIntelligenceModel() => string.IsNullOrWhiteSpace(settings.EffectiveSceneIntelligenceModel) ? "gpt-5" : settings.EffectiveSceneIntelligenceModel; private async Task ReadSourceAsync(StoryIntelligenceFullChapterTestForm form, IFormFile? textFile) { if (textFile is not null && textFile.Length > 0) { if (!textFile.FileName.EndsWith(".txt", StringComparison.OrdinalIgnoreCase)) { return FullChapterSource.Error("Only .txt uploads are supported for the full chapter test harness."); } if (textFile.Length > MaxUploadBytes) { return FullChapterSource.Error("The uploaded file is too large for this test harness. Use a .txt file up to 2 MB."); } await using var stream = textFile.OpenReadStream(); using var reader = new StreamReader(stream, Encoding.UTF8, detectEncodingFromByteOrderMarks: true); var text = await reader.ReadToEndAsync(); return new FullChapterSource(text, Path.GetFileName(textFile.FileName), textFile.Length, null); } return new FullChapterSource(form.ChapterText?.Trim() ?? string.Empty, null, null, null); } private static IReadOnlyList BuildWarnings(string text, int characterCount, int paragraphCount) { var warnings = new List(); if (characterCount >= VeryLargeChapterCharacterThreshold) { warnings.Add("This is a very large chapter. Processing may take several minutes and may expose prompt, model, or timeout limits."); } else if (characterCount >= LargeChapterCharacterThreshold) { warnings.Add("This is a large chapter. Monitor the persisted run for timeout, validation, and scene split behaviour."); } if (paragraphCount > 0 && paragraphCount < LowParagraphCountThreshold) { warnings.Add("The paragraph count is very low. Chapter Structure depends on clear paragraph boundaries, so scene splitting may be poor."); } if (text.Any(ch => char.IsControl(ch) && ch is not '\r' and not '\n' and not '\t')) { warnings.Add("The text contains unusual control characters. Remove binary or corrupted content before queueing if possible."); } if (text.Contains('\uFFFD')) { warnings.Add("The text contains replacement characters, which may indicate an encoding problem."); } if (Regex.IsMatch(text, "<(html|body|script|style|table|div|span)\\b", RegexOptions.IgnoreCase)) { warnings.Add("The text appears to contain HTML or markup. Plain manuscript text is safer for this test."); } var longestLine = text.Split(["\r\n", "\n", "\r"], StringSplitOptions.None).DefaultIfEmpty(string.Empty).Max(line => line.Length); if (longestLine > 8000) { warnings.Add("The text contains an unusually long line. Paragraph breaks may not have been preserved."); } return warnings; } private static string CleanSourceLabel(string? sourceLabel, string? fileName) => !string.IsNullOrWhiteSpace(sourceLabel) ? sourceLabel.Trim() : !string.IsNullOrWhiteSpace(fileName) ? $"Full chapter upload: {fileName}" : "Admin full chapter test"; private static int CountWords(string value) => string.IsNullOrWhiteSpace(value) ? 0 : value.Split((char[]?)null, StringSplitOptions.RemoveEmptyEntries).Length; private static int EstimateTokens(string value) => Math.Max(1, (int)Math.Ceiling(value.Length / 4m)); private static decimal? EstimateInputCost(int tokens, decimal? inputPerMillionUsd, decimal? usdToGbp = null) { if (!inputPerMillionUsd.HasValue) { return null; } var usd = tokens / 1_000_000m * inputPerMillionUsd.Value; return usdToGbp.HasValue ? usd * usdToGbp.Value : usd; } private sealed record FullChapterSource(string Text, string? SourceFileName, long? SourceFileSizeBytes, string? ErrorMessage) { public static FullChapterSource Error(string message) => new(string.Empty, null, null, message); } }