diff --git a/PlotLine.Tests/PlotLine.Tests.csproj b/PlotLine.Tests/PlotLine.Tests.csproj new file mode 100644 index 0000000..ab92f91 --- /dev/null +++ b/PlotLine.Tests/PlotLine.Tests.csproj @@ -0,0 +1,14 @@ + + + + Exe + net10.0 + enable + enable + + + + + + + diff --git a/PlotLine.Tests/Program.cs b/PlotLine.Tests/Program.cs new file mode 100644 index 0000000..e9b1af0 --- /dev/null +++ b/PlotLine.Tests/Program.cs @@ -0,0 +1,122 @@ +using System.Text.Json; +using PlotLine.Models; +using PlotLine.Services; + +var tests = new (string Name, Action Test)[] +{ + ("Malformed confidence 0. is repaired to 0.0", RepairsTrailingDecimalConfidence), + ("Markdown fences are stripped", StripsMarkdownFence), + ("Unescaped backslash in evidence is repaired", RepairsUnescapedBackslash), + ("Truncated JSON is rejected", RejectsTruncatedJson), + ("Repaired JSON deserialises into SceneIntelligenceScene", RepairedJsonDeserialises), + ("Chapter Structure confidence 0. repairs and deserialises", ChapterStructureConfidenceRepairs), + ("Shared parser reports raw chapter output on unrecoverable JSON", SharedParserReportsRawOutput) +}; + +foreach (var test in tests) +{ + test.Test(); + Console.WriteLine($"PASS {test.Name}"); +} + +static void RepairsTrailingDecimalConfidence() +{ + var result = StoryIntelligenceJsonRepair.Repair("""{"observations":[{"confidence":0. }]}"""); + Assert(result.Success, result.ErrorMessage ?? "Repair failed."); + Assert(result.RepairedJson.Contains("\"confidence\":0.0", StringComparison.Ordinal), result.RepairedJson); +} + +static void StripsMarkdownFence() +{ + var result = StoryIntelligenceJsonRepair.Repair(""" + ```json + {"schemaVersion":"1.0"} + ``` + """); + Assert(result.Success, result.ErrorMessage ?? "Repair failed."); + Assert(result.RepairedJson == """{"schemaVersion":"1.0"}""", result.RepairedJson); +} + +static void RepairsUnescapedBackslash() +{ + var result = StoryIntelligenceJsonRepair.Repair("""{"evidence":"Mara reads C:\qnotes"}"""); + Assert(result.Success, result.ErrorMessage ?? "Repair failed."); + using var document = JsonDocument.Parse(result.RepairedJson); + var evidence = document.RootElement.GetProperty("evidence").GetString(); + Assert(evidence == @"Mara reads C:\qnotes", evidence ?? "Evidence was null."); +} + +static void RejectsTruncatedJson() +{ + var result = StoryIntelligenceJsonRepair.Repair("""{"observations":[{"confidence":"""); + Assert(!result.Success, "Truncated JSON should not be silently repaired."); +} + +static void RepairedJsonDeserialises() +{ + var result = StoryIntelligenceJsonRepair.Repair(""" + { + "schemaVersion": "1.0", + "sceneReference": { "sceneId": null }, + "summary": { "short": "A scene.", "detailed": "A scene.", "confidence": 0. }, + "observations": [] + } + """); + Assert(result.Success, result.ErrorMessage ?? "Repair failed."); + var parsed = JsonSerializer.Deserialize( + result.RepairedJson, + new JsonSerializerOptions + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + PropertyNameCaseInsensitive = true + }); + Assert(parsed is not null, "Scene JSON did not deserialise."); +} + +static void ChapterStructureConfidenceRepairs() +{ + var result = StoryIntelligenceAiJsonParser.Parse( + """ + { + "schemaVersion": "1.0", + "chapterSummary": "A chapter.", + "sceneBoundaries": [ + { + "sceneNumber": 1, + "startParagraph": 1, + "endParagraph": 2, + "confidence": 0. , + "reason": "Opening scene." + } + ] + } + """, + JsonOptions()); + + Assert(result.Success, result.ErrorMessage ?? "Chapter Structure parse failed."); + Assert(result.Parsed?.SceneBoundaries?[0].Confidence == 0.0m, "Confidence was not repaired to 0.0."); + Assert(result.Warnings.Any(warning => warning.Contains("$.sceneBoundaries[0].confidence", StringComparison.Ordinal)), "Repair warning did not include JSON path."); +} + +static void SharedParserReportsRawOutput() +{ + const string raw = """{"schemaVersion":"1.0","sceneBoundaries":["""; + var result = StoryIntelligenceAiJsonParser.Parse(raw, JsonOptions()); + Assert(!result.Success, "Unrecoverable JSON should fail."); + Assert(result.RawJson == raw, "Raw malformed output was not retained."); +} + +static JsonSerializerOptions JsonOptions() + => new() + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + PropertyNameCaseInsensitive = true + }; + +static void Assert(bool condition, string message) +{ + if (!condition) + { + throw new InvalidOperationException(message); + } +} diff --git a/PlotLine.slnx b/PlotLine.slnx index e9bbd91..45d5747 100644 --- a/PlotLine.slnx +++ b/PlotLine.slnx @@ -1,3 +1,4 @@ + diff --git a/PlotLine/Controllers/AdminController.cs b/PlotLine/Controllers/AdminController.cs index f7c021d..bee584e 100644 --- a/PlotLine/Controllers/AdminController.cs +++ b/PlotLine/Controllers/AdminController.cs @@ -135,9 +135,9 @@ public sealed class AdminController( } [HttpGet("story-intelligence-full-chapter-test")] - public IActionResult StoryIntelligenceFullChapterTest() + public async Task StoryIntelligenceFullChapterTest(int? sourceRunId) { - return View(fullChapterTest.GetDefault()); + return View(await fullChapterTest.GetDefaultAsync(sourceRunId)); } [HttpPost("story-intelligence-full-chapter-test")] diff --git a/PlotLine/Docs/AI/Chapter-Structure-Prompt-V1.md b/PlotLine/Docs/AI/Chapter-Structure-Prompt-V1.md index 37982a7..9ae859d 100644 --- a/PlotLine/Docs/AI/Chapter-Structure-Prompt-V1.md +++ b/PlotLine/Docs/AI/Chapter-Structure-Prompt-V1.md @@ -55,6 +55,10 @@ Never invent paragraph numbers. Never return an `endParagraph` greater than the highest paragraph number supplied. +Do not create a separate final scene for a closing line unless it corresponds to an actual supplied paragraph number. If the final beat belongs to the previous scene, include it in the previous scene. + +Do not create placeholder scenes or invalid ranges to "ensure coverage". If the previous scene already ends at the final supplied paragraph, stop there. + Scene boundaries must: - start at paragraph `1`; @@ -62,6 +66,8 @@ Scene boundaries must: - not overlap; - not leave gaps; - be contiguous and non-overlapping where possible; +- correspond to actual supplied paragraph numbers; +- never use a `startParagraph` greater than `endParagraph`; - use increasing `sceneNumber` values starting at `1`. ## Output Requirements diff --git a/PlotLine/Models/StoryIntelligenceModels.cs b/PlotLine/Models/StoryIntelligenceModels.cs index 1950e57..1711216 100644 --- a/PlotLine/Models/StoryIntelligenceModels.cs +++ b/PlotLine/Models/StoryIntelligenceModels.cs @@ -69,15 +69,99 @@ public sealed class StoryIntelligenceOptions { public string ApiKey { get; init; } = string.Empty; public string Model { get; init; } = string.Empty; + public string ChapterStructureModel { get; init; } = string.Empty; + public string SceneIntelligenceModel { get; init; } = string.Empty; public int MaxOutputTokens { get; init; } = 4000; + public int? SceneIntelligenceMaxOutputTokens { get; init; } public int TimeoutSeconds { get; init; } = 120; + + public string EffectiveModel => FirstConfigured(Model); + public string EffectiveChapterStructureModel => FirstConfigured(ChapterStructureModel, Model); + public string EffectiveSceneIntelligenceModel => FirstConfigured(SceneIntelligenceModel, Model); + + public static string BuildModelSummary(string? chapterStructureModel, string? sceneIntelligenceModel) + => $"Chapter={FirstConfigured(chapterStructureModel)}; Scene={FirstConfigured(sceneIntelligenceModel)}"; + + public static StoryIntelligenceStageModels ParseModelSummary( + string? modelSummary, + string defaultChapterStructureModel, + string defaultSceneIntelligenceModel) + { + var chapterModel = defaultChapterStructureModel; + var sceneModel = defaultSceneIntelligenceModel; + + if (!string.IsNullOrWhiteSpace(modelSummary)) + { + var segments = modelSummary.Split(';', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + var matchedStage = false; + foreach (var segment in segments) + { + var parts = segment.Split('=', 2, StringSplitOptions.TrimEntries); + if (parts.Length != 2 || string.IsNullOrWhiteSpace(parts[1])) + { + continue; + } + + if (string.Equals(parts[0], "Chapter", StringComparison.OrdinalIgnoreCase)) + { + chapterModel = parts[1]; + matchedStage = true; + } + else if (string.Equals(parts[0], "Scene", StringComparison.OrdinalIgnoreCase)) + { + sceneModel = parts[1]; + matchedStage = true; + } + } + + if (!matchedStage && !string.IsNullOrWhiteSpace(modelSummary)) + { + chapterModel = modelSummary.Trim(); + sceneModel = modelSummary.Trim(); + } + } + + return new StoryIntelligenceStageModels( + FirstConfigured(chapterModel, defaultChapterStructureModel), + FirstConfigured(sceneModel, defaultSceneIntelligenceModel)); + } + + private static string FirstConfigured(params string?[] values) + => values.FirstOrDefault(value => !string.IsNullOrWhiteSpace(value))?.Trim() ?? string.Empty; } +public sealed record StoryIntelligenceStageModels(string ChapterStructureModel, string SceneIntelligenceModel); + public sealed class StoryIntelligencePricingOptions { public decimal? InputPerMillionUsd { get; init; } public decimal? OutputPerMillionUsd { get; init; } public decimal? UsdToGbpRate { get; init; } + public Dictionary Models { get; init; } = new(StringComparer.OrdinalIgnoreCase); + + public StoryIntelligenceModelPricingOptions? GetPricingForModel(string? model) + { + if (!string.IsNullOrWhiteSpace(model) + && Models.TryGetValue(model.Trim(), out var modelPricing) + && (modelPricing.InputPerMillionUsd.HasValue || modelPricing.OutputPerMillionUsd.HasValue)) + { + return modelPricing; + } + + return InputPerMillionUsd.HasValue || OutputPerMillionUsd.HasValue + ? new StoryIntelligenceModelPricingOptions + { + InputPerMillionUsd = InputPerMillionUsd, + OutputPerMillionUsd = OutputPerMillionUsd + } + : null; + } +} + +public sealed class StoryIntelligenceModelPricingOptions +{ + public decimal? InputPerMillionUsd { get; init; } + public decimal? OutputPerMillionUsd { get; init; } } public sealed class StoryIntelligenceRun diff --git a/PlotLine/Models/StoryIntelligencePersistenceModels.cs b/PlotLine/Models/StoryIntelligencePersistenceModels.cs index 7919857..a8a0858 100644 --- a/PlotLine/Models/StoryIntelligencePersistenceModels.cs +++ b/PlotLine/Models/StoryIntelligencePersistenceModels.cs @@ -280,6 +280,8 @@ public sealed class StoryIntelligenceFullChapterTestForm public int? ProjectID { get; init; } public int? BookID { get; init; } public int? ChapterID { get; init; } + public string? ChapterStructureModel { get; init; } + public string? SceneIntelligenceModel { get; init; } public string? ChapterText { get; init; } } @@ -298,6 +300,7 @@ public sealed class StoryIntelligenceFullChapterPreflight public sealed class StoryIntelligenceFullChapterTestViewModel { public StoryIntelligenceFullChapterTestForm Form { get; init; } = new(); + public IReadOnlyList ModelOptions { get; init; } = []; public StoryIntelligenceFullChapterPreflight? Preflight { get; init; } public string? SourceFileName { get; init; } public long? SourceFileSizeBytes { get; init; } diff --git a/PlotLine/Program.cs b/PlotLine/Program.cs index f329096..7525af0 100644 --- a/PlotLine/Program.cs +++ b/PlotLine/Program.cs @@ -41,7 +41,11 @@ public class Program builder.Services.Configure(builder.Configuration.GetSection("EmailSettings")); builder.Services.Configure(builder.Configuration.GetSection("Stripe")); builder.Services.Configure(builder.Configuration.GetSection("Storage")); - builder.Services.Configure(builder.Configuration.GetSection("OpenAI")); + builder.Services.Configure(options => + { + builder.Configuration.GetSection("OpenAI").Bind(options); + builder.Configuration.GetSection("StoryIntelligence").Bind(options); + }); builder.Services.Configure(builder.Configuration.GetSection("StoryIntelligence:Pricing")); builder.Services.AddDataProtection() .PersistKeysToFileSystem(new DirectoryInfo(Path.Combine(builder.Environment.ContentRootPath, "App_Data", "DataProtectionKeys"))); diff --git a/PlotLine/Services/ChapterStoryIntelligenceDryRunService.cs b/PlotLine/Services/ChapterStoryIntelligenceDryRunService.cs index 33d048a..7bb5692 100644 --- a/PlotLine/Services/ChapterStoryIntelligenceDryRunService.cs +++ b/PlotLine/Services/ChapterStoryIntelligenceDryRunService.cs @@ -52,13 +52,20 @@ public sealed class ChapterStoryIntelligenceDryRunService( var chapterTemplate = await prompts.LoadPromptAsync(ChapterPromptFile, cancellationToken); var numberedChapterText = StoryIntelligenceParagraphs.Number(paragraphs); var chapterPrompt = BuildChapterPrompt(chapterTemplate, chapterContextJson, numberedChapterText); - var chapterResult = await client.ExecutePromptAsync(chapterPrompt, chapterPromptVersion, cancellationToken); + var chapterResult = await client.ExecutePromptAsync( + chapterPrompt, + chapterPromptVersion, + cancellationToken, + clientStatus.ChapterStructureModel); var chapterJson = ExtractOutputText(chapterResult.RawResponseText); var parsedChapter = JsonSerializer.Deserialize(chapterJson, JsonOptions); - var chapterValidation = chapterValidator.Validate(parsedChapter, paragraphs.Count); + var chapterNormalisation = ChapterStructureBoundaryNormaliser.Normalise(parsedChapter, paragraphs.Count); + var chapterForProcessing = chapterNormalisation.ChapterStructure; + var chapterValidation = chapterValidator.Validate(chapterForProcessing, paragraphs.Count); + ChapterStructureBoundaryNormaliser.AddIssuesTo(chapterValidation, chapterNormalisation); var sceneBlocks = new List(); - if (!chapterValidation.IsValid || parsedChapter?.SceneBoundaries is null) + if (!chapterValidation.IsValid || chapterForProcessing?.SceneBoundaries is null) { totalStopwatch.Stop(); var stoppedResult = new ChapterStoryIntelligenceDryRunResultViewModel @@ -68,7 +75,9 @@ public sealed class ChapterStoryIntelligenceDryRunService( StopReason = "Chapter Structure validation failed. Scene Intelligence was not run.", ChapterPromptVersion = chapterPromptVersion, ScenePromptVersion = scenePromptVersion, - Model = chapterResult.Model, + Model = StoryIntelligenceOptions.BuildModelSummary( + chapterResult.Model, + clientStatus.SceneIntelligenceModel), ChapterContextJson = chapterContextJson, ParagraphCount = paragraphs.Count, TotalDuration = FormatDuration(totalStopwatch.Elapsed), @@ -82,7 +91,7 @@ public sealed class ChapterStoryIntelligenceDryRunService( ChapterOutputTokens = chapterResult.OutputTokens, ChapterRawResponseText = chapterResult.RawResponseText, ChapterJsonText = chapterJson, - ParsedChapter = parsedChapter, + ParsedChapter = chapterForProcessing, ChapterValidation = chapterValidation, SceneBlocks = sceneBlocks }; @@ -92,7 +101,7 @@ public sealed class ChapterStoryIntelligenceDryRunService( } var sceneTemplate = await prompts.LoadPromptAsync(ScenePromptFile, cancellationToken); - foreach (var boundary in parsedChapter.SceneBoundaries) + foreach (var boundary in chapterForProcessing.SceneBoundaries) { var block = BuildSceneBlock(form, paragraphs, boundary); if (!block.SplitValid) @@ -103,7 +112,13 @@ public sealed class ChapterStoryIntelligenceDryRunService( var sceneContextJson = BuildSceneContextJson(form, boundary); var completedScenePrompt = scenePromptBuilder.BuildPrompt(sceneTemplate, sceneContextJson, block.SceneText); - sceneBlocks.Add(await ExecuteSceneAsync(block, sceneContextJson, completedScenePrompt, scenePromptVersion, cancellationToken)); + sceneBlocks.Add(await ExecuteSceneAsync( + block, + sceneContextJson, + completedScenePrompt, + scenePromptVersion, + clientStatus.SceneIntelligenceModel, + cancellationToken)); } totalStopwatch.Stop(); @@ -113,7 +128,9 @@ public sealed class ChapterStoryIntelligenceDryRunService( PipelineStopped = false, ChapterPromptVersion = chapterPromptVersion, ScenePromptVersion = scenePromptVersion, - Model = chapterResult.Model, + Model = StoryIntelligenceOptions.BuildModelSummary( + chapterResult.Model, + clientStatus.SceneIntelligenceModel), ChapterContextJson = chapterContextJson, ParagraphCount = paragraphs.Count, TotalDuration = FormatDuration(totalStopwatch.Elapsed), @@ -127,7 +144,7 @@ public sealed class ChapterStoryIntelligenceDryRunService( ChapterOutputTokens = chapterResult.OutputTokens, ChapterRawResponseText = chapterResult.RawResponseText, ChapterJsonText = chapterJson, - ParsedChapter = parsedChapter, + ParsedChapter = chapterForProcessing, ChapterValidation = chapterValidation, SceneBlocks = sceneBlocks }; @@ -145,7 +162,9 @@ public sealed class ChapterStoryIntelligenceDryRunService( StopReason = "The combined pipeline could not complete.", ChapterPromptVersion = chapterPromptVersion, ScenePromptVersion = scenePromptVersion, - Model = clientStatus.Model, + Model = StoryIntelligenceOptions.BuildModelSummary( + clientStatus.ChapterStructureModel, + clientStatus.SceneIntelligenceModel), ChapterContextJson = chapterContextJson, ParagraphCount = paragraphs.Count, TotalDuration = FormatDuration(totalStopwatch.Elapsed), @@ -158,7 +177,9 @@ public sealed class ChapterStoryIntelligenceDryRunService( "Admin Chapter Story Intelligence pipeline failed. ChapterPromptVersion={ChapterPromptVersion} ScenePromptVersion={ScenePromptVersion} Model={Model} TotalDurationMs={TotalDurationMs} Success={Success}", chapterPromptVersion, scenePromptVersion, - clientStatus.Model, + StoryIntelligenceOptions.BuildModelSummary( + clientStatus.ChapterStructureModel, + clientStatus.SceneIntelligenceModel), totalStopwatch.ElapsedMilliseconds, false); } @@ -171,11 +192,16 @@ public sealed class ChapterStoryIntelligenceDryRunService( string sceneContextJson, string completedScenePrompt, string scenePromptVersion, + string sceneModel, CancellationToken cancellationToken) { try { - var result = await client.ExecutePromptAsync(completedScenePrompt, scenePromptVersion, cancellationToken); + var result = await client.ExecutePromptAsync( + completedScenePrompt, + scenePromptVersion, + cancellationToken, + sceneModel); var sceneJson = ExtractOutputText(result.RawResponseText); var parsedScene = JsonSerializer.Deserialize(sceneJson, JsonOptions); var validation = sceneValidator.Validate(parsedScene); diff --git a/PlotLine/Services/ChapterStructureBoundaryNormaliser.cs b/PlotLine/Services/ChapterStructureBoundaryNormaliser.cs new file mode 100644 index 0000000..0ba7fee --- /dev/null +++ b/PlotLine/Services/ChapterStructureBoundaryNormaliser.cs @@ -0,0 +1,164 @@ +using System.Text.Json; +using PlotLine.Models; +using PlotLine.Models.StoryIntelligence; + +namespace PlotLine.Services; + +public static class ChapterStructureBoundaryNormaliser +{ + public static ChapterStructureBoundaryNormalisationResult Normalise(ChapterStructureModel? chapterStructure, int paragraphCount) + { + if (chapterStructure?.SceneBoundaries is null || chapterStructure.SceneBoundaries.Count < 2 || paragraphCount <= 0) + { + return new ChapterStructureBoundaryNormalisationResult(chapterStructure, []); + } + + var boundaries = chapterStructure.SceneBoundaries; + var finalIndex = boundaries.Count - 1; + var final = boundaries[finalIndex]; + var previous = boundaries[finalIndex - 1]; + var phantomParagraph = paragraphCount + 1; + + if (IsTrailingEmptyBoundary(final, previous, paragraphCount, phantomParagraph) + || IsTrailingInvalidPlaceholder(final, previous, paragraphCount, phantomParagraph)) + { + var normalisedBoundaries = boundaries + .Take(finalIndex) + .Select((boundary, index) => CopyBoundary(boundary, index + 1)) + .ToList(); + + var normalised = new ChapterStructureModel + { + SchemaVersion = chapterStructure.SchemaVersion, + ChapterSummary = chapterStructure.ChapterSummary, + SceneBoundaries = normalisedBoundaries, + ExtensionData = chapterStructure.ExtensionData + }; + + var issue = new ValidationIssue + { + Severity = "Warning", + Path = $"sceneBoundaries[{finalIndex}]", + Message = BuildNormalisationMessage(final, finalIndex, paragraphCount, phantomParagraph), + SuggestedFix = "Do not return a final scene boundary unless it references an actual supplied paragraph." + }; + + return new ChapterStructureBoundaryNormalisationResult(normalised, [issue]); + } + + return new ChapterStructureBoundaryNormalisationResult(chapterStructure, []); + } + + public static void AddIssuesTo(ValidationResult validation, ChapterStructureBoundaryNormalisationResult normalisation) + { + foreach (var issue in normalisation.Issues) + { + validation.Warnings.Add(issue); + } + } + + public static string SerialiseParsedModel(ChapterStructureModel? chapterStructure, ChapterStructureBoundaryNormalisationResult normalisation, JsonSerializerOptions jsonOptions) + { + if (chapterStructure is null) + { + return string.Empty; + } + + if (normalisation.Issues.Count == 0) + { + return JsonSerializer.Serialize(chapterStructure, jsonOptions); + } + + return JsonSerializer.Serialize( + new + { + chapterStructure.SchemaVersion, + chapterStructure.ChapterSummary, + chapterStructure.SceneBoundaries, + Normalisation = normalisation.Issues.Select(issue => new + { + issue.Severity, + issue.Path, + issue.Message, + issue.SuggestedFix + }) + }, + jsonOptions); + } + + private static ChapterSceneBoundary CopyBoundary(ChapterSceneBoundary boundary, int sceneNumber) + => new() + { + SceneNumber = sceneNumber, + StartParagraph = boundary.StartParagraph, + EndParagraph = boundary.EndParagraph, + Confidence = boundary.Confidence, + Reason = boundary.Reason, + ExtensionData = boundary.ExtensionData + }; + + private static bool IsTrailingEmptyBoundary( + ChapterSceneBoundary final, + ChapterSceneBoundary previous, + int paragraphCount, + int phantomParagraph) + => final.StartParagraph == phantomParagraph + && final.EndParagraph == phantomParagraph + && previous.EndParagraph == paragraphCount + && final.StartParagraph > paragraphCount + && final.EndParagraph > paragraphCount; + + private static bool IsTrailingInvalidPlaceholder( + ChapterSceneBoundary final, + ChapterSceneBoundary previous, + int paragraphCount, + int phantomParagraph) + { + if (!final.StartParagraph.HasValue || !final.EndParagraph.HasValue || previous.EndParagraph != paragraphCount) + { + return false; + } + + var invalidTrailingRange = final.StartParagraph.Value > paragraphCount + && (final.EndParagraph.Value <= paragraphCount + || final.EndParagraph.Value == phantomParagraph + || final.EndParagraph.Value < final.StartParagraph.Value); + + return invalidTrailingRange && HasPlaceholderSignal(final); + } + + private static bool HasPlaceholderSignal(ChapterSceneBoundary boundary) + { + if (boundary.Confidence <= 0.05m) + { + return true; + } + + var reason = boundary.Reason ?? string.Empty; + return reason.Contains("invalid", StringComparison.OrdinalIgnoreCase) + || reason.Contains("placeholder", StringComparison.OrdinalIgnoreCase) + || reason.Contains("ensure coverage", StringComparison.OrdinalIgnoreCase) + || reason.Contains("coverage", StringComparison.OrdinalIgnoreCase); + } + + private static string BuildNormalisationMessage( + ChapterSceneBoundary final, + int finalIndex, + int paragraphCount, + int phantomParagraph) + { + var start = final.StartParagraph ?? 0; + var end = final.EndParagraph ?? 0; + if (start == phantomParagraph && end == phantomParagraph) + { + return $"Discarded trailing empty scene boundary {phantomParagraph}-{phantomParagraph} because submitted chapter has {paragraphCount} paragraphs and previous scene already ended at {paragraphCount}."; + } + + var sceneNumber = final.SceneNumber ?? finalIndex + 1; + return $"Discarded trailing invalid placeholder scene {sceneNumber} with range {start}-{end} because previous scene already ended at paragraph {paragraphCount}."; + } +} + +public sealed record ChapterStructureBoundaryNormalisationResult( + ChapterStructureModel? ChapterStructure, + IReadOnlyList Issues); diff --git a/PlotLine/Services/PersistedStoryIntelligenceRunner.cs b/PlotLine/Services/PersistedStoryIntelligenceRunner.cs index 2611ad4..8e55479 100644 --- a/PlotLine/Services/PersistedStoryIntelligenceRunner.cs +++ b/PlotLine/Services/PersistedStoryIntelligenceRunner.cs @@ -19,6 +19,7 @@ public sealed class PersistedStoryIntelligenceRunner( IStoryIntelligenceClient client, IChapterStructureValidator chapterValidator, IStorySceneValidator sceneValidator, + IOptions options, IOptions pricingOptions, ILogger logger) : IPersistedStoryIntelligenceRunner { @@ -31,6 +32,7 @@ public sealed class PersistedStoryIntelligenceRunner( WriteIndented = true }; + private readonly StoryIntelligenceOptions settings = options.Value; private readonly StoryIntelligencePricingOptions pricing = pricingOptions.Value; public async Task ProcessNextAsync(CancellationToken cancellationToken) @@ -50,6 +52,7 @@ public sealed class PersistedStoryIntelligenceRunner( var totals = new TokenTotals(); var chapterPromptVersion = versions.GetPromptVersion(ChapterPromptFile); var scenePromptVersion = versions.GetPromptVersion(ScenePromptFile); + var stageModels = ResolveStageModels(run); int? chapterResultId = null; var failedScenes = 0; var completedScenes = 0; @@ -86,11 +89,70 @@ public sealed class PersistedStoryIntelligenceRunner( currentMessage: "Running Chapter Structure analysis.", totalDurationMs: stopwatch.ElapsedMilliseconds); - var chapterClientResult = await client.ExecutePromptAsync(chapterPrompt, chapterPromptVersion, cancellationToken); - totals.Add(chapterClientResult); - var chapterJson = ExtractOutputText(chapterClientResult.RawResponseText); - var parsedChapter = JsonSerializer.Deserialize(chapterJson, JsonOptions); - var chapterValidation = chapterValidator.Validate(parsedChapter, paragraphs.Count); + var chapterClientResult = await client.ExecutePromptAsync( + chapterPrompt, + chapterPromptVersion, + cancellationToken, + stageModels.ChapterStructureModel); + totals.Add(chapterClientResult, pricing); + AiJsonParseResult chapterAttempt; + try + { + chapterAttempt = await ParseAiJsonWithRetryAsync( + chapterPrompt, + chapterPromptVersion, + stageModels.ChapterStructureModel, + chapterClientResult, + totals, + "Chapter Structure", + cancellationToken); + } + catch (AiJsonParseException ex) + { + await repository.SaveChapterResultAsync( + run.StoryIntelligenceRunID, + new StoryIntelligenceChapterResultSaveRequest + { + ProjectID = run.ProjectID, + BookID = run.BookID, + ChapterID = run.ChapterID, + ChapterNumber = ChapterNumber(run), + SourceLabel = SourceLabel(run), + PromptVersion = chapterPromptVersion, + Model = stageModels.ChapterStructureModel, + RawResponseJson = ex.RawResponseJson, + OutputTextJson = ex.RawAssistantOutput, + ParsedJson = SerialiseFailedAiJsonParse(ex), + ValidationErrorsCount = 1, + ValidationWarningsCount = ex.RetryAttempted ? 1 : 0, + InputTokens = ex.InputTokens, + OutputTokens = ex.OutputTokens, + TotalTokens = SumTokens(ex.InputTokens, ex.OutputTokens), + DurationMs = Convert.ToInt64(ex.Duration.TotalMilliseconds) + }); + + throw new StoryIntelligenceRunFailureException( + StoryIntelligenceFailureStages.Validation, + "Chapter Structure JSON could not be parsed after one retry.", + ex.ToString()); + } + + var parsedChapter = chapterAttempt.Parsed; + var chapterNormalisation = ChapterStructureBoundaryNormaliser.Normalise(parsedChapter, paragraphs.Count); + var chapterForProcessing = chapterNormalisation.ChapterStructure; + var chapterValidation = chapterValidator.Validate(chapterForProcessing, paragraphs.Count); + foreach (var warning in chapterAttempt.Warnings) + { + chapterValidation.Warnings.Add(new PlotLine.Models.StoryIntelligence.ValidationIssue + { + Severity = "Warning", + Path = "$", + Message = warning, + SuggestedFix = "Review the raw and repaired Chapter Structure JSON." + }); + } + + ChapterStructureBoundaryNormaliser.AddIssuesTo(chapterValidation, chapterNormalisation); hasWarnings = chapterValidation.Warnings.Count > 0; chapterResultId = await repository.SaveChapterResultAsync( run.StoryIntelligenceRunID, @@ -102,19 +164,19 @@ public sealed class PersistedStoryIntelligenceRunner( ChapterNumber = ChapterNumber(run), SourceLabel = SourceLabel(run), PromptVersion = chapterPromptVersion, - Model = chapterClientResult.Model, - RawResponseJson = chapterClientResult.RawResponseText, - OutputTextJson = chapterJson, - ParsedJson = SerializeParsed(parsedChapter), + Model = chapterAttempt.Model, + RawResponseJson = chapterAttempt.RawResponseJson, + OutputTextJson = chapterAttempt.RawAssistantOutput, + ParsedJson = SerialiseChapterParseAudit(chapterForProcessing, chapterNormalisation, chapterAttempt), ValidationErrorsCount = chapterValidation.Errors.Count, ValidationWarningsCount = chapterValidation.Warnings.Count, - InputTokens = chapterClientResult.InputTokens, - OutputTokens = chapterClientResult.OutputTokens, - TotalTokens = SumTokens(chapterClientResult.InputTokens, chapterClientResult.OutputTokens), - DurationMs = Convert.ToInt64(chapterClientResult.Duration.TotalMilliseconds) + InputTokens = chapterAttempt.InputTokens, + OutputTokens = chapterAttempt.OutputTokens, + TotalTokens = SumTokens(chapterAttempt.InputTokens, chapterAttempt.OutputTokens), + DurationMs = Convert.ToInt64(chapterAttempt.Duration.TotalMilliseconds) }); - if (!chapterValidation.IsValid || parsedChapter?.SceneBoundaries is null) + if (!chapterValidation.IsValid || chapterForProcessing?.SceneBoundaries is null) { throw new StoryIntelligenceRunFailureException( StoryIntelligenceFailureStages.ChapterStructure, @@ -122,7 +184,7 @@ public sealed class PersistedStoryIntelligenceRunner( BuildValidationDetail(chapterValidation)); } - var sceneBlocks = parsedChapter.SceneBoundaries + var sceneBlocks = chapterForProcessing.SceneBoundaries .Select(boundary => BuildSceneBlock(run, paragraphs, boundary)) .ToList(); failedScenes += sceneBlocks.Count(block => !block.SplitValid); @@ -138,8 +200,8 @@ public sealed class PersistedStoryIntelligenceRunner( totalOutputTokens: totals.OutputTokens, totalTokens: totals.TotalTokens, totalDurationMs: stopwatch.ElapsedMilliseconds, - estimatedCostUSD: EstimateUsd(totals), - estimatedCostGBP: EstimateGbp(totals)); + estimatedCostUSD: totals.EstimatedCostUSD, + estimatedCostGBP: totals.EstimatedCostGBP); var sceneTemplate = await prompts.LoadPromptAsync(ScenePromptFile, cancellationToken); foreach (var block in sceneBlocks) @@ -163,11 +225,33 @@ public sealed class PersistedStoryIntelligenceRunner( { currentFailureStage = StoryIntelligenceFailureStages.SceneIntelligence; var completedScenePrompt = scenePromptBuilder.BuildPrompt(sceneTemplate, block.SceneContextJson, block.SceneText); - var sceneClientResult = await client.ExecutePromptAsync(completedScenePrompt, scenePromptVersion, cancellationToken); - totals.Add(sceneClientResult); - var sceneJson = ExtractOutputText(sceneClientResult.RawResponseText); - var parsedScene = JsonSerializer.Deserialize(sceneJson, JsonOptions); - var validation = sceneValidator.Validate(parsedScene); + var sceneClientResult = await client.ExecutePromptAsync( + completedScenePrompt, + scenePromptVersion, + cancellationToken, + stageModels.SceneIntelligenceModel, + SceneIntelligenceMaxOutputTokens()); + totals.Add(sceneClientResult, pricing); + var sceneAttempt = await ParseAiJsonWithRetryAsync( + completedScenePrompt, + scenePromptVersion, + stageModels.SceneIntelligenceModel, + sceneClientResult, + totals, + "Scene Intelligence", + cancellationToken); + var validation = sceneValidator.Validate(sceneAttempt.Parsed); + foreach (var warning in sceneAttempt.Warnings) + { + validation.Warnings.Add(new PlotLine.Models.StoryIntelligence.ValidationIssue + { + Severity = "Warning", + Path = "$", + Message = warning, + SuggestedFix = "Review the raw and repaired Scene Intelligence JSON." + }); + } + hasWarnings = hasWarnings || validation.Warnings.Count > 0; await repository.SaveSceneResultAsync( @@ -185,15 +269,15 @@ public sealed class PersistedStoryIntelligenceRunner( SourceLabel = block.SourceLabel, PromptVersion = scenePromptVersion, Model = sceneClientResult.Model, - RawResponseJson = sceneClientResult.RawResponseText, - OutputTextJson = sceneJson, - ParsedJson = SerializeParsed(parsedScene), + RawResponseJson = sceneAttempt.RawResponseJson, + OutputTextJson = sceneAttempt.RawAssistantOutput, + ParsedJson = SerialiseSceneParseAudit(sceneAttempt, validation.IsValid), ValidationErrorsCount = validation.Errors.Count, ValidationWarningsCount = validation.Warnings.Count, - InputTokens = sceneClientResult.InputTokens, - OutputTokens = sceneClientResult.OutputTokens, - TotalTokens = SumTokens(sceneClientResult.InputTokens, sceneClientResult.OutputTokens), - DurationMs = Convert.ToInt64(sceneClientResult.Duration.TotalMilliseconds) + InputTokens = sceneAttempt.InputTokens, + OutputTokens = sceneAttempt.OutputTokens, + TotalTokens = SumTokens(sceneAttempt.InputTokens, sceneAttempt.OutputTokens), + DurationMs = Convert.ToInt64(sceneAttempt.Duration.TotalMilliseconds) }); completedScenes++; @@ -205,7 +289,7 @@ public sealed class PersistedStoryIntelligenceRunner( catch (Exception ex) when (ex is not OperationCanceledException && !IsFatal(ex)) { failedScenes++; - await PersistFailedSceneAsync(run, chapterResultId, block, scenePromptVersion, ex.Message); + await PersistFailedSceneAsync(run, chapterResultId, block, scenePromptVersion, ex.Message, ex as AiJsonParseException); logger.LogWarning(ex, "Story Intelligence scene failed for run {StoryIntelligenceRunID}, scene {TemporarySceneNumber}.", run.StoryIntelligenceRunID, block.TemporarySceneNumber); } @@ -217,8 +301,8 @@ public sealed class PersistedStoryIntelligenceRunner( totalOutputTokens: totals.OutputTokens, totalTokens: totals.TotalTokens, totalDurationMs: stopwatch.ElapsedMilliseconds, - estimatedCostUSD: EstimateUsd(totals), - estimatedCostGBP: EstimateGbp(totals)); + estimatedCostUSD: totals.EstimatedCostUSD, + estimatedCostGBP: totals.EstimatedCostGBP); } var finalStatus = failedScenes > 0 || hasWarnings @@ -232,8 +316,8 @@ public sealed class PersistedStoryIntelligenceRunner( totals.OutputTokens, totals.TotalTokens, stopwatch.ElapsedMilliseconds, - EstimateGbp(totals), - EstimateUsd(totals)); + totals.EstimatedCostGBP, + totals.EstimatedCostUSD); } catch (StoryIntelligenceRunCancelledException) { @@ -274,12 +358,99 @@ public sealed class PersistedStoryIntelligenceRunner( } } + private async Task> ParseAiJsonWithRetryAsync( + string completedPrompt, + string promptVersion, + string model, + StoryIntelligenceClientResult initialClientResult, + TokenTotals totals, + string label, + CancellationToken cancellationToken) + { + var initialRawOutput = TryExtractOutputText(initialClientResult.RawResponseText, out var initialExtractError); + var initialParse = StoryIntelligenceAiJsonParser.Parse(initialRawOutput, JsonOptions); + if (initialParse.Success) + { + return AiJsonParseResult.Success( + initialClientResult.Model, + initialClientResult.RawResponseText, + initialRawOutput, + initialParse.RawJson, + initialParse.RepairedJson, + initialParse.Parsed, + initialClientResult.InputTokens, + initialClientResult.OutputTokens, + initialClientResult.Duration, + retryAttempted: false, + warnings: initialParse.Warnings); + } + + var initialError = initialExtractError ?? initialParse.ErrorMessage ?? $"{label} JSON could not be parsed."; + var retryPrompt = BuildJsonRetryPrompt(completedPrompt, label, initialError, initialRawOutput); + var retryClientResult = await client.ExecutePromptAsync( + retryPrompt, + promptVersion, + cancellationToken, + model, + MaxOutputTokensForLabel(label)); + totals.Add(retryClientResult, pricing); + + var retryRawOutput = TryExtractOutputText(retryClientResult.RawResponseText, out var retryExtractError); + var retryParse = StoryIntelligenceAiJsonParser.Parse(retryRawOutput, JsonOptions); + if (retryParse.Success) + { + var warnings = new List { $"{label} JSON parse failed once and succeeded after one retry." }; + warnings.AddRange(initialParse.Warnings); + warnings.AddRange(retryParse.Warnings); + return AiJsonParseResult.Success( + retryClientResult.Model, + CombineRawResponses(initialClientResult.RawResponseText, retryClientResult.RawResponseText), + CombineAssistantOutputs(initialRawOutput, retryRawOutput), + retryParse.RawJson, + retryParse.RepairedJson, + retryParse.Parsed, + SumTokens(initialClientResult.InputTokens, retryClientResult.InputTokens), + SumTokens(initialClientResult.OutputTokens, retryClientResult.OutputTokens), + initialClientResult.Duration + retryClientResult.Duration, + retryAttempted: true, + warnings: warnings, + initialError: initialError); + } + + var retryError = retryExtractError ?? retryParse.ErrorMessage ?? $"{label} retry JSON could not be parsed."; + throw new AiJsonParseException( + $"{label} JSON could not be parsed after one retry.", + CombineRawResponses(initialClientResult.RawResponseText, retryClientResult.RawResponseText), + CombineAssistantOutputs(initialRawOutput, retryRawOutput), + initialError, + retryAttempted: true, + retryError: retryError, + inputTokens: SumTokens(initialClientResult.InputTokens, retryClientResult.InputTokens), + outputTokens: SumTokens(initialClientResult.OutputTokens, retryClientResult.OutputTokens), + duration: initialClientResult.Duration + retryClientResult.Duration); + } + + private static string TryExtractOutputText(string rawResponseText, out string? error) + { + try + { + error = null; + return ExtractOutputText(rawResponseText); + } + catch (JsonException ex) + { + error = ex.Message; + return string.Empty; + } + } + private async Task PersistFailedSceneAsync( StoryIntelligenceQueuedRun run, int? chapterResultId, StorySceneTextBlock block, string scenePromptVersion, - string message) + string message, + AiJsonParseException? parseException = null) { await repository.SaveSceneResultAsync( run.StoryIntelligenceRunID, @@ -295,32 +466,40 @@ public sealed class PersistedStoryIntelligenceRunner( EndParagraph = block.EndParagraph, SourceLabel = block.SourceLabel, PromptVersion = scenePromptVersion, - Model = run.Model, - RawResponseJson = string.Empty, - OutputTextJson = string.Empty, - ParsedJson = JsonSerializer.Serialize(new { error = message }, JsonOptions), + Model = ResolveStageModels(run).SceneIntelligenceModel, + RawResponseJson = parseException?.RawResponseJson ?? string.Empty, + OutputTextJson = parseException?.RawAssistantOutput ?? string.Empty, + ParsedJson = JsonSerializer.Serialize(new + { + error = message, + parseError = parseException?.InitialError, + retryAttempted = parseException?.RetryAttempted ?? false, + retryError = parseException?.RetryError + }, JsonOptions), ValidationErrorsCount = 1, - ValidationWarningsCount = 0 + ValidationWarningsCount = parseException?.RetryAttempted == true ? 1 : 0, + InputTokens = parseException?.InputTokens, + OutputTokens = parseException?.OutputTokens, + TotalTokens = SumTokens(parseException?.InputTokens, parseException?.OutputTokens), + DurationMs = parseException is null ? null : Convert.ToInt64(parseException.Duration.TotalMilliseconds) }); } - private decimal? EstimateUsd(TokenTotals totals) - { - if (!pricing.InputPerMillionUsd.HasValue || !pricing.OutputPerMillionUsd.HasValue) - { - return null; - } + private StoryIntelligenceStageModels ResolveStageModels(StoryIntelligenceQueuedRun run) + => StoryIntelligenceOptions.ParseModelSummary( + run.Model, + string.IsNullOrWhiteSpace(settings.EffectiveChapterStructureModel) ? settings.EffectiveModel : settings.EffectiveChapterStructureModel, + string.IsNullOrWhiteSpace(settings.EffectiveSceneIntelligenceModel) ? settings.EffectiveModel : settings.EffectiveSceneIntelligenceModel); - var inputCost = (totals.InputTokens ?? 0) / 1_000_000m * pricing.InputPerMillionUsd.Value; - var outputCost = (totals.OutputTokens ?? 0) / 1_000_000m * pricing.OutputPerMillionUsd.Value; - return inputCost + outputCost; - } + private int SceneIntelligenceMaxOutputTokens() + => settings.SceneIntelligenceMaxOutputTokens.GetValueOrDefault(settings.MaxOutputTokens) > 0 + ? settings.SceneIntelligenceMaxOutputTokens.GetValueOrDefault(settings.MaxOutputTokens) + : settings.MaxOutputTokens; - private decimal? EstimateGbp(TokenTotals totals) - { - var usd = EstimateUsd(totals); - return usd.HasValue && pricing.UsdToGbpRate.HasValue ? usd.Value * pricing.UsdToGbpRate.Value : null; - } + private int MaxOutputTokensForLabel(string label) + => label.Contains("Scene", StringComparison.OrdinalIgnoreCase) + ? SceneIntelligenceMaxOutputTokens() + : settings.MaxOutputTokens; private static StorySceneTextBlock BuildSceneBlock(StoryIntelligenceQueuedRun run, IReadOnlyList paragraphs, ChapterSceneBoundary boundary) { @@ -390,6 +569,99 @@ public sealed class PersistedStoryIntelligenceRunner( private static string SerializeParsed(T? value) => value is null ? string.Empty : JsonSerializer.Serialize(value, JsonOptions); + private static string SerialiseChapterParseAudit( + ChapterStructureModel? chapterStructure, + ChapterStructureBoundaryNormalisationResult normalisation, + AiJsonParseResult parseResult) + { + if (!parseResult.RepairApplied && !parseResult.RetryAttempted && normalisation.Issues.Count == 0) + { + return SerializeParsed(chapterStructure); + } + + return JsonSerializer.Serialize(new + { + chapterStructure?.SchemaVersion, + chapterStructure?.ChapterSummary, + chapterStructure?.SceneBoundaries, + repairedJson = parseResult.RepairApplied ? parseResult.RepairedJson : null, + repairApplied = parseResult.RepairApplied, + retryAttempted = parseResult.RetryAttempted, + initialParseError = parseResult.InitialError, + jsonWarnings = parseResult.Warnings, + normalisation = normalisation.Issues.Select(issue => new + { + issue.Severity, + issue.Path, + issue.Message, + issue.SuggestedFix + }) + }, JsonOptions); + } + + private static string SerialiseSceneParseAudit(AiJsonParseResult parseResult, bool validationPassed) + { + if (!parseResult.RepairApplied && !parseResult.RetryAttempted) + { + return SerializeParsed(parseResult.Parsed); + } + + return JsonSerializer.Serialize(new + { + parsedScene = parseResult.Parsed, + repairedJson = parseResult.RepairApplied ? parseResult.RepairedJson : null, + repairApplied = parseResult.RepairApplied, + retryAttempted = parseResult.RetryAttempted, + initialParseError = parseResult.InitialError, + validationPassed, + warnings = parseResult.Warnings + }, JsonOptions); + } + + private static string SerialiseFailedAiJsonParse(AiJsonParseException parseException) + => JsonSerializer.Serialize(new + { + error = parseException.Message, + parseError = parseException.InitialError, + retryAttempted = parseException.RetryAttempted, + retryError = parseException.RetryError + }, JsonOptions); + + private static string BuildJsonRetryPrompt(string originalPrompt, string label, string parseError, string malformedJson) + => string.Concat( + originalPrompt, + Environment.NewLine, + Environment.NewLine, + "## JSON Repair Retry", + Environment.NewLine, + $"Your previous response was not valid JSON. Return the same {label} object as valid JSON only. Do not include markdown. Do not truncate. All numeric confidence values must be valid JSON numbers such as 0.9 or 0.0.", + Environment.NewLine, + "Parse error:", + Environment.NewLine, + parseError, + Environment.NewLine, + "Malformed JSON returned previously:", + Environment.NewLine, + "```json", + Environment.NewLine, + malformedJson, + Environment.NewLine, + "```"); + + private static string CombineRawResponses(string initialRawResponse, string retryRawResponse) + => JsonSerializer.Serialize(new + { + initialRawResponseJson = initialRawResponse, + retryRawResponseJson = retryRawResponse + }, JsonOptions); + + private static string CombineAssistantOutputs(string initialOutput, string retryOutput) + => JsonSerializer.Serialize(new + { + initialAssistantOutput = initialOutput, + retryAssistantOutput = retryOutput + }, JsonOptions); + private static int? SumTokens(int? inputTokens, int? outputTokens) => inputTokens.HasValue || outputTokens.HasValue ? (inputTokens ?? 0) + (outputTokens ?? 0) : null; @@ -423,6 +695,75 @@ public sealed class PersistedStoryIntelligenceRunner( public string? Detail { get; } = detail; } + private sealed class AiJsonParseException( + string message, + string rawResponseJson, + string rawAssistantOutput, + string initialError, + bool retryAttempted, + string? retryError, + int? inputTokens, + int? outputTokens, + TimeSpan duration) : Exception(message) + { + public string RawResponseJson { get; } = rawResponseJson; + public string RawAssistantOutput { get; } = rawAssistantOutput; + public string InitialError { get; } = initialError; + public bool RetryAttempted { get; } = retryAttempted; + public string? RetryError { get; } = retryError; + public int? InputTokens { get; } = inputTokens; + public int? OutputTokens { get; } = outputTokens; + public TimeSpan Duration { get; } = duration; + } + + private sealed record AiJsonParseResult( + string Model, + string RawResponseJson, + string RawAssistantOutput, + string RawJson, + string? RepairedJson, + T? Parsed, + int? InputTokens, + int? OutputTokens, + TimeSpan Duration, + bool RetryAttempted, + IReadOnlyList Warnings, + string? InitialError) + { + public bool RepairApplied => Warnings.Any(warning => + warning.Contains("markdown", StringComparison.OrdinalIgnoreCase) + || warning.Contains("Normalised", StringComparison.OrdinalIgnoreCase) + || warning.Contains("Escaped", StringComparison.OrdinalIgnoreCase) + || warning.Contains("Trimmed", StringComparison.OrdinalIgnoreCase)); + + public static AiJsonParseResult Success( + string model, + string rawResponseJson, + string rawAssistantOutput, + string rawJson, + string? repairedJson, + T? parsed, + int? inputTokens, + int? outputTokens, + TimeSpan duration, + bool retryAttempted, + IReadOnlyList warnings, + string? initialError = null) + => new( + model, + rawResponseJson, + rawAssistantOutput, + rawJson, + repairedJson, + parsed, + inputTokens, + outputTokens, + duration, + retryAttempted, + warnings, + initialError); + } + private sealed class StoryIntelligenceRunCancelledException : Exception { } @@ -441,14 +782,38 @@ public sealed class PersistedStoryIntelligenceRunner( public int? InputTokens { get; private set; } public int? OutputTokens { get; private set; } public int? TotalTokens => InputTokens.HasValue || OutputTokens.HasValue ? (InputTokens ?? 0) + (OutputTokens ?? 0) : null; + public decimal? EstimatedCostUSD { get; private set; } + public decimal? EstimatedCostGBP { get; private set; } - public void Add(StoryIntelligenceClientResult result) + public void Add(StoryIntelligenceClientResult result, StoryIntelligencePricingOptions pricing) { InputTokens = SumNullable(InputTokens, result.InputTokens); OutputTokens = SumNullable(OutputTokens, result.OutputTokens); + AddCost(result, pricing); } private static int? SumNullable(int? current, int? next) => current.HasValue || next.HasValue ? (current ?? 0) + (next ?? 0) : null; + + private void AddCost(StoryIntelligenceClientResult result, StoryIntelligencePricingOptions pricing) + { + var modelPricing = pricing.GetPricingForModel(result.Model); + if (modelPricing?.InputPerMillionUsd is not decimal inputPrice + || modelPricing.OutputPerMillionUsd is not decimal outputPrice + || result.InputTokens is not int inputTokens + || result.OutputTokens is not int outputTokens) + { + EstimatedCostUSD = null; + EstimatedCostGBP = null; + return; + } + + var requestUsd = (inputTokens / 1_000_000m * inputPrice) + + (outputTokens / 1_000_000m * outputPrice); + EstimatedCostUSD = (EstimatedCostUSD ?? 0m) + requestUsd; + EstimatedCostGBP = pricing.UsdToGbpRate.HasValue + ? (EstimatedCostGBP ?? 0m) + (requestUsd * pricing.UsdToGbpRate.Value) + : null; + } } } diff --git a/PlotLine/Services/StoryIntelligenceExistingChapterQueueService.cs b/PlotLine/Services/StoryIntelligenceExistingChapterQueueService.cs index 0ad3acc..7072790 100644 --- a/PlotLine/Services/StoryIntelligenceExistingChapterQueueService.cs +++ b/PlotLine/Services/StoryIntelligenceExistingChapterQueueService.cs @@ -72,7 +72,9 @@ public sealed class StoryIntelligenceExistingChapterQueueService( SourceCharacterCount = selected.SourceCharacterCount ?? sourceText.Length, SourceParagraphCount = StoryIntelligenceParagraphs.Split(sourceText).Count, SourceChapterCount = 1, - Model = clientStatus.Model, + Model = StoryIntelligenceOptions.BuildModelSummary( + clientStatus.ChapterStructureModel, + clientStatus.SceneIntelligenceModel), PromptVersionsSummary = "Chapter=Chapter-Structure-Prompt-V1; Scene=Scene-Prompt-V1" }); diff --git a/PlotLine/Services/StoryIntelligenceFullChapterTestService.cs b/PlotLine/Services/StoryIntelligenceFullChapterTestService.cs index 8d49638..1f0ab2b 100644 --- a/PlotLine/Services/StoryIntelligenceFullChapterTestService.cs +++ b/PlotLine/Services/StoryIntelligenceFullChapterTestService.cs @@ -9,14 +9,14 @@ namespace PlotLine.Services; public interface IStoryIntelligenceFullChapterTestService { - StoryIntelligenceFullChapterTestViewModel GetDefault(); + Task GetDefaultAsync(int? sourceRunId = null); Task AnalyzeAsync(StoryIntelligenceFullChapterTestForm form, IFormFile? textFile); Task QueueAsync(int userId, StoryIntelligenceFullChapterTestForm form, IFormFile? textFile); } public sealed class StoryIntelligenceFullChapterTestService( IStoryIntelligenceResultRepository runs, - IStoryIntelligenceClient client, + IOptions options, IOptions pricingOptions, ILogger logger) : IStoryIntelligenceFullChapterTestService { @@ -25,10 +25,35 @@ public sealed class StoryIntelligenceFullChapterTestService( private const int LowParagraphCountThreshold = 5; private const int MaxUploadBytes = 2 * 1024 * 1024; private const string PromptVersionsSummary = "Chapter=Chapter-Structure-Prompt-V1; Scene=Scene-Prompt-V1"; + private static readonly IReadOnlyList AllowedModelOptions = ["gpt-5", "gpt-5-mini"]; + private readonly StoryIntelligenceOptions settings = options.Value; private readonly StoryIntelligencePricingOptions pricing = pricingOptions.Value; - public StoryIntelligenceFullChapterTestViewModel GetDefault() - => new(); + public async Task GetDefaultAsync(int? sourceRunId = null) + { + if (sourceRunId is int runId) + { + var run = await runs.GetRunAsync(runId); + if (run is not null && !string.IsNullOrWhiteSpace(run.SourceText)) + { + return BuildViewModel( + new StoryIntelligenceFullChapterTestForm + { + SourceLabel = $"{CleanSourceLabel(run.SourceFileName, run.SourceFileName)} rerun", + ProjectID = run.ProjectID, + BookID = run.BookID, + ChapterID = run.ChapterID, + ChapterStructureModel = DefaultChapterStructureModel(), + SceneIntelligenceModel = DefaultSceneIntelligenceModel(), + ChapterText = run.SourceText + }, + new FullChapterSource(run.SourceText, run.SourceFileName, run.SourceFileSizeBytes, null), + queued: false); + } + } + + return BuildViewModel(new StoryIntelligenceFullChapterTestForm(), new FullChapterSource(string.Empty, null, null, null), queued: false); + } public async Task AnalyzeAsync(StoryIntelligenceFullChapterTestForm form, IFormFile? textFile) { @@ -37,7 +62,8 @@ public sealed class StoryIntelligenceFullChapterTestService( { return new StoryIntelligenceFullChapterTestViewModel { - Form = form, + Form = NormalizeForm(form, keepText: true), + ModelOptions = AllowedModelOptions, ErrorMessage = source.ErrorMessage }; } @@ -58,8 +84,8 @@ public sealed class StoryIntelligenceFullChapterTestService( throw new InvalidOperationException("Paste chapter text or upload a .txt chapter file before queueing."); } - var preflight = Analyze(source.Text); - var clientStatus = client.GetConfigurationStatus(); + var modelSelection = NormalizeModelSelection(form); + var preflight = Analyze(source.Text, modelSelection); var runId = await runs.QueueAdminTextAsync(new StoryIntelligenceRunQueueRequest { UserID = userId, @@ -75,12 +101,12 @@ public sealed class StoryIntelligenceFullChapterTestService( SourceCharacterCount = preflight.CharacterCount, SourceParagraphCount = preflight.ParagraphCount, SourceChapterCount = 1, - Model = clientStatus.Model, + Model = StoryIntelligenceOptions.BuildModelSummary(modelSelection.ChapterStructureModel, modelSelection.SceneIntelligenceModel), PromptVersionsSummary = PromptVersionsSummary }); logger.LogInformation( - "Queued full chapter Story Intelligence test run {StoryIntelligenceRunID}. UserID={UserID} ProjectID={ProjectID} BookID={BookID} ChapterID={ChapterID} CharacterCount={CharacterCount} WordCount={WordCount} ParagraphCount={ParagraphCount} EstimatedInputTokens={EstimatedInputTokens} SourceFileName={SourceFileName}", + "Queued full chapter Story Intelligence test run {StoryIntelligenceRunID}. UserID={UserID} ProjectID={ProjectID} BookID={BookID} ChapterID={ChapterID} CharacterCount={CharacterCount} WordCount={WordCount} ParagraphCount={ParagraphCount} EstimatedInputTokens={EstimatedInputTokens} ChapterStructureModel={ChapterStructureModel} SceneIntelligenceModel={SceneIntelligenceModel} SourceFileName={SourceFileName}", runId, userId, form.ProjectID, @@ -90,6 +116,8 @@ public sealed class StoryIntelligenceFullChapterTestService( preflight.WordCount, preflight.ParagraphCount, preflight.EstimatedInputTokens, + modelSelection.ChapterStructureModel, + modelSelection.SceneIntelligenceModel, source.SourceFileName); return runId; @@ -107,14 +135,17 @@ public sealed class StoryIntelligenceFullChapterTestService( ProjectID = form.ProjectID, BookID = form.BookID, ChapterID = form.ChapterID, + ChapterStructureModel = NormalizeModel(form.ChapterStructureModel, DefaultChapterStructureModel()), + SceneIntelligenceModel = NormalizeModel(form.SceneIntelligenceModel, DefaultSceneIntelligenceModel()), ChapterText = queued ? string.Empty : source.Text }, - Preflight = string.IsNullOrWhiteSpace(source.Text) ? null : Analyze(source.Text), + ModelOptions = AllowedModelOptions, + Preflight = string.IsNullOrWhiteSpace(source.Text) ? null : Analyze(source.Text, NormalizeModelSelection(form)), SourceFileName = source.SourceFileName, SourceFileSizeBytes = source.SourceFileSizeBytes }; - private StoryIntelligenceFullChapterPreflight Analyze(string text) + private StoryIntelligenceFullChapterPreflight Analyze(string text, StoryIntelligenceStageModels modelSelection) { var characterCount = text.Length; var wordCount = CountWords(text); @@ -130,12 +161,53 @@ public sealed class StoryIntelligenceFullChapterTestService( ParagraphCount = paragraphCount, EstimatedInputTokens = estimatedInputTokens, EstimatedPipelineInputTokens = estimatedPipelineInputTokens, - EstimatedInputCostUSD = EstimateInputCost(estimatedPipelineInputTokens, pricing.InputPerMillionUsd), - EstimatedInputCostGBP = EstimateInputCost(estimatedPipelineInputTokens, pricing.InputPerMillionUsd, pricing.UsdToGbpRate), + EstimatedInputCostUSD = EstimateInputCost( + estimatedInputTokens, + pricing.GetPricingForModel(modelSelection.ChapterStructureModel)?.InputPerMillionUsd) + + EstimateInputCost( + estimatedInputTokens, + pricing.GetPricingForModel(modelSelection.SceneIntelligenceModel)?.InputPerMillionUsd), + EstimatedInputCostGBP = EstimateInputCost( + estimatedInputTokens, + pricing.GetPricingForModel(modelSelection.ChapterStructureModel)?.InputPerMillionUsd, + pricing.UsdToGbpRate) + + EstimateInputCost( + estimatedInputTokens, + pricing.GetPricingForModel(modelSelection.SceneIntelligenceModel)?.InputPerMillionUsd, + pricing.UsdToGbpRate), Warnings = warnings }; } + private StoryIntelligenceStageModels NormalizeModelSelection(StoryIntelligenceFullChapterTestForm form) + => new( + NormalizeModel(form.ChapterStructureModel, DefaultChapterStructureModel()), + NormalizeModel(form.SceneIntelligenceModel, DefaultSceneIntelligenceModel())); + + private StoryIntelligenceFullChapterTestForm NormalizeForm(StoryIntelligenceFullChapterTestForm form, bool keepText) + => new() + { + SourceLabel = form.SourceLabel, + ProjectID = form.ProjectID, + BookID = form.BookID, + ChapterID = form.ChapterID, + ChapterStructureModel = NormalizeModel(form.ChapterStructureModel, DefaultChapterStructureModel()), + SceneIntelligenceModel = NormalizeModel(form.SceneIntelligenceModel, DefaultSceneIntelligenceModel()), + ChapterText = keepText ? form.ChapterText : null + }; + + private static string NormalizeModel(string? model, string fallback) + { + var value = string.IsNullOrWhiteSpace(model) ? fallback : model.Trim(); + return AllowedModelOptions.Contains(value, StringComparer.OrdinalIgnoreCase) ? value : fallback; + } + + private string DefaultChapterStructureModel() + => string.IsNullOrWhiteSpace(settings.EffectiveChapterStructureModel) ? "gpt-5" : settings.EffectiveChapterStructureModel; + + private string DefaultSceneIntelligenceModel() + => string.IsNullOrWhiteSpace(settings.EffectiveSceneIntelligenceModel) ? "gpt-5" : settings.EffectiveSceneIntelligenceModel; + private async Task ReadSourceAsync(StoryIntelligenceFullChapterTestForm form, IFormFile? textFile) { if (textFile is not null && textFile.Length > 0) diff --git a/PlotLine/Services/StoryIntelligenceJsonRepair.cs b/PlotLine/Services/StoryIntelligenceJsonRepair.cs new file mode 100644 index 0000000..2948da0 --- /dev/null +++ b/PlotLine/Services/StoryIntelligenceJsonRepair.cs @@ -0,0 +1,324 @@ +using System.Text; +using System.Text.Json; +using System.Text.RegularExpressions; + +namespace PlotLine.Services; + +public static partial class StoryIntelligenceJsonRepair +{ + public static StoryIntelligenceJsonRepairResult Repair(string value) + { + var warnings = new List(); + var extraction = ExtractJsonObject(value); + if (!extraction.Success) + { + return StoryIntelligenceJsonRepairResult.Failed(value, extraction.ErrorMessage); + } + + warnings.AddRange(extraction.Warnings); + var numericRepaired = RepairTrailingDecimalNumbers(extraction.Json, warnings); + var slashRepaired = RepairInvalidBackslashesInStrings(numericRepaired, warnings); + + if (!IsValidJson(slashRepaired, out var error)) + { + return StoryIntelligenceJsonRepairResult.Failed(slashRepaired, error); + } + + return new StoryIntelligenceJsonRepairResult( + RepairedJson: slashRepaired, + RepairApplied: warnings.Count > 0, + Warnings: warnings, + ErrorMessage: null); + } + + public static StoryIntelligenceJsonExtractionResult ExtractJsonObject(string value) + { + var warnings = new List(); + if (string.IsNullOrWhiteSpace(value)) + { + return StoryIntelligenceJsonExtractionResult.Failed(value, "Output was empty."); + } + + var candidate = StripMarkdownFence(value.Trim(), warnings); + candidate = TrimToJsonObject(candidate, warnings); + if (string.IsNullOrWhiteSpace(candidate)) + { + return StoryIntelligenceJsonExtractionResult.Failed(value, "No complete JSON object was found."); + } + + return new StoryIntelligenceJsonExtractionResult( + Json: candidate, + Warnings: warnings, + ErrorMessage: null); + } + + private static string StripMarkdownFence(string value, List warnings) + { + var match = MarkdownFenceRegex().Match(value); + if (!match.Success) + { + return value; + } + + warnings.Add("Removed markdown code fence."); + return match.Groups["json"].Value.Trim(); + } + + private static string TrimToJsonObject(string value, List warnings) + { + var start = value.IndexOf('{'); + if (start < 0) + { + return string.Empty; + } + + var depth = 0; + var inString = false; + var escaped = false; + for (var i = start; i < value.Length; i++) + { + var ch = value[i]; + if (inString) + { + if (escaped) + { + escaped = false; + } + else if (ch == '\\') + { + escaped = true; + } + else if (ch == '"') + { + inString = false; + } + + continue; + } + + if (ch == '"') + { + inString = true; + } + else if (ch == '{') + { + depth++; + } + else if (ch == '}') + { + depth--; + if (depth == 0) + { + if (start > 0 || i < value.Length - 1) + { + warnings.Add("Trimmed text outside the JSON object."); + } + + return value[start..(i + 1)].Trim(); + } + } + } + + return string.Empty; + } + + private static string RepairTrailingDecimalNumbers(string value, List warnings) + { + var repaired = TrailingDecimalRegex().Replace(value, match => + { + warnings.Add("Normalised trailing decimal number to valid JSON number."); + return $"{match.Groups["prefix"].Value}{match.Groups["number"].Value}.0"; + }); + + return repaired; + } + + private static string RepairInvalidBackslashesInStrings(string value, List warnings) + { + var builder = new StringBuilder(value.Length); + var inString = false; + var changed = false; + + for (var i = 0; i < value.Length; i++) + { + var ch = value[i]; + if (!inString) + { + builder.Append(ch); + if (ch == '"') + { + inString = true; + } + + continue; + } + + if (ch == '"') + { + inString = false; + builder.Append(ch); + continue; + } + + if (ch == '\\') + { + var next = i + 1 < value.Length ? value[i + 1] : '\0'; + if (IsValidJsonEscape(next)) + { + builder.Append(ch); + if (next != '\0') + { + builder.Append(next); + i++; + } + + continue; + } + + builder.Append(@"\\"); + changed = true; + continue; + } + + builder.Append(ch); + } + + if (changed) + { + warnings.Add("Escaped invalid backslash inside string value."); + } + + return builder.ToString(); + } + + private static bool IsValidJsonEscape(char value) + => value is '"' or '\\' or '/' or 'b' or 'f' or 'n' or 'r' or 't' or 'u'; + + private static bool IsValidJson(string value, out string? error) + { + try + { + using var _ = JsonDocument.Parse(value); + error = null; + return true; + } + catch (JsonException ex) + { + error = ex.Message; + return false; + } + } + + [GeneratedRegex("^```(?:json)?\\s*(?.*?)\\s*```$", RegexOptions.Singleline | RegexOptions.IgnoreCase)] + private static partial Regex MarkdownFenceRegex(); + + [GeneratedRegex("(?:\\s*)(?-?\\d+)\\.\\s*(?=[,}\\]])")] + private static partial Regex TrailingDecimalRegex(); +} + +public static class StoryIntelligenceAiJsonParser +{ + public static StoryIntelligenceAiJsonParseResult Parse(string rawAssistantOutput, JsonSerializerOptions jsonOptions) + { + var extraction = StoryIntelligenceJsonRepair.ExtractJsonObject(rawAssistantOutput); + if (!extraction.Success) + { + return StoryIntelligenceAiJsonParseResult.Failed(rawAssistantOutput, extraction.ErrorMessage ?? "No complete JSON object was found."); + } + + try + { + var parsed = JsonSerializer.Deserialize(extraction.Json, jsonOptions); + return StoryIntelligenceAiJsonParseResult.Ok( + extraction.Json, + repairedJson: null, + parsed, + extraction.Warnings); + } + catch (JsonException strictEx) + { + var repair = StoryIntelligenceJsonRepair.Repair(rawAssistantOutput); + if (!repair.Success) + { + return StoryIntelligenceAiJsonParseResult.Failed(extraction.Json, repair.ErrorMessage ?? strictEx.Message); + } + + try + { + var parsed = JsonSerializer.Deserialize(repair.RepairedJson, jsonOptions); + var warnings = BuildRepairWarnings(repair.Warnings, strictEx.Path); + return StoryIntelligenceAiJsonParseResult.Ok( + extraction.Json, + repair.RepairedJson, + parsed, + warnings); + } + catch (JsonException repairedEx) + { + return StoryIntelligenceAiJsonParseResult.Failed(extraction.Json, repairedEx.Message); + } + } + } + + private static IReadOnlyList BuildRepairWarnings(IReadOnlyList repairWarnings, string? strictErrorPath) + { + var warnings = new List(); + foreach (var warning in repairWarnings) + { + if (warning.Contains("trailing decimal", StringComparison.OrdinalIgnoreCase)) + { + warnings.Add(string.IsNullOrWhiteSpace(strictErrorPath) + ? "Repaired malformed AI JSON numeric value." + : $"Repaired malformed AI JSON numeric value at {strictErrorPath}."); + } + else + { + warnings.Add(warning); + } + } + + return warnings; + } +} + +public sealed record StoryIntelligenceJsonRepairResult( + string RepairedJson, + bool RepairApplied, + IReadOnlyList Warnings, + string? ErrorMessage) +{ + public bool Success => string.IsNullOrWhiteSpace(ErrorMessage); + + public static StoryIntelligenceJsonRepairResult Failed(string value, string? errorMessage) + => new(value, false, [], errorMessage ?? "JSON repair failed."); +} + +public sealed record StoryIntelligenceJsonExtractionResult( + string Json, + IReadOnlyList Warnings, + string? ErrorMessage) +{ + public bool Success => string.IsNullOrWhiteSpace(ErrorMessage); + + public static StoryIntelligenceJsonExtractionResult Failed(string value, string? errorMessage) + => new(value, [], errorMessage ?? "JSON extraction failed."); +} + +public sealed record StoryIntelligenceAiJsonParseResult( + bool Success, + string RawJson, + string? RepairedJson, + T? Parsed, + IReadOnlyList Warnings, + string? ErrorMessage) +{ + public static StoryIntelligenceAiJsonParseResult Ok( + string rawJson, + string? repairedJson, + T? parsed, + IReadOnlyList warnings) + => new(true, rawJson, repairedJson, parsed, warnings, null); + + public static StoryIntelligenceAiJsonParseResult Failed(string rawJson, string errorMessage) + => new(false, rawJson, null, default, [], errorMessage); +} diff --git a/PlotLine/Services/StoryIntelligenceOpenAIInfrastructure.cs b/PlotLine/Services/StoryIntelligenceOpenAIInfrastructure.cs index 992d5bf..3cb6a78 100644 --- a/PlotLine/Services/StoryIntelligenceOpenAIInfrastructure.cs +++ b/PlotLine/Services/StoryIntelligenceOpenAIInfrastructure.cs @@ -28,7 +28,12 @@ public interface IStoryPromptVersionService public interface IStoryIntelligenceClient { - Task ExecutePromptAsync(string completedPrompt, string promptVersion, CancellationToken cancellationToken); + Task ExecutePromptAsync( + string completedPrompt, + string promptVersion, + CancellationToken cancellationToken, + string? modelOverride = null, + int? maxOutputTokensOverride = null); StoryIntelligenceClientConfigurationStatus GetConfigurationStatus(); } @@ -131,23 +136,34 @@ public sealed class StoryIntelligenceClient( => new() { ApiKeyConfigured = !string.IsNullOrWhiteSpace(settings.ApiKey), - Model = settings.Model, + Model = settings.EffectiveModel, + ChapterStructureModel = settings.EffectiveChapterStructureModel, + SceneIntelligenceModel = settings.EffectiveSceneIntelligenceModel, MaxOutputTokens = settings.MaxOutputTokens, TimeoutSeconds = settings.TimeoutSeconds, ClientConstructed = true, ConnectionStatus = "Not checked on diagnostics page. Only the admin dry-run page makes a live OpenAI request." }; - public async Task ExecutePromptAsync(string completedPrompt, string promptVersion, CancellationToken cancellationToken) + public async Task ExecutePromptAsync( + string completedPrompt, + string promptVersion, + CancellationToken cancellationToken, + string? modelOverride = null, + int? maxOutputTokensOverride = null) { if (string.IsNullOrWhiteSpace(settings.ApiKey)) { throw new InvalidOperationException("OpenAI API key is not configured. Add OpenAI:ApiKey to User Secrets or environment variables."); } - if (string.IsNullOrWhiteSpace(settings.Model)) + var model = string.IsNullOrWhiteSpace(modelOverride) + ? settings.EffectiveModel + : modelOverride.Trim(); + + if (string.IsNullOrWhiteSpace(model)) { - throw new InvalidOperationException("OpenAI model is not configured. Set OpenAI:Model in configuration."); + throw new InvalidOperationException("OpenAI model is not configured. Set OpenAI:Model or StoryIntelligence stage models in configuration."); } if (string.IsNullOrWhiteSpace(completedPrompt)) @@ -155,11 +171,17 @@ public sealed class StoryIntelligenceClient( throw new InvalidOperationException("Completed prompt is empty."); } + var maxOutputTokens = maxOutputTokensOverride.GetValueOrDefault(settings.MaxOutputTokens); + if (maxOutputTokens <= 0) + { + maxOutputTokens = settings.MaxOutputTokens; + } + var requestBody = JsonSerializer.Serialize(new { - model = settings.Model, + model = model, input = completedPrompt, - max_output_tokens = settings.MaxOutputTokens, + max_output_tokens = maxOutputTokens, store = false, reasoning = new { @@ -179,7 +201,7 @@ public sealed class StoryIntelligenceClient( logger.LogInformation( "Story Intelligence prompt executed. PromptVersion={PromptVersion} Model={Model} DurationMs={DurationMs} RetryCount={RetryCount} InputTokens={InputTokens} OutputTokens={OutputTokens} EstimatedCost={EstimatedCost} Success={Success}", promptVersion, - settings.Model, + model, stopwatch.ElapsedMilliseconds, retryCount, null, @@ -190,7 +212,7 @@ public sealed class StoryIntelligenceClient( return new StoryIntelligenceClientResult { RawResponseText = responseText, - Model = settings.Model, + Model = model, Duration = stopwatch.Elapsed, RetryCount = retryCount, InputTokens = GetTokenUsage(responseText).InputTokens, @@ -204,7 +226,7 @@ public sealed class StoryIntelligenceClient( ex, "Story Intelligence prompt failed. PromptVersion={PromptVersion} Model={Model} DurationMs={DurationMs} RetryCount={RetryCount} InputTokens={InputTokens} OutputTokens={OutputTokens} EstimatedCost={EstimatedCost} Success={Success}", promptVersion, - settings.Model, + model, stopwatch.ElapsedMilliseconds, null, null, @@ -312,6 +334,8 @@ public sealed class StoryIntelligenceDiagnosticsService( var model = new StoryIntelligenceDiagnosticsViewModel { Model = clientStatus.Model, + ChapterStructureModel = clientStatus.ChapterStructureModel, + SceneIntelligenceModel = clientStatus.SceneIntelligenceModel, ApiKeyConfigured = clientStatus.ApiKeyConfigured, ClientConstructed = clientStatus.ClientConstructed, ConnectionStatus = clientStatus.ConnectionStatus, @@ -379,7 +403,11 @@ public sealed class StoryIntelligenceDryRunService( var template = await prompts.LoadPromptAsync(ScenePromptFile, cancellationToken); var completedPrompt = builder.BuildPrompt(template, contextJson, form.SceneText); - var result = await client.ExecutePromptAsync(completedPrompt, promptVersion, cancellationToken); + var result = await client.ExecutePromptAsync( + completedPrompt, + promptVersion, + cancellationToken, + clientStatus.SceneIntelligenceModel); var sceneJson = ExtractSceneJson(result.RawResponseText); var parsedScene = DeserializeScene(sceneJson); var validation = validator.Validate(parsedScene); @@ -419,7 +447,7 @@ public sealed class StoryIntelligenceDryRunService( { Success = false, PromptVersion = promptVersion, - Model = clientStatus.Model, + Model = clientStatus.SceneIntelligenceModel, SceneContextJson = contextJson, ErrorMessage = $"Scene Intelligence JSON could not be deserialised: {ex.Message}", Validation = validator.Validate(null) @@ -429,7 +457,7 @@ public sealed class StoryIntelligenceDryRunService( ex, "Admin Story Intelligence dry run deserialisation failed. PromptVersion={PromptVersion} Model={Model} Success={Success}", promptVersion, - clientStatus.Model, + clientStatus.SceneIntelligenceModel, false); } catch (Exception ex) when (ex is not OperationCanceledException) @@ -438,7 +466,7 @@ public sealed class StoryIntelligenceDryRunService( { Success = false, PromptVersion = promptVersion, - Model = clientStatus.Model, + Model = clientStatus.SceneIntelligenceModel, SceneContextJson = contextJson, ErrorMessage = ex.Message }; @@ -447,7 +475,7 @@ public sealed class StoryIntelligenceDryRunService( ex, "Admin Story Intelligence dry run failed. PromptVersion={PromptVersion} Model={Model} Success={Success}", promptVersion, - clientStatus.Model, + clientStatus.SceneIntelligenceModel, false); } @@ -530,10 +558,17 @@ public sealed class ChapterStructureDryRunService( var template = await prompts.LoadPromptAsync(ChapterPromptFile, cancellationToken); var numberedChapterText = StoryIntelligenceParagraphs.Number(paragraphs); var completedPrompt = BuildChapterPrompt(template, contextJson, numberedChapterText); - var result = await client.ExecutePromptAsync(completedPrompt, promptVersion, cancellationToken); + var result = await client.ExecutePromptAsync( + completedPrompt, + promptVersion, + cancellationToken, + clientStatus.ChapterStructureModel); var chapterJson = ExtractOutputText(result.RawResponseText); var parsedChapter = DeserializeChapter(chapterJson); - var validation = validator.Validate(parsedChapter, paragraphCount); + var chapterNormalisation = ChapterStructureBoundaryNormaliser.Normalise(parsedChapter, paragraphCount); + var chapterForValidation = chapterNormalisation.ChapterStructure; + var validation = validator.Validate(chapterForValidation, paragraphCount); + ChapterStructureBoundaryNormaliser.AddIssuesTo(validation, chapterNormalisation); model.Result = new ChapterStructureDryRunResultViewModel { @@ -548,7 +583,7 @@ public sealed class ChapterStructureDryRunService( OutputTokens = result.OutputTokens, RawResponseText = result.RawResponseText, ChapterJsonText = chapterJson, - ParsedChapter = parsedChapter, + ParsedChapter = chapterForValidation, Validation = validation }; @@ -571,7 +606,7 @@ public sealed class ChapterStructureDryRunService( { Success = false, PromptVersion = promptVersion, - Model = clientStatus.Model, + Model = clientStatus.ChapterStructureModel, ChapterContextJson = contextJson, ParagraphCount = paragraphCount, ErrorMessage = $"Chapter Structure JSON could not be deserialised: {ex.Message}", @@ -582,7 +617,7 @@ public sealed class ChapterStructureDryRunService( ex, "Admin Chapter Structure dry run deserialisation failed. PromptVersion={PromptVersion} Model={Model} Success={Success}", promptVersion, - clientStatus.Model, + clientStatus.ChapterStructureModel, false); } catch (Exception ex) when (ex is not OperationCanceledException) @@ -591,7 +626,7 @@ public sealed class ChapterStructureDryRunService( { Success = false, PromptVersion = promptVersion, - Model = clientStatus.Model, + Model = clientStatus.ChapterStructureModel, ChapterContextJson = contextJson, ParagraphCount = paragraphCount, ErrorMessage = ex.Message @@ -601,7 +636,7 @@ public sealed class ChapterStructureDryRunService( ex, "Admin Chapter Structure dry run failed. PromptVersion={PromptVersion} Model={Model} Success={Success}", promptVersion, - clientStatus.Model, + clientStatus.ChapterStructureModel, false); } @@ -687,6 +722,8 @@ public sealed class StoryIntelligenceClientConfigurationStatus { public bool ApiKeyConfigured { get; init; } public string Model { get; init; } = string.Empty; + public string ChapterStructureModel { get; init; } = string.Empty; + public string SceneIntelligenceModel { get; init; } = string.Empty; public int MaxOutputTokens { get; init; } public int TimeoutSeconds { get; init; } public bool ClientConstructed { get; init; } diff --git a/PlotLine/Services/StoryIntelligenceResultPersistenceService.cs b/PlotLine/Services/StoryIntelligenceResultPersistenceService.cs index 89de08b..fd84d16 100644 --- a/PlotLine/Services/StoryIntelligenceResultPersistenceService.cs +++ b/PlotLine/Services/StoryIntelligenceResultPersistenceService.cs @@ -83,7 +83,9 @@ public sealed class StoryIntelligenceResultPersistenceService( SourceCharacterCount = form.ChapterText.Length, SourceParagraphCount = StoryIntelligenceParagraphs.Split(form.ChapterText).Count, SourceChapterCount = 1, - Model = clientStatus.Model, + Model = StoryIntelligenceOptions.BuildModelSummary( + clientStatus.ChapterStructureModel, + clientStatus.SceneIntelligenceModel), PromptVersionsSummary = promptVersionsSummary }); } @@ -140,7 +142,7 @@ public sealed class StoryIntelligenceResultPersistenceService( SourceChapterCount = 1, PromptVersion = promptVersionsSummary, PromptVersionsSummary = promptVersionsSummary, - Model = result.Model, + Model = StoryIntelligenceOptions.ParseModelSummary(result.Model, result.Model, result.Model).ChapterStructureModel, StartedUtc = startedUtc, CompletedUtc = completedUtc, FailureStage = DetermineFailureStage(result), @@ -166,7 +168,7 @@ public sealed class StoryIntelligenceResultPersistenceService( ChapterNumber = form.ChapterNumber, SourceLabel = CleanSourceLabel(form.SourceLabel), PromptVersion = result.ChapterPromptVersion, - Model = result.Model, + Model = StoryIntelligenceOptions.ParseModelSummary(result.Model, result.Model, result.Model).SceneIntelligenceModel, RawResponseJson = result.ChapterRawResponseText, OutputTextJson = result.ChapterJsonText, ParsedJson = SerializeParsed(result.ParsedChapter), diff --git a/PlotLine/ViewModels/FeatureRequestViewModels.cs b/PlotLine/ViewModels/FeatureRequestViewModels.cs index eb75b73..02b10ee 100644 --- a/PlotLine/ViewModels/FeatureRequestViewModels.cs +++ b/PlotLine/ViewModels/FeatureRequestViewModels.cs @@ -81,6 +81,8 @@ public sealed class AdminDashboardViewModel public sealed class StoryIntelligenceDiagnosticsViewModel { public string Model { get; set; } = string.Empty; + public string ChapterStructureModel { get; set; } = string.Empty; + public string SceneIntelligenceModel { get; set; } = string.Empty; public int MaxOutputTokens { get; set; } public int TimeoutSeconds { get; set; } public bool ApiKeyConfigured { get; set; } diff --git a/PlotLine/Views/Admin/StoryIntelligenceDiagnostics.cshtml b/PlotLine/Views/Admin/StoryIntelligenceDiagnostics.cshtml index 95e91b7..a666ebe 100644 --- a/PlotLine/Views/Admin/StoryIntelligenceDiagnostics.cshtml +++ b/PlotLine/Views/Admin/StoryIntelligenceDiagnostics.cshtml @@ -48,6 +48,10 @@
Max output tokens
@Model.MaxOutputTokens.ToString("N0")
+
Chapter Structure model
+
@Model.ChapterStructureModel
+
Scene Intelligence model
+
@Model.SceneIntelligenceModel
Timeout
@Model.TimeoutSeconds seconds
Live API status
diff --git a/PlotLine/Views/Admin/StoryIntelligenceFullChapterTest.cshtml b/PlotLine/Views/Admin/StoryIntelligenceFullChapterTest.cshtml index 6a30700..cbd112e 100644 --- a/PlotLine/Views/Admin/StoryIntelligenceFullChapterTest.cshtml +++ b/PlotLine/Views/Admin/StoryIntelligenceFullChapterTest.cshtml @@ -45,6 +45,24 @@ +
+ + +
+
+ + +
@@ -103,6 +121,8 @@
Estimated pipeline input tokens
@Model.Preflight.EstimatedPipelineInputTokens.ToString("N0")
+
Selected models
+
Chapter Structure: @Model.Form.ChapterStructureModel; Scene Intelligence: @Model.Form.SceneIntelligenceModel
Estimated input cost
@DisplayCost(Model.Preflight.EstimatedInputCostGBP, "GBP") / @DisplayCost(Model.Preflight.EstimatedInputCostUSD, "USD")
diff --git a/PlotLine/Views/Admin/StoryIntelligenceRunDetails.cshtml b/PlotLine/Views/Admin/StoryIntelligenceRunDetails.cshtml index bc0695f..ab9a90c 100644 --- a/PlotLine/Views/Admin/StoryIntelligenceRunDetails.cshtml +++ b/PlotLine/Views/Admin/StoryIntelligenceRunDetails.cshtml @@ -120,6 +120,10 @@ else
@Display(run.PromptVersionsSummary)
Model
@run.Model
+
Chapter Structure model used
+
@ChapterStructureModelUsed(Model)
+
Scene Intelligence model used
+
@SceneIntelligenceModelUsed(Model)
Started
@run.StartedUtc.ToString("yyyy-MM-dd HH:mm:ss") UTC
Completed
@@ -148,6 +152,12 @@ else } + else if (CanRerun(run)) + { + + }
@@ -166,6 +176,8 @@ else
@Display(chapter.ChapterNumber)
Prompt version
@chapter.PromptVersion
+
Model
+
@chapter.Model
Validation
@chapter.ValidationErrorsCount.ToString("N0") error(s), @chapter.ValidationWarningsCount.ToString("N0") warning(s)
Duration
@@ -174,12 +186,15 @@ else
@Display(chapter.TotalTokens) total (@Display(chapter.InputTokens) input, @Display(chapter.OutputTokens) output)
-
Highest returned endParagraph
-
@Display(HighestReturnedEndParagraph(chapter))
+
@Display(HighestRawReturnedEndParagraph(chapter))
Returned scene boundary ranges
-
@DisplayBoundaryRanges(chapter)
+
@DisplayRawBoundaryRanges(chapter)
+
Normalised scene boundary ranges used
+
@DisplayNormalisedBoundaryRanges(chapter)
+
Chapter structure normalisation
+
@ChapterStructureNormalisation(chapter)
@if (ReturnedEndParagraphExceedsSubmitted(chapter, run)) { @@ -187,6 +202,7 @@ else The model returned paragraph numbers beyond the submitted chapter. This usually means the prompt was not using explicit numbered paragraphs. } + @@ -214,6 +230,8 @@ else
@scene.SourceLabel
Prompt version
@scene.PromptVersion
+
Model
+
@scene.Model
Validation
@scene.ValidationErrorsCount.ToString("N0") error(s), @scene.ValidationWarningsCount.ToString("N0") warning(s)
@if (IsFailedScene(scene)) @@ -223,7 +241,9 @@ else
Error message
@SceneErrorMessage(scene)
Retry attempted
-
No
+
@(SceneRetryAttempted(scene) ? "Yes" : "No")
+
Retry error
+
@Display(SceneRetryError(scene))
}
Duration
@DisplayDuration(scene.DurationMs)
@@ -233,7 +253,7 @@ else - + @@ -264,6 +284,45 @@ else private static string DisplayCost(decimal? value, string currency) => value.HasValue ? $"{value.Value:0.000000} {currency}" : "None"; + private static string ChapterStructureModelUsed(StoryIntelligenceSavedRunDetailViewModel model) + => !string.IsNullOrWhiteSpace(model.ChapterResult?.Model) + ? model.ChapterResult.Model + : ParseModelPart(model.Run?.Model, "Chapter"); + + private static string SceneIntelligenceModelUsed(StoryIntelligenceSavedRunDetailViewModel model) + { + var sceneModels = model.SceneResults + .Select(scene => scene.Model) + .Where(value => !string.IsNullOrWhiteSpace(value)) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToList(); + if (sceneModels.Count > 0) + { + return string.Join(", ", sceneModels); + } + + return ParseModelPart(model.Run?.Model, "Scene"); + } + + private static string ParseModelPart(string? summary, string key) + { + if (string.IsNullOrWhiteSpace(summary)) + { + return "None"; + } + + foreach (var segment in summary.Split(';', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)) + { + var parts = segment.Split('=', 2, StringSplitOptions.TrimEntries); + if (parts.Length == 2 && string.Equals(parts[0], key, StringComparison.OrdinalIgnoreCase)) + { + return parts[1]; + } + } + + return summary; + } + private static string DisplayName(string? title, int? id, string label) => !string.IsNullOrWhiteSpace(title) ? title @@ -312,9 +371,14 @@ else } private static bool IsFailedScene(StoryIntelligenceSavedSceneResult scene) - => scene.ValidationErrorsCount > 0 - && string.IsNullOrWhiteSpace(scene.RawResponseJson) - && !string.IsNullOrWhiteSpace(scene.ParsedJson); + => scene.ValidationErrorsCount > 0 && SceneJsonString(scene, "error") is not null; + + private static bool CanRerun(StoryIntelligenceSavedRun run) + => !string.IsNullOrWhiteSpace(run.SourceText) + && (run.Status is StoryIntelligenceRunStatuses.Completed + or StoryIntelligenceRunStatuses.CompletedWithWarnings + or StoryIntelligenceRunStatuses.Failed + or StoryIntelligenceRunStatuses.Cancelled); private static string SceneFailureStage(StoryIntelligenceSavedSceneResult scene) => SceneErrorMessage(scene).Contains("boundary", StringComparison.OrdinalIgnoreCase) @@ -322,28 +386,62 @@ else : StoryIntelligenceFailureStages.SceneIntelligence; private static string SceneErrorMessage(StoryIntelligenceSavedSceneResult scene) + => SceneJsonString(scene, "error") ?? "Scene failed. See parsed JSON for details."; + + private static bool SceneRetryAttempted(StoryIntelligenceSavedSceneResult scene) { try { using var document = System.Text.Json.JsonDocument.Parse(scene.ParsedJson); - if (document.RootElement.TryGetProperty("error", out var error) && error.ValueKind == System.Text.Json.JsonValueKind.String) + if (document.RootElement.TryGetProperty("retryAttempted", out var retry) && retry.ValueKind is System.Text.Json.JsonValueKind.True or System.Text.Json.JsonValueKind.False) { - return error.GetString() ?? "Scene failed."; + return retry.GetBoolean(); } } catch (System.Text.Json.JsonException) { } - return "Scene failed. See parsed JSON for details."; + return false; } - private static int? HighestReturnedEndParagraph(StoryIntelligenceSavedChapterResult chapter) - => ReadBoundaryRanges(chapter).Select(range => range.End).DefaultIfEmpty().Max() is var max && max > 0 ? max : null; + private static string? SceneRetryError(StoryIntelligenceSavedSceneResult scene) + => SceneJsonString(scene, "retryError"); - private static string DisplayBoundaryRanges(StoryIntelligenceSavedChapterResult chapter) + private static string? SceneJsonString(StoryIntelligenceSavedSceneResult scene, string propertyName) { - var ranges = ReadBoundaryRanges(chapter).ToList(); + if (string.IsNullOrWhiteSpace(scene.ParsedJson)) + { + return null; + } + + try + { + using var document = System.Text.Json.JsonDocument.Parse(scene.ParsedJson); + if (document.RootElement.TryGetProperty(propertyName, out var value) && value.ValueKind == System.Text.Json.JsonValueKind.String) + { + return value.GetString(); + } + } + catch (System.Text.Json.JsonException) + { + } + + return null; + } + + private static int? HighestRawReturnedEndParagraph(StoryIntelligenceSavedChapterResult chapter) + => ReadBoundaryRanges(chapter.OutputTextJson).Select(range => range.End).DefaultIfEmpty().Max() is var max && max > 0 ? max : null; + + private static string DisplayRawBoundaryRanges(StoryIntelligenceSavedChapterResult chapter) + => DisplayBoundaryRanges(chapter.OutputTextJson); + + private static string DisplayNormalisedBoundaryRanges(StoryIntelligenceSavedChapterResult chapter) + => DisplayBoundaryRanges(chapter.ParsedJson); + + private static string DisplayBoundaryRanges(string json) + { + var ranges = ReadBoundaryRanges(json).ToList(); return ranges.Count == 0 ? "None" : string.Join(", ", ranges.Select(range => $"{range.Start}-{range.End}")); @@ -351,21 +449,53 @@ else private static bool ReturnedEndParagraphExceedsSubmitted(StoryIntelligenceSavedChapterResult chapter, StoryIntelligenceSavedRun run) { - var highest = HighestReturnedEndParagraph(chapter); + var highest = HighestRawReturnedEndParagraph(chapter); var submitted = run.SourceParagraphCount; return highest.HasValue && submitted.HasValue && highest.Value > submitted.Value; } - private static IReadOnlyList<(int Start, int End)> ReadBoundaryRanges(StoryIntelligenceSavedChapterResult chapter) + private static string ChapterStructureNormalisation(StoryIntelligenceSavedChapterResult chapter) { if (string.IsNullOrWhiteSpace(chapter.ParsedJson)) + { + return "None"; + } + + try + { + using var document = System.Text.Json.JsonDocument.Parse(chapter.ParsedJson); + if (!document.RootElement.TryGetProperty("normalisation", out var normalisation) + || normalisation.ValueKind != System.Text.Json.JsonValueKind.Array) + { + return "None"; + } + + var messages = normalisation + .EnumerateArray() + .Select(issue => issue.TryGetProperty("message", out var message) && message.ValueKind == System.Text.Json.JsonValueKind.String + ? message.GetString() + : null) + .Where(message => !string.IsNullOrWhiteSpace(message)) + .ToList(); + + return messages.Count == 0 ? "None" : string.Join(Environment.NewLine, messages); + } + catch (System.Text.Json.JsonException) + { + return "None"; + } + } + + private static IReadOnlyList<(int Start, int End)> ReadBoundaryRanges(string json) + { + if (string.IsNullOrWhiteSpace(json)) { return []; } try { - using var document = System.Text.Json.JsonDocument.Parse(chapter.ParsedJson); + using var document = System.Text.Json.JsonDocument.Parse(json); if (!document.RootElement.TryGetProperty("sceneBoundaries", out var boundaries) || boundaries.ValueKind != System.Text.Json.JsonValueKind.Array) { diff --git a/PlotLine/appsettings.json b/PlotLine/appsettings.json index 3be68e2..3c5d14e 100644 --- a/PlotLine/appsettings.json +++ b/PlotLine/appsettings.json @@ -24,6 +24,11 @@ "MaxOutputTokens": 4000, "TimeoutSeconds": 300 }, + "StoryIntelligence": { + "ChapterStructureModel": "gpt-5-mini", + "SceneIntelligenceModel": "gpt-5-mini", + "SceneIntelligenceMaxOutputTokens": 10000 + }, "Site": { "PublicBaseUrl": "https://plotdirector.com" },