diff --git a/PlotLine.Tests/Program.cs b/PlotLine.Tests/Program.cs index acd4c92..3a6a69e 100644 --- a/PlotLine.Tests/Program.cs +++ b/PlotLine.Tests/Program.cs @@ -232,7 +232,10 @@ var tests = new (string Name, Action Test)[] ("Story Intelligence heavy review modules offer bounded rendering", StoryIntelligenceHeavyReviewModulesOfferBoundedRendering), ("Story Intelligence Review Centre adds no AI calls", StoryIntelligenceReviewCentreAddsNoAiCalls), ("Story Intelligence Job 6 uses bounded concurrent chapter execution", StoryIntelligenceJob6UsesBoundedConcurrentChapterExecution), - ("Story Intelligence global progress uses whole run scene totals", StoryIntelligenceGlobalProgressUsesWholeRunSceneTotals) + ("Story Intelligence global progress uses whole run scene totals", StoryIntelligenceGlobalProgressUsesWholeRunSceneTotals), + ("Story Intelligence recovered provider failures stay operational", StoryIntelligenceRecoveredProviderFailuresStayOperational), + ("Story Intelligence recovered JSON repair stays operational", StoryIntelligenceRecoveredJsonRepairStaysOperational), + ("Story Intelligence terminal failures remain user visible", StoryIntelligenceTerminalFailuresRemainUserVisible) }; foreach (var test in tests) @@ -342,6 +345,93 @@ static void StoryIntelligenceGlobalProgressUsesWholeRunSceneTotals() Assert(migration.Contains("N'Finalising'", StringComparison.Ordinal), "Progress migration should expose a finalising state."); } +static void StoryIntelligenceRecoveredProviderFailuresStayOperational() +{ + var requestCount = 0; + var client = BuildStoryIntelligenceClient(_ => + { + requestCount++; + return requestCount == 1 + ? new HttpResponseMessage(HttpStatusCode.TooManyRequests) + { + Content = new StringContent("{\"error\":{\"code\":\"rate_limit_exceeded\"}}", Encoding.UTF8, "application/json") + } + : OpenAiSuccessResponse("{}"); + }); + + var result = client.ExecutePromptAsync("Analyse this scene.", "Test-Prompt", CancellationToken.None) + .GetAwaiter() + .GetResult(); + + Assert(requestCount == 2, "Recovered rate-limit/provider failures should be retried internally."); + Assert(result.RetryCount == 1, "Recovered provider retries should remain execution metadata."); + + requestCount = 0; + client = BuildStoryIntelligenceClient(_ => + { + requestCount++; + if (requestCount == 1) + { + throw new TaskCanceledException("Simulated timeout."); + } + + return OpenAiSuccessResponse("{}"); + }); + + result = client.ExecutePromptAsync("Analyse this scene.", "Test-Prompt", CancellationToken.None) + .GetAwaiter() + .GetResult(); + + Assert(requestCount == 2, "Recovered timeouts should be retried internally."); + Assert(result.RetryCount == 1, "Recovered timeout retries should not create user-facing scene failures."); +} + +static void StoryIntelligenceRecoveredJsonRepairStaysOperational() +{ + var root = PlotLineRoot(); + var runner = File.ReadAllText(Path.Combine(root, "Services/PersistedStoryIntelligenceRunner.cs")); + var notificationSql = File.ReadAllText(Path.Combine(root, "Sql/149_Phase24_LeanImportCompletionNotifications.sql")); + var progressScript = File.ReadAllText(Path.Combine(root, "wwwroot/js/story-intelligence-progress.js")); + var globalScript = File.ReadAllText(Path.Combine(root, "wwwroot/js/story-intelligence-global-progress.js")); + + var sceneValidationBlock = ExtractBetween( + runner, + "var validation = sceneValidator.Validate(sceneAttempt.Parsed);", + "hasWarnings = hasWarnings || validation.Warnings.Count > 0;"); + var chapterValidationBlock = ExtractBetween( + runner, + "var chapterValidation = chapterValidator.Validate(chapterForProcessing, paragraphs.Count);", + "ChapterStructureBoundaryNormaliser.AddIssuesTo"); + + Assert(sceneValidationBlock.Contains("LogRecoveredJsonWarnings", StringComparison.Ordinal), "Recovered scene JSON repair/retry warnings should stay in operational logging."); + Assert(!sceneValidationBlock.Contains("validation.Warnings.Add", StringComparison.Ordinal), "Recovered scene JSON warnings must not become validation warnings or Needs Attention."); + Assert(chapterValidationBlock.Contains("LogRecoveredJsonWarnings", StringComparison.Ordinal), "Recovered chapter JSON repair/retry warnings should stay in operational logging."); + Assert(!chapterValidationBlock.Contains("chapterValidation.Warnings.Add", StringComparison.Ordinal), "Recovered chapter JSON warnings must not become validation warnings."); + Assert(runner.Contains("SerialiseSceneParseAudit(sceneAttempt, validation.IsValid)", StringComparison.Ordinal), "Recovered scene JSON repair details should remain in technical parse audit."); + Assert(runner.Contains("jsonWarnings = parseResult.Warnings", StringComparison.Ordinal), "Recovered chapter JSON repair details should remain in technical parse audit."); + Assert(runner.Contains("LogRecoveredJsonWarnings(run, $\"Scene {block.TemporarySceneNumber:N0}\", sceneAttempt.Warnings)", StringComparison.Ordinal), "Invalid response repaired by retry should be logged, not surfaced as author review."); + Assert(notificationSql.Contains("Status IN (N'Failed', N'Cancelled')", StringComparison.Ordinal), "Failure notification should be based on final failed/cancelled runs."); + Assert(notificationSql.Contains("SET @NotificationKind = CASE WHEN @HasFailedRuns = 1 THEN N'Failure' ELSE N'Success' END", StringComparison.Ordinal), "Recovered runs should not generate failure notifications."); + Assert(progressScript.Contains("current.failedScenes += failed", StringComparison.Ordinal), "Onboarding Needs review should use persisted terminal FailedScenes only."); + Assert(globalScript.Contains("const failed = readNumber(progress.failedScenes ?? progress.FailedScenes) || 0", StringComparison.Ordinal), "Global banner should use terminal failed-scene count only."); +} + +static void StoryIntelligenceTerminalFailuresRemainUserVisible() +{ + var root = PlotLineRoot(); + var runner = File.ReadAllText(Path.Combine(root, "Services/PersistedStoryIntelligenceRunner.cs")); + var viewModel = File.ReadAllText(Path.Combine(root, "ViewModels/OnboardingViewModels.cs")); + var globalViewModel = File.ReadAllText(Path.Combine(root, "ViewModels/StoryIntelligenceGlobalProgressViewModels.cs")); + + var failedSceneBlock = ExtractBetween(runner, "private async Task PersistFailedSceneAsync", "private StoryIntelligenceStageModels ResolveStageModels"); + Assert(failedSceneBlock.Contains("ValidationErrorsCount = 1", StringComparison.Ordinal), "Retries exhausted for one scene should persist a terminal scene failure."); + Assert(runner.Contains("catch (Exception ex) when (ex is not OperationCanceledException && !IsFatal(ex))", StringComparison.Ordinal), "Only unrecovered scene exceptions should enter terminal scene failure handling."); + Assert(runner.Contains("failedScenes++", StringComparison.Ordinal), "Terminal scene failures should increment the persisted failed-scene count."); + Assert(runner.Contains("failedScenes > 0 || hasWarnings", StringComparison.Ordinal), "Partial terminal scene failures should complete the chapter with warnings rather than forcing whole-run failure."); + Assert(viewModel.Contains("AnalysisIssueCount => Chapters.Count(chapter => chapter.IsFailed) + Chapters.Sum(chapter => chapter.FailedScenes ?? 0)", StringComparison.Ordinal), "Review Centre issue count should be driven by terminal chapter/run failures."); + Assert(globalViewModel.Contains("public bool HasFailures => FailedRunCount > 0 || FailedScenes > 0", StringComparison.Ordinal), "Global banner warning state should be driven by terminal persisted failures."); +} + static void StoryIntelligenceReviewCentreRouteReplacesLinearReviewResume() { var root = PlotLineRoot(); @@ -4639,6 +4729,44 @@ static void Assert(bool condition, string message) } } +static StoryIntelligenceClient BuildStoryIntelligenceClient(Func respond) + => new( + new HttpClient(new CaptureImageRequestHandler(respond)), + Options.Create(new StoryIntelligenceOptions + { + ApiKey = "test-key", + Model = "test-model", + MaxOutputTokens = 64, + TimeoutSeconds = 1 + }), + NullLogger.Instance); + +static HttpResponseMessage OpenAiSuccessResponse(string outputText) + => new(HttpStatusCode.OK) + { + Content = new StringContent( + JsonSerializer.Serialize(new + { + output = new[] + { + new + { + content = new[] + { + new { type = "output_text", text = outputText } + } + } + }, + usage = new + { + input_tokens = 7, + output_tokens = 11 + } + }), + Encoding.UTF8, + "application/json") + }; + internal sealed class CaptureImageRequestHandler(Func respond) : HttpMessageHandler { protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) diff --git a/PlotLine/Services/PersistedStoryIntelligenceRunner.cs b/PlotLine/Services/PersistedStoryIntelligenceRunner.cs index bb60588..8d1392e 100644 --- a/PlotLine/Services/PersistedStoryIntelligenceRunner.cs +++ b/PlotLine/Services/PersistedStoryIntelligenceRunner.cs @@ -251,16 +251,7 @@ public sealed class PersistedStoryIntelligenceRunner( parseStopwatch.Stop(); var validationStopwatch = Stopwatch.StartNew(); var validation = sceneValidator.Validate(sceneAttempt.Parsed); - foreach (var warning in sceneAttempt.Warnings) - { - validation.Warnings.Add(new PlotLine.Models.StoryIntelligence.ValidationIssue - { - Severity = "Warning", - Path = "$", - Message = warning, - SuggestedFix = "Review the raw and repaired Scene Intelligence JSON." - }); - } + LogRecoveredJsonWarnings(run, $"Scene {block.TemporarySceneNumber:N0}", sceneAttempt.Warnings); hasWarnings = hasWarnings || validation.Warnings.Count > 0; validationStopwatch.Stop(); @@ -724,16 +715,7 @@ public sealed class PersistedStoryIntelligenceRunner( var chapterNormalisation = ChapterStructureBoundaryNormaliser.Normalise(parsedChapter, paragraphs.Count); var chapterForProcessing = chapterNormalisation.ChapterStructure; var chapterValidation = chapterValidator.Validate(chapterForProcessing, paragraphs.Count); - foreach (var warning in chapterAttempt.Warnings) - { - chapterValidation.Warnings.Add(new PlotLine.Models.StoryIntelligence.ValidationIssue - { - Severity = "Warning", - Path = "$", - Message = warning, - SuggestedFix = "Review the raw and repaired Chapter Structure JSON." - }); - } + LogRecoveredJsonWarnings(run, "Chapter Structure", chapterAttempt.Warnings); ChapterStructureBoundaryNormaliser.AddIssuesTo(chapterValidation, chapterNormalisation); var chapterResultId = await repository.SaveChapterResultAsync( @@ -1344,6 +1326,24 @@ public sealed class PersistedStoryIntelligenceRunner( retryError = parseException.RetryError }, JsonOptions); + private void LogRecoveredJsonWarnings(StoryIntelligenceQueuedRun run, string label, IReadOnlyList warnings) + { + if (warnings.Count == 0) + { + return; + } + + logger.LogInformation( + "Recovered Story Intelligence JSON warning kept as operational audit only. RunID={RunID} UserID={UserID} ProjectID={ProjectID} BookID={BookID} ChapterID={ChapterID} Label={Label} Warnings={Warnings}", + run.StoryIntelligenceRunID, + run.UserID, + run.ProjectID, + run.BookID, + run.ChapterID, + label, + warnings); + } + private static string BuildJsonRetryPrompt(string originalPrompt, string label, string parseError, string malformedJson) => string.Concat( originalPrompt,