Keep recovered AI failures out of user warnings
This commit is contained in:
parent
0cc25c570c
commit
a319df2455
@ -232,7 +232,10 @@ var tests = new (string Name, Action Test)[]
|
||||
("Story Intelligence heavy review modules offer bounded rendering", StoryIntelligenceHeavyReviewModulesOfferBoundedRendering),
|
||||
("Story Intelligence Review Centre adds no AI calls", StoryIntelligenceReviewCentreAddsNoAiCalls),
|
||||
("Story Intelligence Job 6 uses bounded concurrent chapter execution", StoryIntelligenceJob6UsesBoundedConcurrentChapterExecution),
|
||||
("Story Intelligence global progress uses whole run scene totals", StoryIntelligenceGlobalProgressUsesWholeRunSceneTotals)
|
||||
("Story Intelligence global progress uses whole run scene totals", StoryIntelligenceGlobalProgressUsesWholeRunSceneTotals),
|
||||
("Story Intelligence recovered provider failures stay operational", StoryIntelligenceRecoveredProviderFailuresStayOperational),
|
||||
("Story Intelligence recovered JSON repair stays operational", StoryIntelligenceRecoveredJsonRepairStaysOperational),
|
||||
("Story Intelligence terminal failures remain user visible", StoryIntelligenceTerminalFailuresRemainUserVisible)
|
||||
};
|
||||
|
||||
foreach (var test in tests)
|
||||
@ -342,6 +345,93 @@ static void StoryIntelligenceGlobalProgressUsesWholeRunSceneTotals()
|
||||
Assert(migration.Contains("N'Finalising'", StringComparison.Ordinal), "Progress migration should expose a finalising state.");
|
||||
}
|
||||
|
||||
static void StoryIntelligenceRecoveredProviderFailuresStayOperational()
|
||||
{
|
||||
var requestCount = 0;
|
||||
var client = BuildStoryIntelligenceClient(_ =>
|
||||
{
|
||||
requestCount++;
|
||||
return requestCount == 1
|
||||
? new HttpResponseMessage(HttpStatusCode.TooManyRequests)
|
||||
{
|
||||
Content = new StringContent("{\"error\":{\"code\":\"rate_limit_exceeded\"}}", Encoding.UTF8, "application/json")
|
||||
}
|
||||
: OpenAiSuccessResponse("{}");
|
||||
});
|
||||
|
||||
var result = client.ExecutePromptAsync("Analyse this scene.", "Test-Prompt", CancellationToken.None)
|
||||
.GetAwaiter()
|
||||
.GetResult();
|
||||
|
||||
Assert(requestCount == 2, "Recovered rate-limit/provider failures should be retried internally.");
|
||||
Assert(result.RetryCount == 1, "Recovered provider retries should remain execution metadata.");
|
||||
|
||||
requestCount = 0;
|
||||
client = BuildStoryIntelligenceClient(_ =>
|
||||
{
|
||||
requestCount++;
|
||||
if (requestCount == 1)
|
||||
{
|
||||
throw new TaskCanceledException("Simulated timeout.");
|
||||
}
|
||||
|
||||
return OpenAiSuccessResponse("{}");
|
||||
});
|
||||
|
||||
result = client.ExecutePromptAsync("Analyse this scene.", "Test-Prompt", CancellationToken.None)
|
||||
.GetAwaiter()
|
||||
.GetResult();
|
||||
|
||||
Assert(requestCount == 2, "Recovered timeouts should be retried internally.");
|
||||
Assert(result.RetryCount == 1, "Recovered timeout retries should not create user-facing scene failures.");
|
||||
}
|
||||
|
||||
static void StoryIntelligenceRecoveredJsonRepairStaysOperational()
|
||||
{
|
||||
var root = PlotLineRoot();
|
||||
var runner = File.ReadAllText(Path.Combine(root, "Services/PersistedStoryIntelligenceRunner.cs"));
|
||||
var notificationSql = File.ReadAllText(Path.Combine(root, "Sql/149_Phase24_LeanImportCompletionNotifications.sql"));
|
||||
var progressScript = File.ReadAllText(Path.Combine(root, "wwwroot/js/story-intelligence-progress.js"));
|
||||
var globalScript = File.ReadAllText(Path.Combine(root, "wwwroot/js/story-intelligence-global-progress.js"));
|
||||
|
||||
var sceneValidationBlock = ExtractBetween(
|
||||
runner,
|
||||
"var validation = sceneValidator.Validate(sceneAttempt.Parsed);",
|
||||
"hasWarnings = hasWarnings || validation.Warnings.Count > 0;");
|
||||
var chapterValidationBlock = ExtractBetween(
|
||||
runner,
|
||||
"var chapterValidation = chapterValidator.Validate(chapterForProcessing, paragraphs.Count);",
|
||||
"ChapterStructureBoundaryNormaliser.AddIssuesTo");
|
||||
|
||||
Assert(sceneValidationBlock.Contains("LogRecoveredJsonWarnings", StringComparison.Ordinal), "Recovered scene JSON repair/retry warnings should stay in operational logging.");
|
||||
Assert(!sceneValidationBlock.Contains("validation.Warnings.Add", StringComparison.Ordinal), "Recovered scene JSON warnings must not become validation warnings or Needs Attention.");
|
||||
Assert(chapterValidationBlock.Contains("LogRecoveredJsonWarnings", StringComparison.Ordinal), "Recovered chapter JSON repair/retry warnings should stay in operational logging.");
|
||||
Assert(!chapterValidationBlock.Contains("chapterValidation.Warnings.Add", StringComparison.Ordinal), "Recovered chapter JSON warnings must not become validation warnings.");
|
||||
Assert(runner.Contains("SerialiseSceneParseAudit(sceneAttempt, validation.IsValid)", StringComparison.Ordinal), "Recovered scene JSON repair details should remain in technical parse audit.");
|
||||
Assert(runner.Contains("jsonWarnings = parseResult.Warnings", StringComparison.Ordinal), "Recovered chapter JSON repair details should remain in technical parse audit.");
|
||||
Assert(runner.Contains("LogRecoveredJsonWarnings(run, $\"Scene {block.TemporarySceneNumber:N0}\", sceneAttempt.Warnings)", StringComparison.Ordinal), "Invalid response repaired by retry should be logged, not surfaced as author review.");
|
||||
Assert(notificationSql.Contains("Status IN (N'Failed', N'Cancelled')", StringComparison.Ordinal), "Failure notification should be based on final failed/cancelled runs.");
|
||||
Assert(notificationSql.Contains("SET @NotificationKind = CASE WHEN @HasFailedRuns = 1 THEN N'Failure' ELSE N'Success' END", StringComparison.Ordinal), "Recovered runs should not generate failure notifications.");
|
||||
Assert(progressScript.Contains("current.failedScenes += failed", StringComparison.Ordinal), "Onboarding Needs review should use persisted terminal FailedScenes only.");
|
||||
Assert(globalScript.Contains("const failed = readNumber(progress.failedScenes ?? progress.FailedScenes) || 0", StringComparison.Ordinal), "Global banner should use terminal failed-scene count only.");
|
||||
}
|
||||
|
||||
static void StoryIntelligenceTerminalFailuresRemainUserVisible()
|
||||
{
|
||||
var root = PlotLineRoot();
|
||||
var runner = File.ReadAllText(Path.Combine(root, "Services/PersistedStoryIntelligenceRunner.cs"));
|
||||
var viewModel = File.ReadAllText(Path.Combine(root, "ViewModels/OnboardingViewModels.cs"));
|
||||
var globalViewModel = File.ReadAllText(Path.Combine(root, "ViewModels/StoryIntelligenceGlobalProgressViewModels.cs"));
|
||||
|
||||
var failedSceneBlock = ExtractBetween(runner, "private async Task PersistFailedSceneAsync", "private StoryIntelligenceStageModels ResolveStageModels");
|
||||
Assert(failedSceneBlock.Contains("ValidationErrorsCount = 1", StringComparison.Ordinal), "Retries exhausted for one scene should persist a terminal scene failure.");
|
||||
Assert(runner.Contains("catch (Exception ex) when (ex is not OperationCanceledException && !IsFatal(ex))", StringComparison.Ordinal), "Only unrecovered scene exceptions should enter terminal scene failure handling.");
|
||||
Assert(runner.Contains("failedScenes++", StringComparison.Ordinal), "Terminal scene failures should increment the persisted failed-scene count.");
|
||||
Assert(runner.Contains("failedScenes > 0 || hasWarnings", StringComparison.Ordinal), "Partial terminal scene failures should complete the chapter with warnings rather than forcing whole-run failure.");
|
||||
Assert(viewModel.Contains("AnalysisIssueCount => Chapters.Count(chapter => chapter.IsFailed) + Chapters.Sum(chapter => chapter.FailedScenes ?? 0)", StringComparison.Ordinal), "Review Centre issue count should be driven by terminal chapter/run failures.");
|
||||
Assert(globalViewModel.Contains("public bool HasFailures => FailedRunCount > 0 || FailedScenes > 0", StringComparison.Ordinal), "Global banner warning state should be driven by terminal persisted failures.");
|
||||
}
|
||||
|
||||
static void StoryIntelligenceReviewCentreRouteReplacesLinearReviewResume()
|
||||
{
|
||||
var root = PlotLineRoot();
|
||||
@ -4639,6 +4729,44 @@ static void Assert(bool condition, string message)
|
||||
}
|
||||
}
|
||||
|
||||
static StoryIntelligenceClient BuildStoryIntelligenceClient(Func<HttpRequestMessage, HttpResponseMessage> respond)
|
||||
=> new(
|
||||
new HttpClient(new CaptureImageRequestHandler(respond)),
|
||||
Options.Create(new StoryIntelligenceOptions
|
||||
{
|
||||
ApiKey = "test-key",
|
||||
Model = "test-model",
|
||||
MaxOutputTokens = 64,
|
||||
TimeoutSeconds = 1
|
||||
}),
|
||||
NullLogger<StoryIntelligenceClient>.Instance);
|
||||
|
||||
static HttpResponseMessage OpenAiSuccessResponse(string outputText)
|
||||
=> new(HttpStatusCode.OK)
|
||||
{
|
||||
Content = new StringContent(
|
||||
JsonSerializer.Serialize(new
|
||||
{
|
||||
output = new[]
|
||||
{
|
||||
new
|
||||
{
|
||||
content = new[]
|
||||
{
|
||||
new { type = "output_text", text = outputText }
|
||||
}
|
||||
}
|
||||
},
|
||||
usage = new
|
||||
{
|
||||
input_tokens = 7,
|
||||
output_tokens = 11
|
||||
}
|
||||
}),
|
||||
Encoding.UTF8,
|
||||
"application/json")
|
||||
};
|
||||
|
||||
internal sealed class CaptureImageRequestHandler(Func<HttpRequestMessage, HttpResponseMessage> respond) : HttpMessageHandler
|
||||
{
|
||||
protected override Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
|
||||
|
||||
@ -251,16 +251,7 @@ public sealed class PersistedStoryIntelligenceRunner(
|
||||
parseStopwatch.Stop();
|
||||
var validationStopwatch = Stopwatch.StartNew();
|
||||
var validation = sceneValidator.Validate(sceneAttempt.Parsed);
|
||||
foreach (var warning in sceneAttempt.Warnings)
|
||||
{
|
||||
validation.Warnings.Add(new PlotLine.Models.StoryIntelligence.ValidationIssue
|
||||
{
|
||||
Severity = "Warning",
|
||||
Path = "$",
|
||||
Message = warning,
|
||||
SuggestedFix = "Review the raw and repaired Scene Intelligence JSON."
|
||||
});
|
||||
}
|
||||
LogRecoveredJsonWarnings(run, $"Scene {block.TemporarySceneNumber:N0}", sceneAttempt.Warnings);
|
||||
|
||||
hasWarnings = hasWarnings || validation.Warnings.Count > 0;
|
||||
validationStopwatch.Stop();
|
||||
@ -724,16 +715,7 @@ public sealed class PersistedStoryIntelligenceRunner(
|
||||
var chapterNormalisation = ChapterStructureBoundaryNormaliser.Normalise(parsedChapter, paragraphs.Count);
|
||||
var chapterForProcessing = chapterNormalisation.ChapterStructure;
|
||||
var chapterValidation = chapterValidator.Validate(chapterForProcessing, paragraphs.Count);
|
||||
foreach (var warning in chapterAttempt.Warnings)
|
||||
{
|
||||
chapterValidation.Warnings.Add(new PlotLine.Models.StoryIntelligence.ValidationIssue
|
||||
{
|
||||
Severity = "Warning",
|
||||
Path = "$",
|
||||
Message = warning,
|
||||
SuggestedFix = "Review the raw and repaired Chapter Structure JSON."
|
||||
});
|
||||
}
|
||||
LogRecoveredJsonWarnings(run, "Chapter Structure", chapterAttempt.Warnings);
|
||||
|
||||
ChapterStructureBoundaryNormaliser.AddIssuesTo(chapterValidation, chapterNormalisation);
|
||||
var chapterResultId = await repository.SaveChapterResultAsync(
|
||||
@ -1344,6 +1326,24 @@ public sealed class PersistedStoryIntelligenceRunner(
|
||||
retryError = parseException.RetryError
|
||||
}, JsonOptions);
|
||||
|
||||
private void LogRecoveredJsonWarnings(StoryIntelligenceQueuedRun run, string label, IReadOnlyList<string> warnings)
|
||||
{
|
||||
if (warnings.Count == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
logger.LogInformation(
|
||||
"Recovered Story Intelligence JSON warning kept as operational audit only. RunID={RunID} UserID={UserID} ProjectID={ProjectID} BookID={BookID} ChapterID={ChapterID} Label={Label} Warnings={Warnings}",
|
||||
run.StoryIntelligenceRunID,
|
||||
run.UserID,
|
||||
run.ProjectID,
|
||||
run.BookID,
|
||||
run.ChapterID,
|
||||
label,
|
||||
warnings);
|
||||
}
|
||||
|
||||
private static string BuildJsonRetryPrompt(string originalPrompt, string label, string parseError, string malformedJson)
|
||||
=> string.Concat(
|
||||
originalPrompt,
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user