diff --git a/PlotLine.Tests/Program.cs b/PlotLine.Tests/Program.cs index a043ca9..c5ec402 100644 --- a/PlotLine.Tests/Program.cs +++ b/PlotLine.Tests/Program.cs @@ -82,6 +82,12 @@ var tests = new (string Name, Action Test)[] ("Illustration semantic types prevent bathroom office and ambulance car mismatches", IllustrationSemanticTypesPreventObviousMismatches), ("Illustration demand archetypes stay broad and use active template", IllustrationDemandArchetypesStayBroadAndUseActiveTemplate), ("Story Intelligence evidence extraction prevents observed attribute leakage", StoryIntelligenceEvidenceExtractionPreventsObservedAttributeLeakage), + ("Story Intelligence scene call uses strict structured output", StoryIntelligenceSceneCallUsesStrictStructuredOutput), + ("Story Intelligence scene schema constrains confidence numbers", StoryIntelligenceSceneSchemaConstrainsConfidenceNumbers), + ("Story Intelligence scene schema constrains evidence strings", StoryIntelligenceSceneSchemaConstrainsEvidenceStrings), + ("Story Intelligence structured metric values validate and import", StoryIntelligenceStructuredMetricValuesValidateAndImport), + ("Story Intelligence structured schema covers complete response model", StoryIntelligenceStructuredSchemaCoversCompleteResponseModel), + ("Story Intelligence structured schema remains fixture agnostic", StoryIntelligenceStructuredSchemaRemainsFixtureAgnostic), ("Illustration assignment repair enforces explicit evidence groups and strict subtypes", IllustrationAssignmentRepairEnforcesExplicitEvidenceGroupsAndStrictSubtypes), ("Lean import removes replay links while retaining progress", LeanImportRemovesReplayLinksWhileRetainingProgress), ("Story Intelligence progress does not prematurely report safe close", StoryIntelligenceProgressDoesNotPrematurelyReportSafeClose), @@ -4258,7 +4264,9 @@ static void AiPovJsonParsesIntoStructuralModels() var parsed = JsonSerializer.Deserialize(json, JsonOptions()); Assert(parsed?.DefaultPov?.CharacterName == "Maggie", "AI-returned defaultPov should parse."); Assert(parsed?.MixedPov == true, "AI-returned mixedPov should parse."); - Assert(parsed is not null && parsed.SceneBoundaries[1].PovOverride?.CharacterName == "Beth", "AI-returned scene povOverride should parse."); + var boundaries = parsed?.SceneBoundaries; + var secondBoundary = boundaries is { Count: > 1 } ? boundaries[1] : null; + Assert(secondBoundary?.PovOverride?.CharacterName == "Beth", "AI-returned scene povOverride should parse."); } static void PostCharacterReviewResolvesPersistedStructuralPov() @@ -4717,6 +4725,191 @@ static void NarrativeProductionCodeRemainsFixtureAgnostic() } } +static void StoryIntelligenceSceneCallUsesStrictStructuredOutput() +{ + string? requestBody = null; + var client = BuildStoryIntelligenceClient(request => + { + requestBody = request.Content?.ReadAsStringAsync().GetAwaiter().GetResult(); + return OpenAiSuccessResponse("""{"schemaVersion":"1.0"}"""); + }); + + client.ExecutePromptAsync( + "Analyse this scene.", + "Scene-Prompt-V2", + CancellationToken.None, + "gpt-5-mini", + 128, + StoryIntelligenceSceneStructuredOutputSchema.Contract) + .GetAwaiter() + .GetResult(); + + using var document = JsonDocument.Parse(requestBody ?? throw new InvalidOperationException("No request body captured.")); + var root = document.RootElement; + var format = root.GetProperty("text").GetProperty("format"); + Assert(root.GetProperty("model").GetString() == "gpt-5-mini", "Structured scene request should retain the selected scene model."); + Assert(format.GetProperty("type").GetString() == "json_schema", "Scene request should use Responses API text.format json_schema."); + Assert(format.GetProperty("name").GetString() == StoryIntelligenceSceneStructuredOutputSchema.ContractName, "Scene request should use the scene intelligence contract name."); + Assert(format.GetProperty("strict").GetBoolean(), "Scene request should enable strict structured output."); + Assert(format.GetProperty("schema").GetProperty("additionalProperties").GetBoolean() == false, "Root schema must be closed."); + + var runner = File.ReadAllText(Path.Combine(AppContext.BaseDirectory, "../../../../PlotLine/Services/PersistedStoryIntelligenceRunner.cs")); + Assert(runner.Contains("StoryIntelligenceSceneStructuredOutputSchema.Contract", StringComparison.Ordinal), "Persisted scene analysis should pass the strict scene contract."); + Assert(runner.Contains("var retryClientResult = await client.ExecutePromptAsync(", StringComparison.Ordinal), "Legacy JSON repair fallback should remain present."); +} + +static void StoryIntelligenceSceneSchemaConstrainsConfidenceNumbers() +{ + var schema = SceneSchemaRoot(); + var properties = schema.GetProperty("properties"); + AssertConfidence(properties.GetProperty("scenePurpose").GetProperty("properties").GetProperty("confidence"), "scenePurpose.confidence"); + AssertConfidence(properties.GetProperty("pointOfView").GetProperty("properties").GetProperty("confidence"), "pointOfView.confidence"); + AssertConfidence(properties.GetProperty("setting").GetProperty("properties").GetProperty("confidence"), "setting.confidence"); + AssertConfidence(ArrayItem(properties.GetProperty("characters")).GetProperty("properties").GetProperty("confidence"), "characters.confidence"); + AssertConfidence(ArrayItem(properties.GetProperty("relationships")).GetProperty("properties").GetProperty("confidence"), "relationships.confidence"); + AssertConfidence(ArrayItem(properties.GetProperty("knowledgeChanges")).GetProperty("properties").GetProperty("confidence"), "knowledgeChanges.confidence"); + AssertConfidence(ArrayItem(properties.GetProperty("observations")).GetProperty("properties").GetProperty("confidence"), "observations.confidence"); + + var schemaText = JsonSerializer.Serialize(StoryIntelligenceSceneStructuredOutputSchema.Contract.Schema); + Assert(!schemaText.Contains("0. nine", StringComparison.OrdinalIgnoreCase), "Production schema must not patch the observed malformed confidence phrase."); +} + +static void StoryIntelligenceSceneSchemaConstrainsEvidenceStrings() +{ + var properties = SceneSchemaRoot().GetProperty("properties"); + AssertString(properties.GetProperty("pointOfView").GetProperty("properties").GetProperty("evidence"), "pointOfView.evidence", nullable: true); + AssertString(ArrayItem(properties.GetProperty("relationships")).GetProperty("properties").GetProperty("evidence"), "relationships.evidence"); + AssertString(ArrayItem(properties.GetProperty("knowledgeChanges")).GetProperty("properties").GetProperty("evidence"), "knowledgeChanges.evidence"); + AssertString(ArrayItem(properties.GetProperty("timelineClues")).GetProperty("properties").GetProperty("evidence"), "timelineClues.evidence"); + AssertString(ArrayItem(properties.GetProperty("narrativeArcs")).GetProperty("properties").GetProperty("evidence"), "narrativeArcs.evidence"); + AssertString(ArrayItem(properties.GetProperty("questionsRaised")).GetProperty("properties").GetProperty("evidence"), "questionsRaised.evidence"); + AssertString(ArrayItem(properties.GetProperty("questionsAnswered")).GetProperty("properties").GetProperty("evidence"), "questionsAnswered.evidence"); + AssertString(ArrayItem(properties.GetProperty("observations")).GetProperty("properties").GetProperty("evidence"), "observations.evidence"); + + var schemaText = JsonSerializer.Serialize(StoryIntelligenceSceneStructuredOutputSchema.Contract.Schema); + Assert(!schemaText.Contains("Her mum was late", StringComparison.OrdinalIgnoreCase), "Production schema must not patch the observed malformed evidence sentence."); +} + +static void StoryIntelligenceStructuredMetricValuesValidateAndImport() +{ + var validator = new StorySceneValidator(); + var scene = RepresentativeStructuredScene(); + var validation = validator.Validate(scene); + + Assert(validation.IsValid, string.Join("; ", validation.Errors.Select(error => $"{error.Path}: {error.Message}"))); + + var commit = File.ReadAllText(Path.Combine(AppContext.BaseDirectory, "../../../../PlotLine/Services/StoryIntelligenceImportCommitService.cs")); + Assert(commit.Contains("parsed.MetricValues", StringComparison.Ordinal), "Import should normalise strict metricValues for existing metric persistence."); + Assert(commit.Contains("metrics[metric.Key.Trim()]", StringComparison.Ordinal), "Metric normalisation should key values by configured metric key."); +} + +static void StoryIntelligenceStructuredSchemaCoversCompleteResponseModel() +{ + var properties = SceneSchemaRoot().GetProperty("properties"); + var required = properties.EnumerateObject().Select(property => property.Name).ToHashSet(StringComparer.Ordinal); + var expected = new[] + { + "schemaVersion", + "sceneReference", + "scenePurpose", + "pointOfView", + "setting", + "characters", + "characterAppearance", + "locations", + "assets", + "relationships", + "knowledgeChanges", + "timelineClues", + "narrativeArcs", + "questionsRaised", + "questionsAnswered", + "observations", + "metricValues", + "sourceLimits" + }; + + foreach (var name in expected) + { + Assert(required.Contains(name), $"Strict scene schema is missing {name}."); + } + + Assert(ArrayItem(properties.GetProperty("narrativeArcs")).GetProperty("properties").TryGetProperty("matchingExistingThreadId", out _), "Narrative signals should include existing Thread matching."); + Assert(ArrayItem(properties.GetProperty("characterAppearance")).GetProperty("properties").TryGetProperty("eyeColour", out _), "Character appearance schema should include eyeColour."); + Assert(ArrayItem(properties.GetProperty("characterAppearance")).GetProperty("properties").TryGetProperty("height", out _), "Character appearance schema should include height."); + Assert(!properties.TryGetProperty("summary", out _), "Strict scene schema should not reintroduce duplicate factual scene summaries."); +} + +static void StoryIntelligenceStructuredSchemaRemainsFixtureAgnostic() +{ + var schemaText = JsonSerializer.Serialize(StoryIntelligenceSceneStructuredOutputSchema.Contract.Schema); + var client = File.ReadAllText(Path.Combine(AppContext.BaseDirectory, "../../../../PlotLine/Services/StoryIntelligenceOpenAIInfrastructure.cs")); + var forbidden = new[] { "Memory Tin", "Triumph TR6", "Mrs Patterson", "Maggie", "Beth", "Rob", "Rosie", "Victor", "Alpha Flame" }; + + foreach (var term in forbidden) + { + Assert(!schemaText.Contains(term, StringComparison.OrdinalIgnoreCase), $"Strict schema must not contain fixture term {term}."); + Assert(!client.Contains(term, StringComparison.OrdinalIgnoreCase), $"OpenAI client path must not contain fixture term {term}."); + } +} + +static SceneIntelligenceScene RepresentativeStructuredScene() + => new() + { + SchemaVersion = "1.0", + SceneReference = new SceneIntelligenceSceneReference + { + ProjectId = 12, + BookId = 34, + ChapterId = 56, + SceneId = 78, + ChapterNumber = 2, + SceneNumber = 4, + SourceLabel = "Chapter 2, Scene 4" + }, + ScenePurpose = new SceneIntelligenceScenePurpose { ObservedFunction = "Reveals a missing archive key.", Confidence = 0.86m }, + PointOfView = new SceneIntelligencePointOfView { CharacterName = "Iris Vale", NarrativeMode = "first person", Confidence = 0.92m, Evidence = "PlotDirector scene metadata supplies Iris Vale as POV." }, + Setting = new SceneIntelligenceSetting { TimeOfDay = "night", DateOrTimeReference = null, LocationName = "Clocktower Archive", LocationType = "named", GenericRoomType = null, ParentLocationHint = null, Confidence = 0.88m }, + Characters = [new SceneIntelligenceCharacter { Name = "Iris Vale", RoleInScene = "pointOfView", MentionedOnly = false, Aliases = ["Iris"], Actions = ["finds the archive key"], Confidence = 0.91m, Notes = null }], + CharacterAppearance = [new SceneIntelligenceCharacterAppearance { CanonicalName = "Iris Vale", Aliases = ["Iris"], Title = null, Presentation = "feminine", ExplicitAge = null, InferredAgeBand = "adult", HairColour = null, HairLength = null, HairStyle = null, EyeColour = null, FacialHair = null, Glasses = null, Height = null, Build = null, DistinctiveFeatures = [], Clothing = "raincoat", RoleOrOccupation = "archivist", RelationshipAgeEvidence = null, PronounEvidence = "she", Evidence = "Iris tightened her raincoat.", Confidence = 0.72m, EvidenceKind = "explicit" }], + Locations = [new SceneIntelligenceLocation { Name = "Clocktower Archive", LocationType = "named", GenericRoomType = null, ParentLocationHint = null, PresentInScene = true, MentionedOnly = false, Confidence = 0.9m, Notes = null }], + Assets = [new SceneIntelligenceAsset { Name = "brass archive key", AssetType = "key", Status = "found", OwnerOrHolder = "Iris Vale", MentionedOnly = false, Confidence = 0.87m, Notes = null }], + Relationships = [new SceneIntelligenceRelationship { CharacterA = "Iris Vale", CharacterB = "Jon Calder", RelationshipSignal = "trust", Evidence = "Iris trusts Jon with the key.", Confidence = 0.76m }], + KnowledgeChanges = [new SceneIntelligenceKnowledgeChange { RecipientCharacter = "Iris Vale", KnowledgeItem = "The archive key opens the sealed west cabinet.", ChangeType = "Discovers", SourceCharacter = null, SourceType = "document", Evidence = "The tag reads west cabinet.", Confidence = 0.83m }], + TimelineClues = [new SceneIntelligenceTimelineClue { Clue = "after midnight", RelativeOrder = "after the public lecture", AbsoluteDate = null, Evidence = "The clock had already struck twelve.", Confidence = 0.8m }], + NarrativeArcs = [new SceneIntelligenceNarrativeArc { Title = "The sealed cabinet", Description = "Iris moves closer to opening the sealed cabinet.", ArcType = "mystery", Scale = "secondary", ChangeType = "Developed", Significance = "moderate", ContinuityKey = "sealed cabinet", ParentArcTitle = null, MatchingExistingPlotLineId = null, MatchingExistingThreadId = null, Evidence = "The key is labelled for the west cabinet.", Confidence = 0.82m }], + QuestionsRaised = [new SceneIntelligenceQuestionRaised { Question = "What is hidden in the sealed cabinet?", Scope = "asset", Evidence = "The cabinet has been sealed for years.", Confidence = 0.78m }], + QuestionsAnswered = [new SceneIntelligenceQuestionAnswered { Question = "Where is the archive key?", Answer = "Iris finds it behind the ledger shelf.", Evidence = "Iris finds the key behind the ledger shelf.", Confidence = 0.9m }], + Observations = [new SceneIntelligenceObservation { ObservationType = "AssetInteraction", SubjectEntityType = "Character", SubjectName = "Iris Vale", ObjectEntityType = "Asset", ObjectName = "brass archive key", Predicate = "finds", Description = "Iris finds the brass archive key.", Evidence = "Iris finds the key behind the ledger shelf.", Confidence = 0.9m }], + MetricValues = [new SceneIntelligenceMetricValue { Key = "tension", Score = 7, Confidence = 0.74m }], + SourceLimits = new SceneIntelligenceSourceLimits { ContainsAmbiguity = false, AmbiguityNotes = [] } + }; + +static JsonElement SceneSchemaRoot() + => JsonSerializer.SerializeToElement(StoryIntelligenceSceneStructuredOutputSchema.Contract.Schema); + +static JsonElement ArrayItem(JsonElement arraySchema) + => arraySchema.GetProperty("items"); + +static void AssertConfidence(JsonElement confidenceSchema, string path) +{ + Assert(confidenceSchema.GetProperty("type").GetString() == "number", $"{path} should be a number."); + Assert(confidenceSchema.GetProperty("minimum").GetInt32() == 0, $"{path} should have minimum 0."); + Assert(confidenceSchema.GetProperty("maximum").GetInt32() == 1, $"{path} should have maximum 1."); +} + +static void AssertString(JsonElement stringSchema, string path, bool nullable = false) +{ + if (!nullable) + { + Assert(stringSchema.GetProperty("type").GetString() == "string", $"{path} should be a string."); + return; + } + + var types = stringSchema.GetProperty("type").EnumerateArray().Select(item => item.GetString()).ToHashSet(StringComparer.Ordinal); + Assert(types.SetEquals(["string", "null"]), $"{path} should be nullable string."); +} + static void StoryIntelligenceNarrativeReviewCreatesCanonicalRecordsViaExistingProcs() { var root = Path.Combine(AppContext.BaseDirectory, "../../../../PlotLine"); diff --git a/PlotLine/Docs/AI/Scene-Prompt-V2.md b/PlotLine/Docs/AI/Scene-Prompt-V2.md index dbab588..b1fe208 100644 --- a/PlotLine/Docs/AI/Scene-Prompt-V2.md +++ b/PlotLine/Docs/AI/Scene-Prompt-V2.md @@ -162,7 +162,7 @@ Use the `configuredMetrics` array supplied in `sceneContext`. Do not invent a se For each configured metric: -- use the exact `key` supplied in `sceneContext` as the property name in `metrics`; +- use the exact `key` supplied in `sceneContext` as the `key` value in `metricValues`; - use the supplied `name`, `description`, `minValue`, `maxValue` and `defaultValue` to understand the metric; - score only what occurs in this scene, not imagined future consequences; - treat the scale comparatively across scenes in the book; @@ -172,15 +172,11 @@ For each configured metric: - use the high range for dominant/intense presence; - use the configured maximum only for exceptional/extreme presence. -If no configured metrics are supplied, return an empty `metrics` object. +If no configured metrics are supplied, return an empty `metricValues` array. -Each metric must be an object: +Each metric value must contain `key`, `score` and `confidence`. -```json -{ "score": 8, "confidence": 0.75 } -``` - -Always emit every configured metric. If a metric is barely present, use the configured minimum value with an appropriate confidence. Do not omit configured metric objects. +Always emit every configured metric. If a metric is barely present, use the configured minimum value with an appropriate confidence. Do not omit configured metric values. ## Character Reporting Rules @@ -217,7 +213,7 @@ Mentioned-only characters matter because later stages need to know who is refere Scene understanding is not enough for illustration matching. Report explicit or strongly supported visual facts in `characterAppearance`. -For every named or recurring character with supported appearance evidence, include exactly these properties: `canonicalName`, `aliases`, `title`, `presentation`, `explicitAge`, `inferredAgeBand`, `hairColour`, `hairLength`, `hairStyle`, `facialHair`, `glasses`, `build`, `distinctiveFeatures`, `clothing`, `roleOrOccupation`, `relationshipAgeEvidence`, `pronounEvidence`, `evidence`, `confidence`, `evidenceKind`. +For every named or recurring character with supported appearance evidence, include exactly these properties: `canonicalName`, `aliases`, `title`, `presentation`, `explicitAge`, `inferredAgeBand`, `hairColour`, `hairLength`, `hairStyle`, `eyeColour`, `facialHair`, `glasses`, `height`, `build`, `distinctiveFeatures`, `clothing`, `roleOrOccupation`, `relationshipAgeEvidence`, `pronounEvidence`, `evidence`, `confidence`, `evidenceKind`. Rules: @@ -581,192 +577,11 @@ Incorrect: ## JSON Output Requirements -Return exactly one JSON object matching this shape: +Return exactly one JSON object that satisfies the API-provided `scene_intelligence_v1` structured-output schema. -```json -{ - "schemaVersion": "1.0", - "sceneReference": { - "projectId": null, - "bookId": null, - "chapterId": null, - "sceneId": null, - "chapterNumber": null, - "sceneNumber": null, - "sourceLabel": "" - }, - "scenePurpose": { - "observedFunction": "", - "confidence": 0.0 - }, - "pointOfView": { - "characterName": null, - "narrativeMode": null, - "confidence": 0.0, - "evidence": null - }, - "setting": { - "timeOfDay": null, - "dateOrTimeReference": null, - "locationName": null, - "locationType": "unclear", - "genericRoomType": null, - "parentLocationHint": null, - "confidence": 0.0 - }, - "characters": [ - { - "name": "", - "roleInScene": "unclear", - "mentionedOnly": false, - "aliases": [], - "actions": [], - "confidence": 0.0, - "notes": null - } - ], - "characterAppearance": [ - { - "canonicalName": "", - "aliases": [], - "title": null, - "presentation": null, - "explicitAge": null, - "inferredAgeBand": null, - "hairColour": null, - "hairLength": null, - "hairStyle": null, - "facialHair": null, - "glasses": null, - "build": null, - "distinctiveFeatures": [], - "clothing": null, - "roleOrOccupation": null, - "relationshipAgeEvidence": null, - "pronounEvidence": null, - "evidence": "", - "confidence": 0.0, - "evidenceKind": "explicit" - } - ], - "locations": [ - { - "name": "", - "locationType": "unclear", - "genericRoomType": null, - "parentLocationHint": null, - "presentInScene": false, - "mentionedOnly": false, - "confidence": 0.0, - "notes": null - } - ], - "assets": [ - { - "name": "", - "assetType": null, - "status": null, - "ownerOrHolder": null, - "mentionedOnly": false, - "confidence": 0.0, - "notes": null - } - ], - "relationships": [ - { - "characterA": "", - "characterB": "", - "relationshipSignal": "", - "evidence": "", - "confidence": 0.0 - } - ], - "knowledgeChanges": [ - { - "recipientCharacter": "", - "knowledgeItem": "", - "changeType": "Learns", - "sourceCharacter": null, - "sourceType": null, - "evidence": "", - "confidence": 0.0 - } - ], - "timelineClues": [ - { - "clue": "", - "relativeOrder": null, - "absoluteDate": null, - "evidence": "", - "confidence": 0.0 - } - ], - "narrativeArcs": [ - { - "title": "", - "description": "", - "arcType": "other", - "scale": "major", - "changeType": "Introduced", - "significance": "major", - "continuityKey": "", - "parentArcTitle": null, - "matchingExistingPlotLineId": null, - "matchingExistingThreadId": null, - "evidence": "", - "confidence": 0.0 - } - ], - "questionsRaised": [ - { - "question": "", - "scope": "plot", - "evidence": "", - "confidence": 0.0 - } - ], - "questionsAnswered": [ - { - "question": "", - "answer": "", - "evidence": "", - "confidence": 0.0 - } - ], - "observations": [ - { - "observationType": "CharacterAction", - "subjectEntityType": "Character", - "subjectName": "", - "objectEntityType": null, - "objectName": null, - "predicate": "observes", - "description": "", - "evidence": "", - "confidence": 0.0 - } - ], - "metrics": { - "configuredMetricKey": { "score": 1, "confidence": 0.0 } - }, - "sourceLimits": { - "containsAmbiguity": false, - "ambiguityNotes": [] - } -} -``` +Use `[]` for any array with no supported items. Never output placeholder objects with empty required strings. -The example arrays above show object shape. In the actual response, use `[]` for any array with no supported items. Never output placeholder objects with empty required strings. - -All arrays must be present. Every object in an array must include every property shown for that object type. Every required object must be present. Unknown scalar values must be `null` only where the schema permits null. Unknown lists must be `[]`. Do not omit properties. - -Use valid JSON: - -- double quotes for property names and strings; -- no trailing commas; -- no comments; -- no Markdown; -- no prose outside JSON. +All arrays must be present. Every object in an array must include every schema property for that object type. Every required object must be present. Unknown scalar values must be `null` only where the schema permits null. Unknown lists must be `[]`. Do not omit properties. ## Runtime Data Insertion diff --git a/PlotLine/Models/SceneIntelligenceModels.cs b/PlotLine/Models/SceneIntelligenceModels.cs index d783d9e..33bc4ff 100644 --- a/PlotLine/Models/SceneIntelligenceModels.cs +++ b/PlotLine/Models/SceneIntelligenceModels.cs @@ -23,6 +23,7 @@ public sealed class SceneIntelligenceScene public List? QuestionsAnswered { get; init; } public List? Observations { get; init; } public Dictionary? Metrics { get; init; } + public List? MetricValues { get; init; } public SceneIntelligenceSourceLimits? SourceLimits { get; init; } [JsonExtensionData] @@ -263,6 +264,16 @@ public sealed class SceneIntelligenceMetric public Dictionary? ExtensionData { get; init; } } +public sealed class SceneIntelligenceMetricValue +{ + public string? Key { get; init; } + public int? Score { get; init; } + public decimal? Confidence { get; init; } + + [JsonExtensionData] + public Dictionary? ExtensionData { get; init; } +} + public sealed class SceneIntelligenceSourceLimits { public bool? ContainsAmbiguity { get; init; } diff --git a/PlotLine/Models/StoryIntelligenceModels.cs b/PlotLine/Models/StoryIntelligenceModels.cs index 5effbf9..4fd7631 100644 --- a/PlotLine/Models/StoryIntelligenceModels.cs +++ b/PlotLine/Models/StoryIntelligenceModels.cs @@ -236,3 +236,10 @@ public sealed class StoryIntelligenceClientResult public int? OutputTokens { get; init; } public decimal? EstimatedCost { get; init; } } + +public sealed class StoryIntelligenceResponseContract +{ + public string Name { get; init; } = string.Empty; + public object Schema { get; init; } = new(); + public bool Strict { get; init; } = true; +} diff --git a/PlotLine/Services/PersistedStoryIntelligenceRunner.cs b/PlotLine/Services/PersistedStoryIntelligenceRunner.cs index 8d1392e..47bc436 100644 --- a/PlotLine/Services/PersistedStoryIntelligenceRunner.cs +++ b/PlotLine/Services/PersistedStoryIntelligenceRunner.cs @@ -236,7 +236,8 @@ public sealed class PersistedStoryIntelligenceRunner( scenePromptVersion, cancellationToken, stageModels.SceneIntelligenceModel, - SceneIntelligenceMaxOutputTokens()); + SceneIntelligenceMaxOutputTokens(), + StoryIntelligenceSceneStructuredOutputSchema.Contract); openAiStopwatch.Stop(); totals.Add(sceneClientResult, pricing); var parseStopwatch = Stopwatch.StartNew(); diff --git a/PlotLine/Services/StoryIntelligenceImportCommitService.cs b/PlotLine/Services/StoryIntelligenceImportCommitService.cs index 8198b48..e094a8b 100644 --- a/PlotLine/Services/StoryIntelligenceImportCommitService.cs +++ b/PlotLine/Services/StoryIntelligenceImportCommitService.cs @@ -462,14 +462,15 @@ public sealed class StoryIntelligenceImportCommitService( private static IReadOnlyList BuildMetrics(SceneIntelligenceScene parsed, IReadOnlyDictionary metricMap) { - if (parsed.Metrics is null || parsed.Metrics.Count == 0) + var metrics = ReadMetrics(parsed); + if (metrics.Count == 0) { return []; } var values = new List(); var importedMetricTypeIds = new HashSet(); - foreach (var (name, metric) in parsed.Metrics) + foreach (var (name, metric) in metrics) { if (!metric.Score.HasValue || !metricMap.TryGetValue(name, out var metricType)) { @@ -493,6 +494,36 @@ public sealed class StoryIntelligenceImportCommitService( return values; } + private static IReadOnlyDictionary ReadMetrics(SceneIntelligenceScene parsed) + { + if (parsed.Metrics is { Count: > 0 }) + { + return parsed.Metrics; + } + + if (parsed.MetricValues is null || parsed.MetricValues.Count == 0) + { + return new Dictionary(StringComparer.OrdinalIgnoreCase); + } + + var metrics = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var metric in parsed.MetricValues) + { + if (string.IsNullOrWhiteSpace(metric.Key)) + { + continue; + } + + metrics[metric.Key.Trim()] = new SceneIntelligenceMetric + { + Score = metric.Score, + Confidence = metric.Confidence + }; + } + + return metrics; + } + private static IReadOnlyList MatchPurposes(SceneIntelligenceScene parsed, IReadOnlyDictionary purposeMap) { var values = ReadPurposeTexts(parsed).Select(Clean).Where(value => !string.IsNullOrWhiteSpace(value)).Distinct(StringComparer.OrdinalIgnoreCase).ToList(); diff --git a/PlotLine/Services/StoryIntelligenceOpenAIInfrastructure.cs b/PlotLine/Services/StoryIntelligenceOpenAIInfrastructure.cs index cd38d74..66a0f1f 100644 --- a/PlotLine/Services/StoryIntelligenceOpenAIInfrastructure.cs +++ b/PlotLine/Services/StoryIntelligenceOpenAIInfrastructure.cs @@ -33,7 +33,8 @@ public interface IStoryIntelligenceClient string promptVersion, CancellationToken cancellationToken, string? modelOverride = null, - int? maxOutputTokensOverride = null); + int? maxOutputTokensOverride = null, + StoryIntelligenceResponseContract? responseContract = null); StoryIntelligenceClientConfigurationStatus GetConfigurationStatus(); } @@ -168,7 +169,8 @@ public sealed class StoryIntelligenceClient( string promptVersion, CancellationToken cancellationToken, string? modelOverride = null, - int? maxOutputTokensOverride = null) + int? maxOutputTokensOverride = null, + StoryIntelligenceResponseContract? responseContract = null) { if (string.IsNullOrWhiteSpace(settings.ApiKey)) { @@ -195,6 +197,20 @@ public sealed class StoryIntelligenceClient( maxOutputTokens = settings.MaxOutputTokens; } + var textOptions = responseContract is null + ? new Dictionary { ["verbosity"] = "low" } + : new Dictionary + { + ["verbosity"] = "low", + ["format"] = new + { + type = "json_schema", + name = responseContract.Name, + strict = responseContract.Strict, + schema = responseContract.Schema + } + }; + var requestBody = JsonSerializer.Serialize(new { model = model, @@ -205,16 +221,14 @@ public sealed class StoryIntelligenceClient( { effort = "minimal" }, - text = new - { - verbosity = "low" - } + text = textOptions }, JsonOptions); var stopwatch = Stopwatch.StartNew(); try { var (responseText, retryCount) = await SendWithRetryAsync(requestBody, cancellationToken); + EnsureUsableResponse(responseText, responseContract); stopwatch.Stop(); logger.LogInformation( "Story Intelligence prompt executed. PromptVersion={PromptVersion} Model={Model} DurationMs={DurationMs} RetryCount={RetryCount} InputTokens={InputTokens} OutputTokens={OutputTokens} EstimatedCost={EstimatedCost} Success={Success}", @@ -358,6 +372,33 @@ public sealed class StoryIntelligenceClient( return (null, null); } } + + private static void EnsureUsableResponse(string responseText, StoryIntelligenceResponseContract? responseContract) + { + if (responseContract is null) + { + return; + } + + var response = JsonSerializer.Deserialize(responseText, JsonOptions) + ?? throw new InvalidOperationException("OpenAI structured response envelope was empty."); + if (!string.IsNullOrWhiteSpace(response.Status) + && !string.Equals(response.Status, "completed", StringComparison.OrdinalIgnoreCase)) + { + var detail = response.IncompleteDetails?.Reason ?? response.Error?.Message ?? response.Status; + throw new InvalidOperationException($"OpenAI structured response did not complete: {detail}"); + } + + var refusal = response.Output? + .SelectMany(item => item.Content ?? []) + .FirstOrDefault(content => string.Equals(content.Type, "refusal", StringComparison.OrdinalIgnoreCase) + || !string.IsNullOrWhiteSpace(content.Refusal)) + ?.Refusal; + if (!string.IsNullOrWhiteSpace(refusal)) + { + throw new InvalidOperationException($"OpenAI structured response was refused: {TrimForLog(refusal)}"); + } + } } public sealed class StoryIntelligenceDiagnosticsService( @@ -447,7 +488,8 @@ public sealed class StoryIntelligenceDryRunService( completedPrompt, promptVersion, cancellationToken, - clientStatus.SceneIntelligenceModel); + clientStatus.SceneIntelligenceModel, + responseContract: StoryIntelligenceSceneStructuredOutputSchema.Contract); var sceneJson = ExtractSceneJson(result.RawResponseText); var parsedScene = DeserializeScene(sceneJson); var validation = validator.Validate(parsedScene); @@ -742,8 +784,14 @@ public sealed class ChapterStructureDryRunService( internal sealed class OpenAIResponseEnvelope { + public string? Status { get; init; } public List? Output { get; init; } public OpenAIResponseUsage? Usage { get; init; } + + [JsonPropertyName("incomplete_details")] + public OpenAIResponseIncompleteDetails? IncompleteDetails { get; init; } + + public OpenAIResponseError? Error { get; init; } } internal sealed class OpenAIResponseOutputItem @@ -755,6 +803,7 @@ internal sealed class OpenAIResponseContentItem { public string? Type { get; init; } public string? Text { get; init; } + public string? Refusal { get; init; } } internal sealed class OpenAIResponseUsage @@ -766,6 +815,16 @@ internal sealed class OpenAIResponseUsage public int? OutputTokens { get; init; } } +internal sealed class OpenAIResponseIncompleteDetails +{ + public string? Reason { get; init; } +} + +internal sealed class OpenAIResponseError +{ + public string? Message { get; init; } +} + public sealed class StoryIntelligenceClientConfigurationStatus { public bool ApiKeyConfigured { get; init; } diff --git a/PlotLine/Services/StoryIntelligenceSceneStructuredOutputSchema.cs b/PlotLine/Services/StoryIntelligenceSceneStructuredOutputSchema.cs new file mode 100644 index 0000000..24b697a --- /dev/null +++ b/PlotLine/Services/StoryIntelligenceSceneStructuredOutputSchema.cs @@ -0,0 +1,210 @@ +using PlotLine.Models; + +namespace PlotLine.Services; + +public static class StoryIntelligenceSceneStructuredOutputSchema +{ + public const string ContractName = "scene_intelligence_v1"; + + public static StoryIntelligenceResponseContract Contract { get; } = new() + { + Name = ContractName, + Strict = true, + Schema = BuildSchema() + }; + + private static Dictionary BuildSchema() + { + var sceneReference = Obj(Props( + ("projectId", NullableInteger()), + ("bookId", NullableInteger()), + ("chapterId", NullableInteger()), + ("sceneId", NullableInteger()), + ("chapterNumber", NullableNumber()), + ("sceneNumber", NullableNumber()), + ("sourceLabel", Str()))); + var purpose = Obj(Props(("observedFunction", Str()), ("confidence", Confidence()))); + var pointOfView = Obj(Props( + ("characterName", NullableString()), + ("narrativeMode", NullableString()), + ("confidence", Confidence()), + ("evidence", NullableString()))); + var setting = Obj(Props( + ("timeOfDay", NullableString()), + ("dateOrTimeReference", NullableString()), + ("locationName", NullableString()), + ("locationType", Enum("named", "generic", "unclear")), + ("genericRoomType", NullableString()), + ("parentLocationHint", NullableString()), + ("confidence", Confidence()))); + + var characters = Arr(Obj(Props( + ("name", Str()), + ("roleInScene", Enum("present", "pointOfView", "speaker", "mentioned", "unclear")), + ("mentionedOnly", Bool()), + ("aliases", Arr(Str())), + ("actions", Arr(Str())), + ("confidence", Confidence()), + ("notes", NullableString())))); + var characterAppearance = Arr(Obj(Props( + ("canonicalName", Str()), + ("aliases", Arr(Str())), + ("title", NullableString()), + ("presentation", NullableString()), + ("explicitAge", NullableString()), + ("inferredAgeBand", NullableString()), + ("hairColour", NullableString()), + ("hairLength", NullableString()), + ("hairStyle", NullableString()), + ("eyeColour", NullableString()), + ("facialHair", NullableString()), + ("glasses", NullableString()), + ("height", NullableString()), + ("build", NullableString()), + ("distinctiveFeatures", Arr(Str())), + ("clothing", NullableString()), + ("roleOrOccupation", NullableString()), + ("relationshipAgeEvidence", NullableString()), + ("pronounEvidence", NullableString()), + ("evidence", Str()), + ("confidence", Confidence()), + ("evidenceKind", Enum("explicit", "inferred"))))); + var locations = Arr(Obj(Props( + ("name", Str()), + ("locationType", Enum("named", "generic", "unclear")), + ("genericRoomType", NullableString()), + ("parentLocationHint", NullableString()), + ("presentInScene", Bool()), + ("mentionedOnly", Bool()), + ("confidence", Confidence()), + ("notes", NullableString())))); + var assets = Arr(Obj(Props( + ("name", Str()), + ("assetType", NullableString()), + ("status", NullableString()), + ("ownerOrHolder", NullableString()), + ("mentionedOnly", Bool()), + ("confidence", Confidence()), + ("notes", NullableString())))); + var relationships = Arr(Obj(Props( + ("characterA", Str()), + ("characterB", Str()), + ("relationshipSignal", Enum("trust", "mistrust", "affection", "fear", "authority", "secrecy", "conflict", "dependence", "protection", "rivalry", "obligation")), + ("evidence", Str()), + ("confidence", Confidence())))); + var knowledgeChanges = Arr(Obj(Props( + ("recipientCharacter", Str()), + ("knowledgeItem", Str()), + ("changeType", Enum("Learns", "Realises", "IsTold", "Discovers", "Confirms", "Suspects", "Misunderstands")), + ("sourceCharacter", NullableString()), + ("sourceType", NullableString()), + ("evidence", Str()), + ("confidence", Confidence())))); + var timelineClues = Arr(Obj(Props( + ("clue", Str()), + ("relativeOrder", NullableString()), + ("absoluteDate", NullableString()), + ("evidence", Str()), + ("confidence", Confidence())))); + var narrativeArcs = Arr(Obj(Props( + ("title", Str()), + ("description", Str()), + ("arcType", Str()), + ("scale", Enum("major", "secondary", "thread")), + ("changeType", Enum("Introduced", "Developed", "Complicated", "Revealed", "Escalated", "Resolved", "Reopened", "Mentioned")), + ("significance", Enum("major", "moderate", "minor")), + ("continuityKey", Str()), + ("parentArcTitle", NullableString()), + ("matchingExistingPlotLineId", NullableInteger()), + ("matchingExistingThreadId", NullableInteger()), + ("evidence", Str()), + ("confidence", Confidence())))); + var questionsRaised = Arr(Obj(Props(("question", Str()), ("scope", Str()), ("evidence", Str()), ("confidence", Confidence())))); + var questionsAnswered = Arr(Obj(Props(("question", Str()), ("answer", Str()), ("evidence", Str()), ("confidence", Confidence())))); + var observations = Arr(Obj(Props( + ("observationType", Enum("CharacterAction", "CharacterKnowledge", "CharacterLocation", "RelationshipSignal", "AssetInteraction", "LocationSignal", "TimelineSignal", "QuestionSignal")), + ("subjectEntityType", Enum("Character", "Location", "Asset", "Knowledge", "Relationship", "Timeline", "Scene", "Unknown")), + ("subjectName", Str()), + ("objectEntityType", NullableEnum("Character", "Location", "Asset", "Knowledge", "Relationship", "Timeline", "Scene", "Unknown")), + ("objectName", NullableString()), + ("predicate", Str()), + ("description", Str()), + ("evidence", Str()), + ("confidence", Confidence())))); + var metricValues = Arr(Obj(Props( + ("key", Str("Configured metric key from sceneContext.configuredMetrics.")), + ("score", Integer()), + ("confidence", Confidence())))); + var sourceLimits = Obj(Props(("containsAmbiguity", Bool()), ("ambiguityNotes", Arr(Str())))); + + return Obj(Props( + ("schemaVersion", Str("Scene Intelligence schema version. Use 1.0.")), + ("sceneReference", sceneReference), + ("scenePurpose", purpose), + ("pointOfView", pointOfView), + ("setting", setting), + ("characters", characters), + ("characterAppearance", characterAppearance), + ("locations", locations), + ("assets", assets), + ("relationships", relationships), + ("knowledgeChanges", knowledgeChanges), + ("timelineClues", timelineClues), + ("narrativeArcs", narrativeArcs), + ("questionsRaised", questionsRaised), + ("questionsAnswered", questionsAnswered), + ("observations", observations), + ("metricValues", metricValues), + ("sourceLimits", sourceLimits))); + } + + private static Dictionary Obj(Dictionary properties) + => new() + { + ["type"] = "object", + ["properties"] = properties, + ["required"] = properties.Keys.ToArray(), + ["additionalProperties"] = false + }; + + private static Dictionary Arr(object items) + => new() { ["type"] = "array", ["items"] = items }; + + private static Dictionary Str(string? description = null) + { + var value = new Dictionary { ["type"] = "string" }; + if (!string.IsNullOrWhiteSpace(description)) + { + value["description"] = description; + } + + return value; + } + + private static Dictionary NullableString() + => new() { ["type"] = new[] { "string", "null" } }; + + private static Dictionary Integer() + => new() { ["type"] = "integer" }; + + private static Dictionary NullableInteger() + => new() { ["type"] = new[] { "integer", "null" } }; + + private static Dictionary NullableNumber() + => new() { ["type"] = new[] { "number", "null" } }; + + private static Dictionary Bool() + => new() { ["type"] = "boolean" }; + + private static Dictionary Confidence() + => new() { ["type"] = "number", ["minimum"] = 0, ["maximum"] = 1 }; + + private static Dictionary Enum(params string[] values) + => new() { ["type"] = "string", ["enum"] = values }; + + private static Dictionary NullableEnum(params string[] values) + => new() { ["type"] = new[] { "string", "null" }, ["enum"] = values.Cast().Append(null).ToArray() }; + + private static Dictionary Props(params (string Name, object Schema)[] properties) + => properties.ToDictionary(property => property.Name, property => property.Schema, StringComparer.Ordinal); +} diff --git a/PlotLine/Services/StorySceneValidator.cs b/PlotLine/Services/StorySceneValidator.cs index 64b5acf..4dd4959 100644 --- a/PlotLine/Services/StorySceneValidator.cs +++ b/PlotLine/Services/StorySceneValidator.cs @@ -47,6 +47,7 @@ public sealed class StorySceneValidator : IStorySceneValidator ValidateQuestionsAnswered(scene.QuestionsAnswered, result); ValidateObservations(scene.Observations, result); ValidateMetrics(scene.Metrics, result); + ValidateMetricValues(scene.MetricValues, result); ValidateSourceLimits(scene.SourceLimits, result); if (result.IsValid) @@ -84,7 +85,10 @@ public sealed class StorySceneValidator : IStorySceneValidator RequiredArray(scene.QuestionsRaised, "questionsRaised", result); RequiredArray(scene.QuestionsAnswered, "questionsAnswered", result); RequiredArray(scene.Observations, "observations", result); - RequiredObject(scene.Metrics, "metrics", result); + if (scene.Metrics is null && scene.MetricValues is null) + { + AddError(result, "metrics", "Metric output is required.", "Return the legacy metrics object or strict metricValues array."); + } RequiredObject(scene.SourceLimits, "sourceLimits", result); Unknown(scene.ExtensionData, "$", result); } @@ -442,6 +446,34 @@ public sealed class StorySceneValidator : IStorySceneValidator } } + private static void ValidateMetricValues(List? values, ValidationResult result) + { + if (values is null) + { + return; + } + + var seen = new HashSet(StringComparer.OrdinalIgnoreCase); + for (var i = 0; i < values.Count; i++) + { + var item = values[i]; + var path = $"metricValues[{i}]"; + RequiredString(item.Key, $"{path}.key", result); + if (!string.IsNullOrWhiteSpace(item.Key) && !seen.Add(item.Key.Trim())) + { + AddWarning(result, path, "Duplicate metric key found.", "Return one metric value per configured metric key."); + } + + if (!item.Score.HasValue) + { + AddError(result, $"{path}.score", "Metric score is required.", "Return a numeric score using the configured metric scale."); + } + + Confidence(item.Confidence, $"{path}.confidence", result); + Unknown(item.ExtensionData, path, result); + } + } + private static void ValidateSourceLimits(SceneIntelligenceSourceLimits? value, ValidationResult result) { if (value is null)