Promote character enrichment structured facts

PlotLine.Tests/Program.cs

@@ -339,6 +339,7 @@ var tests = new (string Name, Action Test)[]
     ("Story Intelligence character enrichment uses persisted SignalR progress", StoryIntelligenceCharacterEnrichmentUsesPersistedSignalRProgress),
     ("Story Intelligence character enrichment preserves author fields", StoryIntelligenceCharacterEnrichmentPreservesAuthorFields),
     ("Story Intelligence character enrichment batches by prompt size", StoryIntelligenceCharacterEnrichmentBatchesByPromptSize),
+    ("Story Intelligence character enrichment promotes structured facts", StoryIntelligenceCharacterEnrichmentPromotesStructuredFacts),
     ("Character detail can queue single-character enrichment", CharacterDetailCanQueueSingleCharacterEnrichment),
     ("Single-character enrichment is persisted and duplicate safe", SingleCharacterEnrichmentIsPersistedAndDuplicateSafe),
     ("Character detail enrichment status uses scoped SignalR", CharacterDetailEnrichmentStatusUsesScopedSignalR),
@@ -1271,6 +1272,28 @@ static void StoryIntelligenceCharacterEnrichmentBatchesByPromptSize()
         && service.Contains("IncludedScenes", StringComparison.Ordinal), "Restrained character context should be logged with auditable scene counts.");
 }
 
+static void StoryIntelligenceCharacterEnrichmentPromotesStructuredFacts()
+{
+    var root = PlotLineRoot();
+    var models = File.ReadAllText(Path.Combine(root, "Models/CharacterEnrichmentModels.cs"));
+    var service = File.ReadAllText(Path.Combine(root, "Services/CharacterEnrichmentService.cs"));
+    var migration = File.ReadAllText(Path.Combine(root, "Sql/188_Phase27O_CharacterEnrichmentStructuredFacts.sql"));
+
+    Assert(models.Contains("CharacterEnrichmentFactResult Sex", StringComparison.Ordinal), "Character enrichment should extract sex as an evidenced structured fact.");
+    Assert(service.Contains("Do not place ascertainable facts only in the summary", StringComparison.Ordinal), "Prompt should require structured facts as well as summary prose.");
+    Assert(service.Contains("NormaliseAgeFact", StringComparison.Ordinal)
+        && service.Contains("NormaliseDateOfBirth", StringComparison.Ordinal)
+        && service.Contains("NormaliseSexFact", StringComparison.Ordinal), "Character enrichment should normalise structured facts before persistence.");
+    Assert(service.Contains("\"sex\"", StringComparison.Ordinal)
+        && service.Contains("\"ageAtStartOfSeries\"", StringComparison.Ordinal), "Structured output schema should include sex and age facts.");
+    Assert(migration.Contains("SexValueID = CASE", StringComparison.Ordinal)
+        && migration.Contains("N'Sex'", StringComparison.Ordinal), "SQL completion should promote evidenced sex without overwriting author values.");
+    Assert(migration.Contains("DELETE evidence", StringComparison.Ordinal)
+        && migration.Contains("INNER JOIN @Results", StringComparison.Ordinal), "SQL completion should replace same-run evidence idempotently.");
+    Assert(migration.Contains("BEGIN TRANSACTION", StringComparison.Ordinal)
+        && migration.Contains("COMMIT TRANSACTION", StringComparison.Ordinal), "Character enrichment claim and completion should be transactional.");
+}
+
 static void CharacterDetailCanQueueSingleCharacterEnrichment()
 {
     var root = PlotLineRoot();

PlotLine/Models/CharacterEnrichmentModels.cs

@@ -70,6 +70,7 @@ public sealed class CharacterEnrichmentCharacterStatus
     public string? Summary { get; init; }
     public string? DateOfBirth { get; init; }
     public string? AgeAtStartOfSeries { get; init; }
+    public string? Sex { get; init; }
     public string? Height { get; init; }
     public string? EyeColour { get; init; }
     public DateTime UpdatedUtc { get; init; } = DateTime.UtcNow;
@@ -148,6 +149,7 @@ public sealed class CharacterEnrichmentCharacterResult
     public decimal Confidence { get; init; }
     public CharacterEnrichmentFactResult DateOfBirth { get; init; } = new();
     public CharacterEnrichmentFactResult AgeAtStartOfSeries { get; init; } = new();
+    public CharacterEnrichmentFactResult Sex { get; init; } = new();
     public CharacterEnrichmentFactResult Height { get; init; } = new();
     public CharacterEnrichmentFactResult EyeColour { get; init; } = new();
     public IReadOnlyList<int> EvidenceSceneIDs { get; init; } = [];

PlotLine/Services/CharacterEnrichmentService.cs

@@ -1,7 +1,9 @@
 using System.Diagnostics;
+using System.Globalization;
 using System.Text;
 using System.Text.Json;
 using System.Text.Json.Serialization;
+using System.Text.RegularExpressions;
 using Microsoft.Extensions.Options;
 using PlotLine.Data;
 using PlotLine.Models;
@@ -37,6 +39,16 @@ public sealed class CharacterEnrichmentService(
     private const int MinimumSingleCharacterScenes = 8;
     private const decimal SummaryMinimumConfidence = 0.65m;
     private const decimal FactMinimumConfidence = 0.9m;
+    private static readonly Regex LeadingAgeRegex = new(@"\b(?<age>\d{1,3})\b", RegexOptions.Compiled);
+    private static readonly Regex BirthdayAgeDateRegex = new(@"\b(?<age>\d{1,3})(?:st|nd|rd|th)?\s+(?:birthday|on)\s+(?:on\s+)?(?<date>\d{1,2}(?:st|nd|rd|th)?\s+[A-Za-z]+\s+\d{4})", RegexOptions.Compiled | RegexOptions.IgnoreCase);
+    private static readonly string[] DateFormats =
+    [
+        "d MMMM yyyy",
+        "dd MMMM yyyy",
+        "d MMM yyyy",
+        "dd MMM yyyy",
+        "yyyy-MM-dd"
+    ];
     private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web)
     {
         WriteIndented = true
@@ -411,8 +423,9 @@ public sealed class CharacterEnrichmentService(
                 CharacterID = result.CharacterID,
                 Summary = Clamp(result.Confidence) >= SummaryMinimumConfidence ? Clean(result.Summary, 2500) : string.Empty,
                 Confidence = Clamp(result.Confidence),
-                DateOfBirth = StrongFact(result.DateOfBirth),
-                AgeAtStartOfSeries = StrongFact(result.AgeAtStartOfSeries),
+                DateOfBirth = NormaliseDateOfBirth(result.DateOfBirth, result.AgeAtStartOfSeries),
+                AgeAtStartOfSeries = NormaliseAgeFact(result.AgeAtStartOfSeries),
+                Sex = NormaliseSexFact(result.Sex),
                 Height = StrongFact(result.Height),
                 EyeColour = StrongFact(result.EyeColour),
                 EvidenceSceneIDs = result.EvidenceSceneIDs.Where(id => id > 0).Distinct().Take(20).ToList()
@@ -420,10 +433,110 @@ public sealed class CharacterEnrichmentService(
             .Where(result => !string.IsNullOrWhiteSpace(result.Summary)
                 || !string.IsNullOrWhiteSpace(result.DateOfBirth.Value)
                 || !string.IsNullOrWhiteSpace(result.AgeAtStartOfSeries.Value)
+                || !string.IsNullOrWhiteSpace(result.Sex.Value)
                 || !string.IsNullOrWhiteSpace(result.Height.Value)
                 || !string.IsNullOrWhiteSpace(result.EyeColour.Value))
             .ToList();
 
+    private static CharacterEnrichmentFactResult NormaliseDateOfBirth(
+        CharacterEnrichmentFactResult dateOfBirth,
+        CharacterEnrichmentFactResult ageAtStartOfSeries)
+    {
+        var direct = StrongFact(dateOfBirth);
+        if (!string.IsNullOrWhiteSpace(direct.Value))
+        {
+            var parsed = ParseDate(direct.Value);
+            return parsed.HasValue
+                ? CopyFact(direct, parsed.Value.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture))
+                : direct;
+        }
+
+        if (Clamp(ageAtStartOfSeries.Confidence) < FactMinimumConfidence || ageAtStartOfSeries.SceneID is not > 0)
+        {
+            return new CharacterEnrichmentFactResult();
+        }
+
+        var source = string.Join(" ", ageAtStartOfSeries.Value, ageAtStartOfSeries.Evidence);
+        var match = BirthdayAgeDateRegex.Match(source);
+        if (!match.Success
+            || !int.TryParse(match.Groups["age"].Value, NumberStyles.None, CultureInfo.InvariantCulture, out var age)
+            || age <= 0
+            || age > 120)
+        {
+            return new CharacterEnrichmentFactResult();
+        }
+
+        var birthday = ParseDate(match.Groups["date"].Value);
+        if (!birthday.HasValue)
+        {
+            return new CharacterEnrichmentFactResult();
+        }
+
+        return new CharacterEnrichmentFactResult
+        {
+            Value = birthday.Value.AddYears(-age).ToString("yyyy-MM-dd", CultureInfo.InvariantCulture),
+            Confidence = Clamp(ageAtStartOfSeries.Confidence),
+            Evidence = Clean($"Derived from age evidence: {ageAtStartOfSeries.Evidence}", 700),
+            SceneID = ageAtStartOfSeries.SceneID
+        };
+    }
+
+    private static CharacterEnrichmentFactResult NormaliseAgeFact(CharacterEnrichmentFactResult fact)
+    {
+        var strong = StrongFact(fact);
+        if (string.IsNullOrWhiteSpace(strong.Value))
+        {
+            return strong;
+        }
+
+        var match = LeadingAgeRegex.Match(strong.Value);
+        if (!match.Success
+            || !int.TryParse(match.Groups["age"].Value, NumberStyles.None, CultureInfo.InvariantCulture, out var age)
+            || age <= 0
+            || age > 120)
+        {
+            return new CharacterEnrichmentFactResult();
+        }
+
+        return CopyFact(strong, age.ToString(CultureInfo.InvariantCulture));
+    }
+
+    private static CharacterEnrichmentFactResult NormaliseSexFact(CharacterEnrichmentFactResult fact)
+    {
+        var strong = StrongFact(fact);
+        var value = strong.Value?.Trim();
+        if (string.IsNullOrWhiteSpace(value))
+        {
+            return new CharacterEnrichmentFactResult();
+        }
+
+        var normalised = value.Equals("female", StringComparison.OrdinalIgnoreCase) ? "Female"
+            : value.Equals("male", StringComparison.OrdinalIgnoreCase) ? "Male"
+            : value.Equals("other", StringComparison.OrdinalIgnoreCase) ? "Other"
+            : string.Empty;
+        return string.IsNullOrWhiteSpace(normalised)
+            ? new CharacterEnrichmentFactResult()
+            : CopyFact(strong, normalised);
+    }
+
+    private static CharacterEnrichmentFactResult CopyFact(CharacterEnrichmentFactResult fact, string value)
+        => new()
+        {
+            Value = value,
+            Confidence = fact.Confidence,
+            Evidence = fact.Evidence,
+            SceneID = fact.SceneID
+        };
+
+    private static DateTime? ParseDate(string? value)
+    {
+        var clean = Regex.Replace(value ?? string.Empty, @"\b(\d{1,2})(st|nd|rd|th)\b", "$1", RegexOptions.IgnoreCase).Trim();
+        return DateTime.TryParseExact(clean, DateFormats, CultureInfo.InvariantCulture, DateTimeStyles.None, out var parsed)
+            || DateTime.TryParse(clean, CultureInfo.GetCultureInfo("en-GB"), DateTimeStyles.None, out parsed)
+            ? parsed.Date
+            : null;
+    }
+
     private static CharacterEnrichmentFactResult StrongFact(CharacterEnrichmentFactResult fact)
         => Clamp(fact.Confidence) >= FactMinimumConfidence && !string.IsNullOrWhiteSpace(fact.Value) && fact.SceneID is > 0
             ? new CharacterEnrichmentFactResult
@@ -441,7 +554,11 @@ public sealed class CharacterEnrichmentService(
         builder.AppendLine("You are PlotDirector's Character Enrichment engine.");
         builder.AppendLine("Create concise author-facing character summaries and extract only strongly evidenced character facts.");
         builder.AppendLine("Use only the supplied manuscript scenes. Do not invent, infer from behaviour alone, speculate about future events, or list every appearance.");
+        builder.AppendLine("Do not place ascertainable facts only in the summary. When sex, date of birth, age at start of series, height, or eye colour can be ascertained from direct wording, populate the matching structured field too.");
+        builder.AppendLine("For sex, return Male, Female, or Other only when direct wording, pronouns, or relationship wording strongly supports it. Otherwise return null value and confidence 0.");
         builder.AppendLine("For dateOfBirth, ageAtStartOfSeries, height, and eyeColour, return a value only when direct manuscript wording strongly supports it. Otherwise return null value and confidence 0.");
+        builder.AppendLine("For ageAtStartOfSeries, prefer a plain integer value such as 18. Put dates or explanation in evidence, not in the value.");
+        builder.AppendLine("For dateOfBirth, prefer ISO yyyy-MM-dd when directly stated or safely derived from an explicit age and birthday date.");
         builder.AppendLine("If a character already has a stored value, preserve it by returning null unless the manuscript contains stronger direct evidence; the save layer will still refuse to overwrite existing author data.");
         builder.AppendLine();
         builder.AppendLine("[BOOK]");
@@ -553,10 +670,11 @@ public static class CharacterEnrichmentStructuredOutputSchema
                     ["confidence"] = NumberSchema(),
                     ["dateOfBirth"] = FactSchema(),
                     ["ageAtStartOfSeries"] = FactSchema(),
+                    ["sex"] = FactSchema(),
                     ["height"] = FactSchema(),
                     ["eyeColour"] = FactSchema(),
                     ["evidenceSceneIDs"] = ArraySchema(IntegerSchema())
-                }, "characterID", "summary", "confidence", "dateOfBirth", "ageAtStartOfSeries", "height", "eyeColour", "evidenceSceneIDs"))
+                }, "characterID", "summary", "confidence", "dateOfBirth", "ageAtStartOfSeries", "sex", "height", "eyeColour", "evidenceSceneIDs"))
             }
         }
     };

PlotLine/Sql/188_Phase27O_CharacterEnrichmentStructuredFacts.sql

@@ -0,0 +1,472 @@
+CREATE OR ALTER PROCEDURE dbo.CharacterEnrichmentRun_ClaimNext
+    @LeaseMinutes int = 90
+AS
+BEGIN
+    SET NOCOUNT ON;
+    SET XACT_ABORT ON;
+
+    DECLARE @RunID int;
+
+    BEGIN TRANSACTION;
+
+    SELECT TOP (1) @RunID = CharacterEnrichmentRunID
+    FROM dbo.CharacterEnrichmentRuns WITH (UPDLOCK, HOLDLOCK, READPAST)
+    WHERE IsCurrent = 1
+        AND (
+            Status = N'Queued'
+            OR (Status = N'Running' AND StartedUtc < DATEADD(minute, -ABS(@LeaseMinutes), SYSUTCDATETIME()))
+        )
+    ORDER BY QueuedUtc, CharacterEnrichmentRunID;
+
+    IF @RunID IS NULL
+    BEGIN
+        COMMIT TRANSACTION;
+        SELECT TOP (0) * FROM dbo.CharacterEnrichmentRuns;
+        RETURN;
+    END;
+
+    UPDATE dbo.CharacterEnrichmentRuns
+    SET Status = N'Running',
+        StartedUtc = SYSUTCDATETIME(),
+        AttemptCount = AttemptCount + 1,
+        CurrentStage = N'Preparing character analysis',
+        CurrentMessage = N'Preparing character analysis...',
+        UpdatedUtc = SYSUTCDATETIME()
+    WHERE CharacterEnrichmentRunID = @RunID
+        AND IsCurrent = 1
+        AND (
+            Status = N'Queued'
+            OR (Status = N'Running' AND StartedUtc < DATEADD(minute, -ABS(@LeaseMinutes), SYSUTCDATETIME()))
+        );
+
+    IF @@ROWCOUNT = 0
+    BEGIN
+        COMMIT TRANSACTION;
+        SELECT TOP (0) * FROM dbo.CharacterEnrichmentRuns;
+        RETURN;
+    END;
+
+    SELECT * FROM dbo.CharacterEnrichmentRuns WHERE CharacterEnrichmentRunID = @RunID;
+
+    COMMIT TRANSACTION;
+END;
+GO
+
+CREATE OR ALTER PROCEDURE dbo.CharacterEnrichmentRun_Complete
+    @CharacterEnrichmentRunID int,
+    @Model nvarchar(100),
+    @InputTokens int = NULL,
+    @OutputTokens int = NULL,
+    @TotalTokens int = NULL,
+    @DurationMs bigint = NULL,
+    @ResultsJson nvarchar(max)
+AS
+BEGIN
+    SET NOCOUNT ON;
+    SET XACT_ABORT ON;
+
+    DECLARE @ProjectID int, @BookID int, @PipelineID int;
+
+    DECLARE @Results TABLE
+    (
+        CharacterID int NOT NULL PRIMARY KEY,
+        Summary nvarchar(max) NULL,
+        Confidence decimal(5,4) NOT NULL,
+        DateOfBirthValue nvarchar(120) NULL,
+        DateOfBirthConfidence decimal(5,4) NULL,
+        DateOfBirthEvidence nvarchar(700) NULL,
+        DateOfBirthSceneID int NULL,
+        AgeValue nvarchar(120) NULL,
+        AgeConfidence decimal(5,4) NULL,
+        AgeEvidence nvarchar(700) NULL,
+        AgeSceneID int NULL,
+        SexValue nvarchar(120) NULL,
+        SexConfidence decimal(5,4) NULL,
+        SexEvidence nvarchar(700) NULL,
+        SexSceneID int NULL,
+        HeightValue nvarchar(120) NULL,
+        HeightConfidence decimal(5,4) NULL,
+        HeightEvidence nvarchar(700) NULL,
+        HeightSceneID int NULL,
+        EyeColourValue nvarchar(120) NULL,
+        EyeColourConfidence decimal(5,4) NULL,
+        EyeColourEvidence nvarchar(700) NULL,
+        EyeColourSceneID int NULL
+    );
+
+    INSERT @Results
+    SELECT
+        CharacterID,
+        NULLIF(LTRIM(RTRIM(Summary)), N''),
+        Confidence,
+        NULLIF(LTRIM(RTRIM(DateOfBirthValue)), N''),
+        DateOfBirthConfidence,
+        NULLIF(LTRIM(RTRIM(DateOfBirthEvidence)), N''),
+        DateOfBirthSceneID,
+        NULLIF(LTRIM(RTRIM(AgeValue)), N''),
+        AgeConfidence,
+        NULLIF(LTRIM(RTRIM(AgeEvidence)), N''),
+        AgeSceneID,
+        CASE
+            WHEN LOWER(NULLIF(LTRIM(RTRIM(SexValue)), N'')) = N'female' THEN N'Female'
+            WHEN LOWER(NULLIF(LTRIM(RTRIM(SexValue)), N'')) = N'male' THEN N'Male'
+            WHEN LOWER(NULLIF(LTRIM(RTRIM(SexValue)), N'')) = N'other' THEN N'Other'
+            ELSE NULL
+        END,
+        SexConfidence,
+        NULLIF(LTRIM(RTRIM(SexEvidence)), N''),
+        SexSceneID,
+        NULLIF(LTRIM(RTRIM(HeightValue)), N''),
+        HeightConfidence,
+        NULLIF(LTRIM(RTRIM(HeightEvidence)), N''),
+        HeightSceneID,
+        NULLIF(LTRIM(RTRIM(EyeColourValue)), N''),
+        EyeColourConfidence,
+        NULLIF(LTRIM(RTRIM(EyeColourEvidence)), N''),
+        EyeColourSceneID
+    FROM OPENJSON(@ResultsJson)
+    WITH
+    (
+        CharacterID int '$.characterID',
+        Summary nvarchar(max) '$.summary',
+        Confidence decimal(5,4) '$.confidence',
+        DateOfBirthValue nvarchar(120) '$.dateOfBirth.value',
+        DateOfBirthConfidence decimal(5,4) '$.dateOfBirth.confidence',
+        DateOfBirthEvidence nvarchar(700) '$.dateOfBirth.evidence',
+        DateOfBirthSceneID int '$.dateOfBirth.sceneID',
+        AgeValue nvarchar(120) '$.ageAtStartOfSeries.value',
+        AgeConfidence decimal(5,4) '$.ageAtStartOfSeries.confidence',
+        AgeEvidence nvarchar(700) '$.ageAtStartOfSeries.evidence',
+        AgeSceneID int '$.ageAtStartOfSeries.sceneID',
+        SexValue nvarchar(120) '$.sex.value',
+        SexConfidence decimal(5,4) '$.sex.confidence',
+        SexEvidence nvarchar(700) '$.sex.evidence',
+        SexSceneID int '$.sex.sceneID',
+        HeightValue nvarchar(120) '$.height.value',
+        HeightConfidence decimal(5,4) '$.height.confidence',
+        HeightEvidence nvarchar(700) '$.height.evidence',
+        HeightSceneID int '$.height.sceneID',
+        EyeColourValue nvarchar(120) '$.eyeColour.value',
+        EyeColourConfidence decimal(5,4) '$.eyeColour.confidence',
+        EyeColourEvidence nvarchar(700) '$.eyeColour.evidence',
+        EyeColourSceneID int '$.eyeColour.sceneID'
+    )
+    WHERE CharacterID IS NOT NULL;
+
+    DECLARE @SexValues TABLE (SexName nvarchar(120) NOT NULL PRIMARY KEY, CharacterSexValueID int NOT NULL);
+    INSERT @SexValues (SexName, CharacterSexValueID)
+    SELECT SexName, CharacterSexValueID
+    FROM dbo.CharacterSexValues
+    WHERE IsSystem = 1
+        AND IsActive = 1
+        AND SexName IN (N'Male', N'Female', N'Other');
+
+    BEGIN TRANSACTION;
+
+    SELECT @ProjectID = ProjectID, @BookID = BookID, @PipelineID = StoryIntelligenceBookPipelineID
+    FROM dbo.CharacterEnrichmentRuns WITH (UPDLOCK, HOLDLOCK)
+    WHERE CharacterEnrichmentRunID = @CharacterEnrichmentRunID;
+
+    UPDATE c
+    SET DefaultDescription = CASE WHEN NULLIF(LTRIM(RTRIM(c.DefaultDescription)), N'') IS NULL THEN r.Summary ELSE c.DefaultDescription END,
+        BirthDate = CASE WHEN c.BirthDate IS NULL THEN TRY_CONVERT(date, r.DateOfBirthValue) ELSE c.BirthDate END,
+        AgeAtSeriesStart = CASE WHEN c.AgeAtSeriesStart IS NULL THEN TRY_CONVERT(int, r.AgeValue) ELSE c.AgeAtSeriesStart END,
+        SexValueID = CASE
+            WHEN c.SexValueID IS NULL
+                AND (NULLIF(LTRIM(RTRIM(c.Sex)), N'') IS NULL OR c.Sex = N'Unknown')
+                AND sexValue.CharacterSexValueID IS NOT NULL
+            THEN sexValue.CharacterSexValueID
+            ELSE c.SexValueID
+        END,
+        Sex = CASE
+            WHEN c.SexValueID IS NULL
+                AND (NULLIF(LTRIM(RTRIM(c.Sex)), N'') IS NULL OR c.Sex = N'Unknown')
+                AND sexValue.CharacterSexValueID IS NOT NULL
+            THEN r.SexValue
+            ELSE c.Sex
+        END,
+        Height = CASE WHEN NULLIF(LTRIM(RTRIM(c.Height)), N'') IS NULL THEN r.HeightValue ELSE c.Height END,
+        EyeColour = CASE WHEN NULLIF(LTRIM(RTRIM(c.EyeColour)), N'') IS NULL THEN r.EyeColourValue ELSE c.EyeColour END,
+        UpdatedDate = SYSUTCDATETIME()
+    FROM dbo.Characters c
+    INNER JOIN @Results r ON r.CharacterID = c.CharacterID
+    LEFT JOIN @SexValues sexValue ON sexValue.SexName = r.SexValue
+    WHERE c.ProjectID = @ProjectID;
+
+    MERGE dbo.CharacterIntelligenceProfiles WITH (HOLDLOCK) AS target
+    USING
+    (
+        SELECT
+            ImportSessionID = @PipelineID,
+            ProjectID = @ProjectID,
+            BookID = @BookID,
+            CanonicalIdentityKey = CONCAT(N'character-enrichment-', r.CharacterID),
+            CanonicalName = c.CharacterName,
+            ProfileJson = (SELECT r.* FOR JSON PATH, WITHOUT_ARRAY_WRAPPER),
+            Confidence = r.Confidence,
+            NarrativeImportance = CAST(NULL AS nvarchar(60))
+        FROM @Results r
+        INNER JOIN dbo.Characters c ON c.CharacterID = r.CharacterID
+        WHERE @PipelineID IS NOT NULL
+    ) AS source
+    ON target.ImportSessionID = source.ImportSessionID
+        AND target.CanonicalIdentityKey = source.CanonicalIdentityKey
+    WHEN MATCHED THEN
+        UPDATE SET CanonicalName = source.CanonicalName,
+            ProfileJson = source.ProfileJson,
+            Confidence = source.Confidence,
+            EvidenceVersion = N'CharacterEnrichmentV1',
+            UpdatedUtc = SYSUTCDATETIME()
+    WHEN NOT MATCHED THEN
+        INSERT (ImportSessionID, ProjectID, BookID, CanonicalIdentityKey, CanonicalName, ProfileJson, Confidence, NarrativeImportance, IsRecurring, EvidenceVersion, LastUpdatedSceneResultID)
+        VALUES (source.ImportSessionID, source.ProjectID, source.BookID, source.CanonicalIdentityKey, source.CanonicalName, source.ProfileJson, source.Confidence, source.NarrativeImportance, 1, N'CharacterEnrichmentV1', NULL);
+
+    DELETE evidence
+    FROM dbo.CharacterEnrichmentEvidence evidence
+    INNER JOIN @Results r ON r.CharacterID = evidence.CharacterID
+    WHERE evidence.CharacterEnrichmentRunID = @CharacterEnrichmentRunID;
+
+    INSERT dbo.CharacterEnrichmentEvidence (CharacterEnrichmentRunID, CharacterID, SceneID, FieldName, Value, Evidence, Confidence)
+    SELECT @CharacterEnrichmentRunID, CharacterID, NULL, N'Summary', Summary, NULL, Confidence
+    FROM @Results
+    WHERE Summary IS NOT NULL
+    UNION ALL
+    SELECT @CharacterEnrichmentRunID, CharacterID, DateOfBirthSceneID, N'DateOfBirth', DateOfBirthValue, DateOfBirthEvidence, COALESCE(DateOfBirthConfidence, 0)
+    FROM @Results WHERE DateOfBirthValue IS NOT NULL AND DateOfBirthSceneID IS NOT NULL
+    UNION ALL
+    SELECT @CharacterEnrichmentRunID, CharacterID, AgeSceneID, N'AgeAtStartOfSeries', AgeValue, AgeEvidence, COALESCE(AgeConfidence, 0)
+    FROM @Results WHERE AgeValue IS NOT NULL AND AgeSceneID IS NOT NULL
+    UNION ALL
+    SELECT @CharacterEnrichmentRunID, CharacterID, SexSceneID, N'Sex', SexValue, SexEvidence, COALESCE(SexConfidence, 0)
+    FROM @Results WHERE SexValue IS NOT NULL AND SexSceneID IS NOT NULL
+    UNION ALL
+    SELECT @CharacterEnrichmentRunID, CharacterID, HeightSceneID, N'Height', HeightValue, HeightEvidence, COALESCE(HeightConfidence, 0)
+    FROM @Results WHERE HeightValue IS NOT NULL AND HeightSceneID IS NOT NULL
+    UNION ALL
+    SELECT @CharacterEnrichmentRunID, CharacterID, EyeColourSceneID, N'EyeColour', EyeColourValue, EyeColourEvidence, COALESCE(EyeColourConfidence, 0)
+    FROM @Results WHERE EyeColourValue IS NOT NULL AND EyeColourSceneID IS NOT NULL;
+
+    UPDATE dbo.CharacterEnrichmentRuns
+    SET Status = N'Completed',
+        ProcessedCharacters = TotalCharacters,
+        CurrentStage = N'Complete',
+        CurrentMessage = N'Character summaries are ready.',
+        CompletedUtc = SYSUTCDATETIME(),
+        Model = @Model,
+        InputTokens = @InputTokens,
+        OutputTokens = @OutputTokens,
+        TotalTokens = @TotalTokens,
+        DurationMs = @DurationMs,
+        ErrorMessage = NULL,
+        ErrorDetail = NULL,
+        UpdatedUtc = SYSUTCDATETIME()
+    WHERE CharacterEnrichmentRunID = @CharacterEnrichmentRunID;
+
+    COMMIT TRANSACTION;
+END;
+GO
+
+CREATE OR ALTER PROCEDURE dbo.CharacterEnrichmentCharacterStatus_Get
+    @CharacterID int,
+    @UserID int
+AS
+BEGIN
+    SET NOCOUNT ON;
+
+    DECLARE @ProjectID int;
+    SELECT @ProjectID = c.ProjectID
+    FROM dbo.Characters c
+    INNER JOIN dbo.ProjectUserAccess pua ON pua.ProjectID = c.ProjectID
+    WHERE c.CharacterID = @CharacterID
+        AND c.IsArchived = 0
+        AND pua.UserID = @UserID
+        AND pua.AccessRole = N'Owner'
+        AND pua.IsActive = 1;
+
+    IF @ProjectID IS NULL
+    BEGIN
+        SELECT TOP (0)
+            CharacterID = CAST(NULL AS int),
+            BookID = CAST(NULL AS int),
+            CharacterEnrichmentRunID = CAST(NULL AS int),
+            Status = CAST(NULL AS nvarchar(30)),
+            CurrentStage = CAST(NULL AS nvarchar(120)),
+            CurrentMessage = CAST(NULL AS nvarchar(300)),
+            LastAnalysedUtc = CAST(NULL AS datetime2),
+            ErrorMessage = CAST(NULL AS nvarchar(max)),
+            Summary = CAST(NULL AS nvarchar(max)),
+            DateOfBirth = CAST(NULL AS nvarchar(500)),
+            AgeAtStartOfSeries = CAST(NULL AS nvarchar(500)),
+            Sex = CAST(NULL AS nvarchar(500)),
+            Height = CAST(NULL AS nvarchar(500)),
+            EyeColour = CAST(NULL AS nvarchar(500)),
+            UpdatedUtc = CAST(NULL AS datetime2);
+        RETURN;
+    END;
+
+    DECLARE @Attempt TABLE
+    (
+        CharacterEnrichmentRunID int NULL,
+        BookID int NULL,
+        Status nvarchar(30) NULL,
+        CurrentStage nvarchar(120) NULL,
+        CurrentMessage nvarchar(300) NULL,
+        ErrorMessage nvarchar(max) NULL,
+        UpdatedUtc datetime2 NULL
+    );
+
+    INSERT @Attempt
+    SELECT TOP (1)
+        runRows.CharacterEnrichmentRunID,
+        runRows.BookID,
+        runRows.Status,
+        runRows.CurrentStage,
+        runRows.CurrentMessage,
+        runRows.ErrorMessage,
+        runRows.UpdatedUtc
+    FROM dbo.CharacterEnrichmentRuns runRows
+    WHERE runRows.UserID = @UserID
+        AND runRows.ProjectID = @ProjectID
+        AND runRows.CharacterID = @CharacterID
+        AND runRows.IsCurrent = 1
+    ORDER BY CASE runRows.Status WHEN N'Queued' THEN 0 WHEN N'Running' THEN 0 WHEN N'Failed' THEN 1 WHEN N'Completed' THEN 2 ELSE 9 END,
+        runRows.UpdatedUtc DESC,
+        runRows.CharacterEnrichmentRunID DESC;
+
+    DECLARE @SuccessRunID int, @SuccessBookID int, @LastAnalysedUtc datetime2;
+    SELECT TOP (1)
+        @SuccessRunID = runRows.CharacterEnrichmentRunID,
+        @SuccessBookID = runRows.BookID,
+        @LastAnalysedUtc = runRows.CompletedUtc
+    FROM dbo.CharacterEnrichmentRuns runRows
+    WHERE runRows.UserID = @UserID
+        AND runRows.ProjectID = @ProjectID
+        AND runRows.Status = N'Completed'
+        AND runRows.CompletedUtc IS NOT NULL
+        AND (
+            runRows.CharacterID = @CharacterID
+            OR EXISTS
+            (
+                SELECT 1
+                FROM dbo.CharacterEnrichmentEvidence evidence
+                WHERE evidence.CharacterEnrichmentRunID = runRows.CharacterEnrichmentRunID
+                    AND evidence.CharacterID = @CharacterID
+            )
+        )
+    ORDER BY runRows.CompletedUtc DESC, runRows.CharacterEnrichmentRunID DESC;
+
+    SELECT
+        CharacterID = @CharacterID,
+        BookID = COALESCE((SELECT TOP (1) BookID FROM @Attempt), @SuccessBookID),
+        CharacterEnrichmentRunID = COALESCE((SELECT TOP (1) CharacterEnrichmentRunID FROM @Attempt), @SuccessRunID),
+        Status = COALESCE((SELECT TOP (1) Status FROM @Attempt), CASE WHEN @SuccessRunID IS NULL THEN N'Not analysed' ELSE N'Completed' END),
+        CurrentStage = (SELECT TOP (1) CurrentStage FROM @Attempt),
+        CurrentMessage = (SELECT TOP (1) CurrentMessage FROM @Attempt),
+        LastAnalysedUtc = @LastAnalysedUtc,
+        ErrorMessage = (SELECT TOP (1) ErrorMessage FROM @Attempt),
+        Summary = MAX(CASE WHEN evidence.FieldName = N'Summary' THEN evidence.Value END),
+        DateOfBirth = MAX(CASE WHEN evidence.FieldName = N'DateOfBirth' THEN evidence.Value END),
+        AgeAtStartOfSeries = MAX(CASE WHEN evidence.FieldName = N'AgeAtStartOfSeries' THEN evidence.Value END),
+        Sex = MAX(CASE WHEN evidence.FieldName = N'Sex' THEN evidence.Value END),
+        Height = MAX(CASE WHEN evidence.FieldName = N'Height' THEN evidence.Value END),
+        EyeColour = MAX(CASE WHEN evidence.FieldName = N'EyeColour' THEN evidence.Value END),
+        UpdatedUtc = COALESCE((SELECT TOP (1) UpdatedUtc FROM @Attempt), @LastAnalysedUtc, SYSUTCDATETIME())
+    FROM (SELECT 1 AS Anchor) anchor
+    LEFT JOIN dbo.CharacterEnrichmentEvidence evidence
+        ON evidence.CharacterEnrichmentRunID = @SuccessRunID
+        AND evidence.CharacterID = @CharacterID;
+END;
+GO
+
+;WITH LatestEvidence AS
+(
+    SELECT
+        evidence.CharacterID,
+        evidence.FieldName,
+        evidence.Value,
+        runRows.ProjectID,
+        rn = ROW_NUMBER() OVER
+        (
+            PARTITION BY evidence.CharacterID, evidence.FieldName
+            ORDER BY runRows.CompletedUtc DESC, evidence.CharacterEnrichmentEvidenceID DESC
+        )
+    FROM dbo.CharacterEnrichmentEvidence evidence
+    INNER JOIN dbo.CharacterEnrichmentRuns runRows ON runRows.CharacterEnrichmentRunID = evidence.CharacterEnrichmentRunID
+    WHERE runRows.Status = N'Completed'
+        AND evidence.FieldName IN (N'DateOfBirth', N'AgeAtStartOfSeries', N'Height', N'EyeColour', N'Sex')
+),
+Pivoted AS
+(
+    SELECT
+        CharacterID,
+        ProjectID = MAX(ProjectID),
+        DateOfBirthValue = MAX(CASE WHEN FieldName = N'DateOfBirth' THEN Value END),
+        AgeValue = MAX(CASE WHEN FieldName = N'AgeAtStartOfSeries' THEN Value END),
+        HeightValue = MAX(CASE WHEN FieldName = N'Height' THEN Value END),
+        EyeColourValue = MAX(CASE WHEN FieldName = N'EyeColour' THEN Value END),
+        SexValue = MAX(CASE WHEN FieldName = N'Sex' THEN Value END)
+    FROM LatestEvidence
+    WHERE rn = 1
+    GROUP BY CharacterID
+),
+Normalised AS
+(
+    SELECT
+        p.CharacterID,
+        p.ProjectID,
+        DateOfBirthValue = TRY_CONVERT(date, p.DateOfBirthValue),
+        AgeValue = TRY_CONVERT(int, CASE
+            WHEN p.AgeValue IS NULL THEN NULL
+            WHEN PATINDEX(N'%[^0-9]%', p.AgeValue + N'x') = 1 THEN NULL
+            ELSE LEFT(p.AgeValue, PATINDEX(N'%[^0-9]%', p.AgeValue + N'x') - 1)
+        END),
+        p.HeightValue,
+        p.EyeColourValue,
+        SexValue = CASE
+            WHEN LOWER(NULLIF(LTRIM(RTRIM(p.SexValue)), N'')) = N'female' THEN N'Female'
+            WHEN LOWER(NULLIF(LTRIM(RTRIM(p.SexValue)), N'')) = N'male' THEN N'Male'
+            WHEN LOWER(NULLIF(LTRIM(RTRIM(p.SexValue)), N'')) = N'other' THEN N'Other'
+            ELSE NULL
+        END
+    FROM Pivoted p
+),
+SexValues AS
+(
+    SELECT SexName, CharacterSexValueID
+    FROM dbo.CharacterSexValues
+    WHERE IsSystem = 1
+        AND IsActive = 1
+        AND SexName IN (N'Male', N'Female', N'Other')
+)
+UPDATE c
+SET BirthDate = CASE WHEN c.BirthDate IS NULL THEN n.DateOfBirthValue ELSE c.BirthDate END,
+    AgeAtSeriesStart = CASE WHEN c.AgeAtSeriesStart IS NULL THEN n.AgeValue ELSE c.AgeAtSeriesStart END,
+    Height = CASE WHEN NULLIF(LTRIM(RTRIM(c.Height)), N'') IS NULL THEN n.HeightValue ELSE c.Height END,
+    EyeColour = CASE WHEN NULLIF(LTRIM(RTRIM(c.EyeColour)), N'') IS NULL THEN n.EyeColourValue ELSE c.EyeColour END,
+    SexValueID = CASE
+        WHEN c.SexValueID IS NULL
+            AND (NULLIF(LTRIM(RTRIM(c.Sex)), N'') IS NULL OR c.Sex = N'Unknown')
+            AND sexValue.CharacterSexValueID IS NOT NULL
+        THEN sexValue.CharacterSexValueID
+        ELSE c.SexValueID
+    END,
+    Sex = CASE
+        WHEN c.SexValueID IS NULL
+            AND (NULLIF(LTRIM(RTRIM(c.Sex)), N'') IS NULL OR c.Sex = N'Unknown')
+            AND sexValue.CharacterSexValueID IS NOT NULL
+        THEN n.SexValue
+        ELSE c.Sex
+    END,
+    UpdatedDate = SYSUTCDATETIME()
+FROM dbo.Characters c
+INNER JOIN Normalised n ON n.CharacterID = c.CharacterID
+LEFT JOIN SexValues sexValue ON sexValue.SexName = n.SexValue
+WHERE c.ProjectID = n.ProjectID
+    AND (
+        (c.BirthDate IS NULL AND n.DateOfBirthValue IS NOT NULL)
+        OR (c.AgeAtSeriesStart IS NULL AND n.AgeValue IS NOT NULL)
+        OR (NULLIF(LTRIM(RTRIM(c.Height)), N'') IS NULL AND n.HeightValue IS NOT NULL)
+        OR (NULLIF(LTRIM(RTRIM(c.EyeColour)), N'') IS NULL AND n.EyeColourValue IS NOT NULL)
+        OR (c.SexValueID IS NULL AND (NULLIF(LTRIM(RTRIM(c.Sex)), N'') IS NULL OR c.Sex = N'Unknown') AND sexValue.CharacterSexValueID IS NOT NULL)
+    );
+GO

PlotLine/Sql/189_Phase27P_BackfillDateOfBirthFromAgeEvidence.sql

@@ -0,0 +1,95 @@
+;WITH MonthNames AS
+(
+    SELECT MonthNumber = 1, MonthName = N'January' UNION ALL
+    SELECT 2, N'February' UNION ALL
+    SELECT 3, N'March' UNION ALL
+    SELECT 4, N'April' UNION ALL
+    SELECT 5, N'May' UNION ALL
+    SELECT 6, N'June' UNION ALL
+    SELECT 7, N'July' UNION ALL
+    SELECT 8, N'August' UNION ALL
+    SELECT 9, N'September' UNION ALL
+    SELECT 10, N'October' UNION ALL
+    SELECT 11, N'November' UNION ALL
+    SELECT 12, N'December'
+),
+LatestAgeEvidence AS
+(
+    SELECT
+        evidence.CharacterID,
+        runRows.ProjectID,
+        EvidenceText = CONCAT(evidence.Value, N' ', evidence.Evidence),
+        AgeValue = TRY_CONVERT(int, CASE
+            WHEN evidence.Value IS NULL THEN NULL
+            WHEN PATINDEX(N'%[^0-9]%', evidence.Value + N'x') = 1 THEN NULL
+            ELSE LEFT(evidence.Value, PATINDEX(N'%[^0-9]%', evidence.Value + N'x') - 1)
+        END),
+        rn = ROW_NUMBER() OVER
+        (
+            PARTITION BY evidence.CharacterID
+            ORDER BY runRows.CompletedUtc DESC, evidence.CharacterEnrichmentEvidenceID DESC
+        )
+    FROM dbo.CharacterEnrichmentEvidence evidence
+    INNER JOIN dbo.CharacterEnrichmentRuns runRows ON runRows.CharacterEnrichmentRunID = evidence.CharacterEnrichmentRunID
+    WHERE runRows.Status = N'Completed'
+        AND evidence.FieldName = N'AgeAtStartOfSeries'
+),
+MatchedMonth AS
+(
+    SELECT
+        ageEvidence.CharacterID,
+        ageEvidence.ProjectID,
+        ageEvidence.AgeValue,
+        monthNames.MonthNumber,
+        MonthPosition = CHARINDEX(monthNames.MonthName, ageEvidence.EvidenceText),
+        ageEvidence.EvidenceText
+    FROM LatestAgeEvidence ageEvidence
+    INNER JOIN MonthNames monthNames ON CHARINDEX(monthNames.MonthName, ageEvidence.EvidenceText) > 0
+    WHERE ageEvidence.rn = 1
+        AND ageEvidence.AgeValue BETWEEN 1 AND 120
+),
+Parts AS
+(
+    SELECT
+        matched.CharacterID,
+        matched.ProjectID,
+        matched.AgeValue,
+        matched.MonthNumber,
+        DayValue = TRY_CONVERT(int, REPLACE(REPLACE(REPLACE(REPLACE(dayParts.DayToken, N'st', N''), N'nd', N''), N'rd', N''), N'th', N'')),
+        YearValue = TRY_CONVERT(int, LEFT(LTRIM(SUBSTRING(matched.EvidenceText, matched.MonthPosition + LEN(monthNames.MonthName), 20)), 4))
+    FROM MatchedMonth matched
+    INNER JOIN MonthNames monthNames ON monthNames.MonthNumber = matched.MonthNumber
+    CROSS APPLY
+    (
+        SELECT Prefix = RTRIM(CASE WHEN matched.MonthPosition > 1 THEN LEFT(matched.EvidenceText, matched.MonthPosition - 1) ELSE N'' END)
+    ) prefixParts
+    CROSS APPLY
+    (
+        SELECT LastSpaceFromEnd = CHARINDEX(N' ', REVERSE(prefixParts.Prefix))
+    ) spaceParts
+    CROSS APPLY
+    (
+        SELECT DayToken = CASE
+            WHEN LEN(prefixParts.Prefix) = 0 THEN N''
+            WHEN spaceParts.LastSpaceFromEnd = 0 THEN prefixParts.Prefix
+            ELSE RIGHT(prefixParts.Prefix, spaceParts.LastSpaceFromEnd - 1)
+        END
+    ) dayParts
+),
+Derived AS
+(
+    SELECT
+        CharacterID,
+        ProjectID,
+        DateOfBirth = DATEADD(year, -AgeValue, DATEFROMPARTS(YearValue, MonthNumber, DayValue))
+    FROM Parts
+    WHERE DayValue BETWEEN 1 AND 31
+        AND YearValue BETWEEN 1000 AND 9999
+)
+UPDATE c
+SET BirthDate = derived.DateOfBirth,
+    UpdatedDate = SYSUTCDATETIME()
+FROM dbo.Characters c
+INNER JOIN Derived derived ON derived.CharacterID = c.CharacterID AND derived.ProjectID = c.ProjectID
+WHERE c.BirthDate IS NULL;
+GO