From b78a9bea9bed7829d98f38f6c20828ff53499536 Mon Sep 17 00:00:00 2001 From: Nick Beckley Date: Sun, 12 Jul 2026 20:03:01 +0000 Subject: [PATCH] Phase 21I - Intelligent Illustration Matching Engine --- ...elligenceIllustrationMatchingRepository.cs | 93 ++++ ...yIntelligenceIllustrationMatchingModels.cs | 60 +++ PlotLine/Program.cs | 2 + .../Services/IllustrationLibraryServices.cs | 24 +- ...IntelligenceIllustrationMatchingService.cs | 508 ++++++++++++++++++ ...ntelligenceVisualisationSnapshotService.cs | 6 + ...ase21I_IntelligentIllustrationMatching.sql | 79 +++ ...telligenceExperiencePrototypeViewModels.cs | 6 + ...story-intelligence-experience-prototype.js | 8 +- 9 files changed, 784 insertions(+), 2 deletions(-) create mode 100644 PlotLine/Data/StoryIntelligenceIllustrationMatchingRepository.cs create mode 100644 PlotLine/Models/StoryIntelligenceIllustrationMatchingModels.cs create mode 100644 PlotLine/Services/StoryIntelligenceIllustrationMatchingService.cs create mode 100644 PlotLine/Sql/138_Phase21I_IntelligentIllustrationMatching.sql diff --git a/PlotLine/Data/StoryIntelligenceIllustrationMatchingRepository.cs b/PlotLine/Data/StoryIntelligenceIllustrationMatchingRepository.cs new file mode 100644 index 0000000..5368bd2 --- /dev/null +++ b/PlotLine/Data/StoryIntelligenceIllustrationMatchingRepository.cs @@ -0,0 +1,93 @@ +using Dapper; +using PlotLine.Models; + +namespace PlotLine.Data; + +public interface IStoryIntelligenceIllustrationMatchingRepository +{ + Task> ListAssignmentsAsync(int projectId); + Task UpsertAssignmentAsync(StoryIntelligenceCharacterIllustrationAssignmentSave request); + Task UpsertDemandAsync(StoryIntelligenceIllustrationDemandRequest request); +} + +public sealed class StoryIntelligenceIllustrationMatchingRepository(ISqlConnectionFactory connectionFactory) : IStoryIntelligenceIllustrationMatchingRepository +{ + public async Task> ListAssignmentsAsync(int projectId) + { + using var connection = connectionFactory.CreateConnection(); + var rows = await connection.QueryAsync( + """ + SELECT * + FROM dbo.StoryIntelligenceCharacterIllustrationAssignments + WHERE ProjectID = @ProjectID + AND IsActive = 1; + """, + new { ProjectID = projectId }); + return rows.ToList(); + } + + public async Task UpsertAssignmentAsync(StoryIntelligenceCharacterIllustrationAssignmentSave request) + { + using var connection = connectionFactory.CreateConnection(); + return await connection.QuerySingleAsync( + """ + SET XACT_ABORT ON; + + MERGE dbo.StoryIntelligenceCharacterIllustrationAssignments WITH (HOLDLOCK) AS target + USING (SELECT @ProjectID AS ProjectID, @CharacterKey AS CharacterKey) AS source + ON target.ProjectID = source.ProjectID + AND target.CharacterKey = source.CharacterKey + AND target.IsActive = 1 + WHEN MATCHED THEN + UPDATE SET + CharacterName = @CharacterName, + IllustrationLibraryItemID = @IllustrationLibraryItemID, + StableCode = @StableCode, + MatchConfidence = @MatchConfidence, + MatchingScore = @MatchingScore, + EvidenceJson = @EvidenceJson, + CandidateDiagnosticsJson = @CandidateDiagnosticsJson, + ReasonChosen = @ReasonChosen, + ReasonReassigned = @ReasonReassigned, + UpdatedUtc = SYSUTCDATETIME() + WHEN NOT MATCHED THEN + INSERT (ProjectID, CharacterKey, CharacterName, IllustrationLibraryItemID, StableCode, MatchConfidence, MatchingScore, EvidenceJson, CandidateDiagnosticsJson, ReasonChosen, ReasonReassigned) + VALUES (@ProjectID, @CharacterKey, @CharacterName, @IllustrationLibraryItemID, @StableCode, @MatchConfidence, @MatchingScore, @EvidenceJson, @CandidateDiagnosticsJson, @ReasonChosen, @ReasonReassigned); + + SELECT * + FROM dbo.StoryIntelligenceCharacterIllustrationAssignments + WHERE ProjectID = @ProjectID + AND CharacterKey = @CharacterKey + AND IsActive = 1; + """, + request); + } + + public async Task UpsertDemandAsync(StoryIntelligenceIllustrationDemandRequest request) + { + using var connection = connectionFactory.CreateConnection(); + return await connection.QuerySingleAsync( + """ + SET XACT_ABORT ON; + + MERGE dbo.StoryIntelligenceIllustrationDemands WITH (HOLDLOCK) AS target + USING (SELECT @ProjectID AS ProjectID, @DemandKey AS DemandKey) AS source + ON ISNULL(target.ProjectID, -1) = ISNULL(source.ProjectID, -1) + AND target.DemandKey = source.DemandKey + WHEN MATCHED THEN + UPDATE SET + RequestCount = RequestCount + 1, + LastRequestedUtc = SYSUTCDATETIME(), + UpdatedUtc = SYSUTCDATETIME() + WHEN NOT MATCHED THEN + INSERT (ProjectID, DemandKey, AgeBand, Presentation, HairColour, SkinTone, HairLength, Glasses, FacialHair) + VALUES (@ProjectID, @DemandKey, @AgeBand, @Presentation, @HairColour, @SkinTone, @HairLength, @Glasses, @FacialHair); + + SELECT * + FROM dbo.StoryIntelligenceIllustrationDemands + WHERE ISNULL(ProjectID, -1) = ISNULL(@ProjectID, -1) + AND DemandKey = @DemandKey; + """, + request); + } +} diff --git a/PlotLine/Models/StoryIntelligenceIllustrationMatchingModels.cs b/PlotLine/Models/StoryIntelligenceIllustrationMatchingModels.cs new file mode 100644 index 0000000..6b8c16c --- /dev/null +++ b/PlotLine/Models/StoryIntelligenceIllustrationMatchingModels.cs @@ -0,0 +1,60 @@ +namespace PlotLine.Models; + +public sealed class StoryIntelligenceCharacterIllustrationAssignment +{ + public int StoryIntelligenceCharacterIllustrationAssignmentID { get; init; } + public int ProjectID { get; init; } + public string CharacterKey { get; init; } = string.Empty; + public string CharacterName { get; init; } = string.Empty; + public int? IllustrationLibraryItemID { get; init; } + public string? StableCode { get; init; } + public decimal MatchConfidence { get; init; } + public int MatchingScore { get; init; } + public string? EvidenceJson { get; init; } + public string? CandidateDiagnosticsJson { get; init; } + public string? ReasonChosen { get; init; } + public string? ReasonReassigned { get; init; } + public bool IsActive { get; init; } + public DateTime CreatedUtc { get; init; } + public DateTime UpdatedUtc { get; init; } +} + +public sealed record StoryIntelligenceCharacterIllustrationAssignmentSave( + int ProjectID, + string CharacterKey, + string CharacterName, + int? IllustrationLibraryItemID, + string? StableCode, + decimal MatchConfidence, + int MatchingScore, + string? EvidenceJson, + string? CandidateDiagnosticsJson, + string? ReasonChosen, + string? ReasonReassigned); + +public sealed class StoryIntelligenceIllustrationDemand +{ + public int StoryIntelligenceIllustrationDemandID { get; init; } + public int? ProjectID { get; init; } + public string DemandKey { get; init; } = string.Empty; + public string AgeBand { get; init; } = string.Empty; + public string Presentation { get; init; } = string.Empty; + public string HairColour { get; init; } = string.Empty; + public string SkinTone { get; init; } = string.Empty; + public string? HairLength { get; init; } + public string? Glasses { get; init; } + public string? FacialHair { get; init; } + public int RequestCount { get; init; } + public DateTime LastRequestedUtc { get; init; } +} + +public sealed record StoryIntelligenceIllustrationDemandRequest( + int? ProjectID, + string DemandKey, + string AgeBand, + string Presentation, + string HairColour, + string SkinTone, + string? HairLength, + string? Glasses, + string? FacialHair); diff --git a/PlotLine/Program.cs b/PlotLine/Program.cs index 837b29f..6f6cebb 100644 --- a/PlotLine/Program.cs +++ b/PlotLine/Program.cs @@ -144,6 +144,7 @@ public class Program builder.Services.AddScoped(); builder.Services.AddScoped(); builder.Services.AddScoped(); + builder.Services.AddScoped(); builder.Services.AddScoped(); builder.Services.AddScoped(); builder.Services.AddScoped(); @@ -223,6 +224,7 @@ public class Program builder.Services.AddHttpClient(); builder.Services.AddScoped(); builder.Services.AddScoped(); + builder.Services.AddScoped(); builder.Services.AddScoped(); builder.Services.AddScoped(); builder.Services.AddScoped(); diff --git a/PlotLine/Services/IllustrationLibraryServices.cs b/PlotLine/Services/IllustrationLibraryServices.cs index 765892d..3f78798 100644 --- a/PlotLine/Services/IllustrationLibraryServices.cs +++ b/PlotLine/Services/IllustrationLibraryServices.cs @@ -32,7 +32,21 @@ public sealed class IllustrationPromptBuilder : IIllustrationPromptBuilder throw new ArgumentException(string.Join(" ", errors), nameof(specification)); } - var metadataJson = JsonSerializer.Serialize(specification.Metadata, JsonOptions); + var metadata = new Dictionary(specification.Metadata, StringComparer.OrdinalIgnoreCase); + if (string.Equals(specification.Category, IllustrationLibraryCategories.Character, StringComparison.OrdinalIgnoreCase)) + { + AddMetadata("ageBand", specification.ApparentAgeBand); + AddMetadata("presentation", specification.Presentation); + AddMetadata("hairColour", specification.HairColour); + AddMetadata("skinTone", specification.SkinTone); + AddMetadata("hairLength", specification.HairLength); + AddMetadata("glasses", specification.Features.Any(feature => feature.Contains("glasses", StringComparison.OrdinalIgnoreCase)) ? "Yes" : "Unknown"); + AddMetadata("facialHair", specification.Features.FirstOrDefault(feature => feature.Contains("beard", StringComparison.OrdinalIgnoreCase) || feature.Contains("moustache", StringComparison.OrdinalIgnoreCase))); + AddMetadata("clothingEra", specification.ClothingEra); + AddMetadata("clothingStyle", specification.ClothingStyle); + } + + var metadataJson = JsonSerializer.Serialize(metadata, JsonOptions); var specificationJson = JsonSerializer.Serialize(specification, JsonOptions); var palette = specification.Palette.Count == 0 ? "balanced editorial colour, not monochrome" : string.Join(", ", specification.Palette); var tags = specification.MetadataTags.Count == 0 ? "generic reusable story identifier" : string.Join(", ", specification.MetadataTags); @@ -71,6 +85,14 @@ public sealed class IllustrationPromptBuilder : IIllustrationPromptBuilder """; return new(prompt, CurrentTemplateVersion, specificationJson, metadataJson); + + void AddMetadata(string key, string? value) + { + if (!string.IsNullOrWhiteSpace(value)) + { + metadata[key] = value; + } + } } private static string CategoryRules(string category) diff --git a/PlotLine/Services/StoryIntelligenceIllustrationMatchingService.cs b/PlotLine/Services/StoryIntelligenceIllustrationMatchingService.cs new file mode 100644 index 0000000..6a7a037 --- /dev/null +++ b/PlotLine/Services/StoryIntelligenceIllustrationMatchingService.cs @@ -0,0 +1,508 @@ +using System.Text.Json; +using System.Text.RegularExpressions; +using PlotLine.Data; +using PlotLine.Models; +using PlotLine.ViewModels; + +namespace PlotLine.Services; + +public interface IStoryIntelligenceIllustrationMatchingService +{ + Task ApplyCharacterIllustrationsAsync(int projectId, StoryIntelligenceExperiencePrototypeViewModel prototype); +} + +public sealed class StoryIntelligenceIllustrationMatchingService( + IIllustrationLibraryRepository library, + IStoryIntelligenceIllustrationMatchingRepository assignments, + IUploadStorageService uploadStorage, + ILogger logger) : IStoryIntelligenceIllustrationMatchingService +{ + private const int SuitableScore = 48; + private const int ReassignmentMargin = 24; + private static readonly JsonSerializerOptions JsonOptions = new() + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + WriteIndented = false + }; + + public async Task ApplyCharacterIllustrationsAsync(int projectId, StoryIntelligenceExperiencePrototypeViewModel prototype) + { + if (projectId <= 0 || prototype.Scenes.Count == 0) + { + return; + } + + var catalogue = (await library.ListAsync(new IllustrationLibraryFilter { Category = IllustrationLibraryCategories.Character })) + .Where(item => item.Status is IllustrationLibraryStatuses.Approved or IllustrationLibraryStatuses.Generated) + .Select(CharacterCandidate.From) + .Where(candidate => candidate is not null) + .Cast() + .Select(candidate => candidate with { PublicUrl = ResolveExistingPublicUploadUrl(candidate.Item.ThumbnailPath) ?? ResolveExistingPublicUploadUrl(candidate.Item.FilePath) }) + .Where(candidate => !string.IsNullOrWhiteSpace(candidate.PublicUrl)) + .ToList(); + + if (catalogue.Count == 0) + { + logger.LogWarning("No generated character illustrations are available for Story Intelligence matching."); + return; + } + + var existingAssignments = (await assignments.ListAssignmentsAsync(projectId)) + .ToDictionary(item => item.CharacterKey, StringComparer.OrdinalIgnoreCase); + var assignedItemIds = existingAssignments.Values + .Where(item => item.IllustrationLibraryItemID.HasValue) + .Select(item => item.IllustrationLibraryItemID!.Value) + .ToHashSet(); + var observations = BuildCharacterObservations(prototype); + + foreach (var observation in observations.Values.OrderByDescending(item => item.Significance).ThenBy(item => item.Name)) + { + var existing = existingAssignments.GetValueOrDefault(observation.CharacterKey); + var ownExistingItemId = existing?.IllustrationLibraryItemID; + var allScores = catalogue + .Select(candidate => Score( + candidate, + observation, + assignedItemIds.Contains(candidate.Item.IllustrationLibraryItemID) + && candidate.Item.IllustrationLibraryItemID != ownExistingItemId)) + .OrderByDescending(score => score.Score) + .ThenBy(score => score.IsAlreadyAssignedToAnotherSignificantCharacter) + .ThenByDescending(score => score.Candidate.Item.Status == IllustrationLibraryStatuses.Approved) + .ThenByDescending(score => score.Candidate.Item.UpdatedUtc) + .ToList(); + + var existingScore = existing?.IllustrationLibraryItemID is int existingItemId + ? allScores.FirstOrDefault(score => score.Candidate.Item.IllustrationLibraryItemID == existingItemId) + : null; + var best = allScores + .Where(score => observation.IsSignificant ? !score.IsAlreadyAssignedToAnotherSignificantCharacter : true) + .FirstOrDefault(score => score.Score >= SuitableScore); + + var chosen = ChooseAssignment(observation, existing, existingScore, best); + if (chosen is null) + { + await RecordDemandAsync(projectId, observation, allScores); + ApplyDemandDiagnostics(prototype, observation, allScores); + continue; + } + + var reasonReassigned = existingScore is not null && chosen.Candidate.Item.IllustrationLibraryItemID != existing?.IllustrationLibraryItemID + ? ReassignmentReason(observation, existingScore, chosen) + : null; + var reason = reasonReassigned ?? (existing is null + ? $"Assigned {chosen.Candidate.Item.StableCode} from structured character evidence." + : "Preserved existing Story Intelligence character illustration assignment."); + var candidateDiagnostics = JsonSerializer.Serialize(allScores.Take(6).Select(MatchDiagnostics.From), JsonOptions); + var evidenceJson = JsonSerializer.Serialize(observation.Evidence, JsonOptions); + + var saved = await assignments.UpsertAssignmentAsync(new StoryIntelligenceCharacterIllustrationAssignmentSave( + projectId, + observation.CharacterKey, + observation.Name, + chosen.Candidate.Item.IllustrationLibraryItemID, + chosen.Candidate.Item.StableCode, + Confidence(chosen.Score), + chosen.Score, + evidenceJson, + candidateDiagnostics, + reason, + reasonReassigned)); + + assignedItemIds.Add(chosen.Candidate.Item.IllustrationLibraryItemID); + ApplyAssignment(prototype, observation, chosen, saved, reason, reasonReassigned, candidateDiagnostics); + } + } + + private MatchScore? ChooseAssignment( + CharacterObservation observation, + StoryIntelligenceCharacterIllustrationAssignment? existing, + MatchScore? existingScore, + MatchScore? best) + { + if (existingScore is not null && existingScore.Candidate.PublicUrl is not null) + { + if (best is null) + { + return existingScore; + } + + var materiallyWrong = MateriallyWrong(existingScore.Candidate.Metadata, observation.Evidence); + if (!materiallyWrong || best.Score < existingScore.Score + ReassignmentMargin) + { + return existingScore; + } + } + + if (best is not null) + { + return best; + } + + return existing is null ? null : existingScore; + } + + private static bool MateriallyWrong(CharacterIllustrationMetadata candidate, CharacterEvidence evidence) + => KnownMismatch(evidence.AgeBand, candidate.AgeBand) + || KnownMismatch(evidence.Presentation, candidate.Presentation) + || KnownMismatch(evidence.HairColour, candidate.HairColour) + || KnownMismatch(evidence.SkinTone, candidate.SkinTone); + + private static bool KnownMismatch(string value, string candidate) + => !IsUnknown(value) && !IsUnknown(candidate) && !string.Equals(value, candidate, StringComparison.OrdinalIgnoreCase); + + private async Task RecordDemandAsync(int projectId, CharacterObservation observation, IReadOnlyList allScores) + { + var evidence = observation.Evidence; + var request = new StoryIntelligenceIllustrationDemandRequest( + projectId, + DemandKey(evidence), + evidence.AgeBand, + evidence.Presentation, + evidence.HairColour, + evidence.SkinTone, + evidence.HairLength, + evidence.Glasses, + evidence.FacialHair); + var demand = await assignments.UpsertDemandAsync(request); + logger.LogInformation( + "Recorded Story Intelligence illustration demand {DemandKey} for project {ProjectId}. Request count {RequestCount}. Best score {BestScore}.", + demand.DemandKey, + projectId, + demand.RequestCount, + allScores.FirstOrDefault()?.Score ?? 0); + } + + private static string DemandKey(CharacterEvidence evidence) + => string.Join('|', evidence.AgeBand, evidence.Presentation, evidence.HairColour, evidence.SkinTone, evidence.HairLength, evidence.Glasses, evidence.FacialHair).ToLowerInvariant(); + + private static Dictionary BuildCharacterObservations(StoryIntelligenceExperiencePrototypeViewModel prototype) + { + var observations = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var scene in prototype.Scenes) + { + foreach (var character in scene.Characters) + { + if (string.IsNullOrWhiteSpace(character.Id) || string.IsNullOrWhiteSpace(character.Name)) + { + continue; + } + + if (!observations.TryGetValue(character.Id, out var observation)) + { + observation = new CharacterObservation(character.Id, character.Name); + observations[character.Id] = observation; + } + + observation.Significance = Math.Max(observation.Significance, character.Weight); + observation.TextEvidence.Add(character.Name); + observation.TextEvidence.Add(character.Role); + observation.TextEvidence.Add(character.Relevance); + foreach (var relationship in scene.Relationships.Where(item => string.Equals(item.SourceId, character.Id, StringComparison.OrdinalIgnoreCase) + || string.Equals(item.TargetId, character.Id, StringComparison.OrdinalIgnoreCase))) + { + observation.TextEvidence.Add(relationship.Label); + observation.TextEvidence.Add(relationship.State); + } + } + } + + foreach (var observation in observations.Values) + { + observation.Evidence = InferEvidence(observation.TextEvidence); + } + + return observations; + } + + private static CharacterEvidence InferEvidence(IEnumerable values) + { + var text = string.Join(' ', values.Where(value => !string.IsNullOrWhiteSpace(value))).ToLowerInvariant(); + var age = ContainsAny(text, "grandfather", "grandmother", "elderly", "old man", "old woman", "older woman", "older man") ? "Elderly" + : ContainsAny(text, "middle aged", "middle-aged", "mother", "father", "parent", "aunt", "uncle") ? "Middle Aged" + : ContainsAny(text, "young teen", "child", "boy", "girl") ? "Child" + : ContainsAny(text, "older teen", "teenage", "teenager", "fifteen", "sixteen", "seventeen") ? "Older Teen" + : ContainsAny(text, "young adult", "student", "young woman", "young man") ? "Young Adult" + : "Unknown"; + var presentation = ContainsAny(text, "she ", " her ", "herself", "woman", "girl", "mother", "daughter", "sister", "feminine") ? "Feminine" + : ContainsAny(text, " he ", " him ", "himself", "man", "boy", "father", "son", "brother", "masculine") ? "Masculine" + : "Androgynous"; + var hairColour = ContainsAny(text, "black hair", "dark hair") ? "Black" + : ContainsAny(text, "brown hair", "brunette") ? "Brown" + : ContainsAny(text, "blonde", "fair hair") ? "Blonde" + : ContainsAny(text, "red hair", "red-haired", "auburn", "ginger") ? "Red" + : ContainsAny(text, "grey hair", "gray hair") ? "Grey" + : ContainsAny(text, "white hair") ? "White" + : "Unknown"; + var skinTone = ContainsAny(text, "light skin", "pale skin", "fair skin") ? "Light" + : ContainsAny(text, "medium skin", "olive skin", "brown skin") ? "Medium" + : ContainsAny(text, "dark skin", "deep skin", "black skin") ? "Dark" + : "Unknown"; + var hairLength = ContainsAny(text, "bald", "shaved head") ? "Bald" + : ContainsAny(text, "short hair") ? "Short" + : ContainsAny(text, "long hair") ? "Long" + : ContainsAny(text, "medium hair", "shoulder length") ? "Medium" + : "Unknown"; + var glasses = ContainsAny(text, "glasses", "spectacles") ? "Yes" : "Unknown"; + var facialHair = ContainsAny(text, "moustache", "mustache") ? "Moustache" + : ContainsAny(text, "beard", "bearded") ? "Beard" + : "Unknown"; + + return new(age, presentation, hairColour, skinTone, hairLength, glasses, facialHair); + } + + private static bool ContainsAny(string text, params string[] needles) + => needles.Any(needle => text.Contains(needle, StringComparison.OrdinalIgnoreCase)); + + private static MatchScore Score(CharacterCandidate candidate, CharacterObservation observation, bool alreadyAssigned) + { + var score = 0; + var rejected = new List(); + AddScore(observation.Evidence.AgeBand, candidate.Metadata.AgeBand, 25, "age band"); + AddScore(observation.Evidence.Presentation, candidate.Metadata.Presentation, 25, "presentation"); + AddScore(observation.Evidence.HairColour, candidate.Metadata.HairColour, 20, "hair colour"); + AddScore(observation.Evidence.SkinTone, candidate.Metadata.SkinTone, 20, "skin tone"); + AddScore(observation.Evidence.HairLength, candidate.Metadata.HairLength, 8, "hair length"); + AddScore(observation.Evidence.Glasses, candidate.Metadata.Glasses, 7, "glasses"); + AddScore(observation.Evidence.FacialHair, candidate.Metadata.FacialHair, 7, "facial hair"); + + if (candidate.Item.Status == IllustrationLibraryStatuses.Approved) + { + score += 3; + } + + if (alreadyAssigned && observation.IsSignificant) + { + score -= 12; + rejected.Add("already assigned to another significant character"); + } + + return new(candidate, Math.Clamp(score, 0, 100), alreadyAssigned && observation.IsSignificant, rejected); + + void AddScore(string evidence, string value, int weight, string label) + { + if (IsUnknown(evidence)) + { + score += weight / 2; + return; + } + + if (string.Equals(evidence, value, StringComparison.OrdinalIgnoreCase)) + { + score += weight; + return; + } + + if (IsUnknown(value)) + { + score += weight / 3; + rejected.Add($"{label} unknown in illustration metadata"); + return; + } + + rejected.Add($"{label} mismatch: wanted {evidence}, candidate {value}"); + } + } + + private static decimal Confidence(int score) + => Math.Round(Math.Clamp(score, 0, 100) / 100m, 2); + + private static string ReassignmentReason(CharacterObservation observation, MatchScore existing, MatchScore chosen) + => $"Reassigned {observation.Name} because new structured evidence made {existing.Candidate.Item.StableCode} materially weaker than {chosen.Candidate.Item.StableCode}."; + + private void ApplyAssignment( + StoryIntelligenceExperiencePrototypeViewModel prototype, + CharacterObservation observation, + MatchScore chosen, + StoryIntelligenceCharacterIllustrationAssignment saved, + string reason, + string? reasonReassigned, + string candidateDiagnostics) + { + foreach (var character in prototype.Scenes.SelectMany(scene => scene.Characters).Where(character => string.Equals(character.Id, observation.CharacterKey, StringComparison.OrdinalIgnoreCase))) + { + var fallbackImageUrl = string.IsNullOrWhiteSpace(character.ImageResolution.FallbackImageUrl) + ? character.ImagePath + : character.ImageResolution.FallbackImageUrl; + character.LibraryCode = chosen.Candidate.Item.StableCode; + character.ImagePath = chosen.Candidate.PublicUrl!; + character.ImageResolution = new StoryIntelligenceExperienceImageResolution + { + EntityId = character.Id, + StableCode = chosen.Candidate.Item.StableCode, + IllustrationLibraryItemId = chosen.Candidate.Item.IllustrationLibraryItemID, + Status = chosen.Candidate.Item.Status, + FinalImageUrl = chosen.Candidate.PublicUrl, + FallbackImageUrl = fallbackImageUrl, + FallbackUsed = false, + AssignmentConfidence = saved.MatchConfidence, + MatchingScore = saved.MatchingScore, + RejectedCandidates = candidateDiagnostics, + ReasonChosen = reason, + ReasonReassigned = reasonReassigned, + DemandStatus = "Satisfied" + }; + } + } + + private static void ApplyDemandDiagnostics(StoryIntelligenceExperiencePrototypeViewModel prototype, CharacterObservation observation, IReadOnlyList allScores) + { + var diagnostics = JsonSerializer.Serialize(allScores.Take(6).Select(MatchDiagnostics.From), JsonOptions); + foreach (var character in prototype.Scenes.SelectMany(scene => scene.Characters).Where(character => string.Equals(character.Id, observation.CharacterKey, StringComparison.OrdinalIgnoreCase))) + { + character.ImageResolution = new StoryIntelligenceExperienceImageResolution + { + EntityId = character.Id, + StableCode = character.LibraryCode, + IllustrationLibraryItemId = character.ImageResolution.IllustrationLibraryItemId, + Status = character.ImageResolution.Status, + FinalImageUrl = character.ImageResolution.FinalImageUrl, + FallbackImageUrl = character.ImageResolution.FallbackImageUrl, + FallbackUsed = character.ImageResolution.FallbackUsed, + MatchingScore = allScores.FirstOrDefault()?.Score, + RejectedCandidates = diagnostics, + ReasonChosen = "No suitable unused illustration matched structured character evidence.", + DemandStatus = "Demand recorded" + }; + } + } + + private string? ResolveExistingPublicUploadUrl(string? storedPath) + { + var physicalPath = uploadStorage.TryResolvePublicPath(storedPath, "uploads/story-intelligence-library"); + if (physicalPath is null || !File.Exists(physicalPath)) + { + return null; + } + + var normalised = storedPath!.Replace('\\', '/').Trim(); + if (!normalised.StartsWith('/')) + { + normalised = "/" + normalised.TrimStart('/'); + } + + return EncodeUrlPath(normalised); + } + + private static string EncodeUrlPath(string path) + => string.Join( + '/', + path.Split('/', StringSplitOptions.None) + .Select((segment, index) => index == 0 && segment.Length == 0 ? string.Empty : Uri.EscapeDataString(segment))); + + private static bool IsUnknown(string? value) + => string.IsNullOrWhiteSpace(value) || string.Equals(value, "Unknown", StringComparison.OrdinalIgnoreCase); + + private sealed class CharacterObservation(string characterKey, string name) + { + public string CharacterKey { get; } = characterKey; + public string Name { get; } = name; + public int Significance { get; set; } + public bool IsSignificant => Significance >= 50; + public List TextEvidence { get; } = []; + public CharacterEvidence Evidence { get; set; } = CharacterEvidence.Unknown; + } + + private sealed record CharacterEvidence(string AgeBand, string Presentation, string HairColour, string SkinTone, string HairLength, string Glasses, string FacialHair) + { + public static CharacterEvidence Unknown { get; } = new("Unknown", "Androgynous", "Unknown", "Unknown", "Unknown", "Unknown", "Unknown"); + } + + private sealed record CharacterCandidate(IllustrationLibraryItem Item, CharacterIllustrationMetadata Metadata, string? PublicUrl) + { + public static CharacterCandidate? From(IllustrationLibraryItem item) + { + var metadata = CharacterIllustrationMetadata.From(item); + return metadata is null ? null : new(item, metadata, null); + } + } + + private sealed record CharacterIllustrationMetadata(string AgeBand, string Presentation, string HairColour, string SkinTone, string HairLength, string Glasses, string FacialHair) + { + public static CharacterIllustrationMetadata? From(IllustrationLibraryItem item) + { + IllustrationGenerationSpecification? specification = null; + if (!string.IsNullOrWhiteSpace(item.SpecificationJson)) + { + try + { + specification = JsonSerializer.Deserialize(item.SpecificationJson, new JsonSerializerOptions { PropertyNameCaseInsensitive = true }); + } + catch (JsonException) + { + } + } + + return new( + NormaliseAge(specification?.ApparentAgeBand, item.StableCode), + NormalisePresentation(specification?.Presentation, item.StableCode), + NormaliseHairColour(specification?.HairColour, item.StableCode), + NormaliseSkinTone(specification?.SkinTone, item.StableCode), + NormaliseHairLength(specification?.HairLength, item.StableCode), + FeatureValue(specification?.Features, "Glasses") ? "Yes" : "Unknown", + FeatureValue(specification?.Features, "Beard") ? "Beard" : FeatureValue(specification?.Features, "Moustache") ? "Moustache" : "Unknown"); + } + + private static bool FeatureValue(IReadOnlyList? features, string value) + => features?.Any(feature => feature.Contains(value, StringComparison.OrdinalIgnoreCase)) == true; + + private static string NormaliseAge(string? value, string code) + { + var text = $"{value} {code}".ToLowerInvariant(); + if (text.Contains("child")) return "Child"; + if (text.Contains("youngteen")) return "Young Teen"; + if (text.Contains("teen")) return "Older Teen"; + if (text.Contains("youngadult") || text.Contains("young-adult")) return "Young Adult"; + if (text.Contains("middleaged") || text.Contains("middle-aged")) return "Middle Aged"; + if (text.Contains("elderly") || text.Contains("older")) return "Elderly"; + return "Unknown"; + } + + private static string NormalisePresentation(string? value, string code) + { + var text = $"{value} {code}".ToLowerInvariant(); + if (text.Contains("feminine") || text.Contains("woman") || text.Contains("female")) return "Feminine"; + if (text.Contains("masculine") || text.Contains("man") || text.Contains("male")) return "Masculine"; + if (text.Contains("androgynous")) return "Androgynous"; + return "Androgynous"; + } + + private static string NormaliseHairColour(string? value, string code) + { + var text = $"{value} {code}".ToLowerInvariant(); + if (text.Contains("black")) return "Black"; + if (text.Contains("brown") || text.Contains("brunette") || text.Contains("darkbrown")) return "Brown"; + if (text.Contains("blonde")) return "Blonde"; + if (text.Contains("red") || text.Contains("auburn")) return "Red"; + if (text.Contains("grey") || text.Contains("gray")) return "Grey"; + if (text.Contains("white")) return "White"; + return "Unknown"; + } + + private static string NormaliseSkinTone(string? value, string code) + { + var text = $"{value} {code}".ToLowerInvariant(); + if (text.Contains("deep") || text.Contains("dark")) return "Dark"; + if (text.Contains("medium") || text.Contains("lightmedium")) return "Medium"; + if (text.Contains("light")) return "Light"; + return "Unknown"; + } + + private static string NormaliseHairLength(string? value, string code) + { + var text = $"{value} {code}".ToLowerInvariant(); + if (text.Contains("bald")) return "Bald"; + if (text.Contains("short")) return "Short"; + if (text.Contains("long")) return "Long"; + if (text.Contains("medium")) return "Medium"; + return "Unknown"; + } + } + + private sealed record MatchScore(CharacterCandidate Candidate, int Score, bool IsAlreadyAssignedToAnotherSignificantCharacter, IReadOnlyList Rejections); + + private sealed record MatchDiagnostics(int IllustrationLibraryItemID, string StableCode, int Score, IReadOnlyList Rejections) + { + public static MatchDiagnostics From(MatchScore score) + => new(score.Candidate.Item.IllustrationLibraryItemID, score.Candidate.Item.StableCode, score.Score, score.Rejections); + } +} diff --git a/PlotLine/Services/StoryIntelligenceVisualisationSnapshotService.cs b/PlotLine/Services/StoryIntelligenceVisualisationSnapshotService.cs index c3d3580..4f05533 100644 --- a/PlotLine/Services/StoryIntelligenceVisualisationSnapshotService.cs +++ b/PlotLine/Services/StoryIntelligenceVisualisationSnapshotService.cs @@ -21,6 +21,7 @@ public sealed class StoryIntelligenceVisualisationSnapshotService( IStoryIntelligencePipelineRepository pipelines, IStoryIntelligenceSourceRepository sources, IIllustrationLibraryService illustrationLibrary, + IStoryIntelligenceIllustrationMatchingService illustrationMatcher, ILogger logger) : IStoryIntelligenceVisualisationSnapshotService { private const string FallbackRoot = "/images/story-intelligence/prototype"; @@ -137,6 +138,10 @@ public sealed class StoryIntelligenceVisualisationSnapshotService( StabiliseCharacterIllustrations(model); await illustrationLibrary.ApplyApprovedIllustrationsAsync(model); + if (run.ProjectID.HasValue) + { + await illustrationMatcher.ApplyCharacterIllustrationsAsync(run.ProjectID.Value, model); + } logger.LogInformation("Built Story Intelligence visualisation snapshot for run {RunId} in {ElapsedMs}ms.", runId, stopwatch.ElapsedMilliseconds); return model; } @@ -239,6 +244,7 @@ public sealed class StoryIntelligenceVisualisationSnapshotService( StabiliseCharacterIllustrations(model); await illustrationLibrary.ApplyApprovedIllustrationsAsync(model); + await illustrationMatcher.ApplyCharacterIllustrationsAsync(session.ProjectID, model); logger.LogInformation("Built Story Intelligence import session visualisation snapshot for session {ImportSessionId} in {ElapsedMs}ms.", importSessionId, stopwatch.ElapsedMilliseconds); return model; } diff --git a/PlotLine/Sql/138_Phase21I_IntelligentIllustrationMatching.sql b/PlotLine/Sql/138_Phase21I_IntelligentIllustrationMatching.sql new file mode 100644 index 0000000..5a68260 --- /dev/null +++ b/PlotLine/Sql/138_Phase21I_IntelligentIllustrationMatching.sql @@ -0,0 +1,79 @@ +SET ANSI_NULLS ON; +SET QUOTED_IDENTIFIER ON; +GO + +IF OBJECT_ID(N'dbo.StoryIntelligenceCharacterIllustrationAssignments', N'U') IS NULL +BEGIN + CREATE TABLE dbo.StoryIntelligenceCharacterIllustrationAssignments + ( + StoryIntelligenceCharacterIllustrationAssignmentID int IDENTITY(1,1) NOT NULL CONSTRAINT PK_StoryIntelligenceCharacterIllustrationAssignments PRIMARY KEY, + ProjectID int NOT NULL, + CharacterKey nvarchar(160) NOT NULL, + CharacterName nvarchar(200) NOT NULL, + IllustrationLibraryItemID int NULL, + StableCode nvarchar(120) NULL, + MatchConfidence decimal(5,2) NOT NULL CONSTRAINT DF_SICharacterIllustrationAssignments_MatchConfidence DEFAULT 0, + MatchingScore int NOT NULL CONSTRAINT DF_SICharacterIllustrationAssignments_MatchingScore DEFAULT 0, + EvidenceJson nvarchar(max) NULL, + CandidateDiagnosticsJson nvarchar(max) NULL, + ReasonChosen nvarchar(500) NULL, + ReasonReassigned nvarchar(500) NULL, + IsActive bit NOT NULL CONSTRAINT DF_SICharacterIllustrationAssignments_IsActive DEFAULT 1, + CreatedUtc datetime2(0) NOT NULL CONSTRAINT DF_SICharacterIllustrationAssignments_CreatedUtc DEFAULT SYSUTCDATETIME(), + UpdatedUtc datetime2(0) NOT NULL CONSTRAINT DF_SICharacterIllustrationAssignments_UpdatedUtc DEFAULT SYSUTCDATETIME() + ); +END; +GO + +IF NOT EXISTS (SELECT 1 FROM sys.indexes WHERE name = N'UX_SICharacterIllustrationAssignments_ProjectCharacter' AND object_id = OBJECT_ID(N'dbo.StoryIntelligenceCharacterIllustrationAssignments')) +BEGIN + CREATE UNIQUE INDEX UX_SICharacterIllustrationAssignments_ProjectCharacter + ON dbo.StoryIntelligenceCharacterIllustrationAssignments(ProjectID, CharacterKey) + WHERE IsActive = 1; +END; +GO + +IF NOT EXISTS (SELECT 1 FROM sys.foreign_keys WHERE name = N'FK_SICharacterIllustrationAssignments_Project') +BEGIN + ALTER TABLE dbo.StoryIntelligenceCharacterIllustrationAssignments + ADD CONSTRAINT FK_SICharacterIllustrationAssignments_Project + FOREIGN KEY (ProjectID) REFERENCES dbo.Projects(ProjectID); +END; +GO + +IF NOT EXISTS (SELECT 1 FROM sys.foreign_keys WHERE name = N'FK_SICharacterIllustrationAssignments_Illustration') +BEGIN + ALTER TABLE dbo.StoryIntelligenceCharacterIllustrationAssignments + ADD CONSTRAINT FK_SICharacterIllustrationAssignments_Illustration + FOREIGN KEY (IllustrationLibraryItemID) REFERENCES dbo.IllustrationLibraryItems(IllustrationLibraryItemID); +END; +GO + +IF OBJECT_ID(N'dbo.StoryIntelligenceIllustrationDemands', N'U') IS NULL +BEGIN + CREATE TABLE dbo.StoryIntelligenceIllustrationDemands + ( + StoryIntelligenceIllustrationDemandID int IDENTITY(1,1) NOT NULL CONSTRAINT PK_StoryIntelligenceIllustrationDemands PRIMARY KEY, + ProjectID int NULL, + DemandKey nvarchar(240) NOT NULL, + AgeBand nvarchar(40) NOT NULL, + Presentation nvarchar(40) NOT NULL, + HairColour nvarchar(40) NOT NULL, + SkinTone nvarchar(40) NOT NULL, + HairLength nvarchar(40) NULL, + Glasses nvarchar(20) NULL, + FacialHair nvarchar(40) NULL, + RequestCount int NOT NULL CONSTRAINT DF_StoryIntelligenceIllustrationDemands_RequestCount DEFAULT 1, + LastRequestedUtc datetime2(0) NOT NULL CONSTRAINT DF_StoryIntelligenceIllustrationDemands_LastRequestedUtc DEFAULT SYSUTCDATETIME(), + CreatedUtc datetime2(0) NOT NULL CONSTRAINT DF_StoryIntelligenceIllustrationDemands_CreatedUtc DEFAULT SYSUTCDATETIME(), + UpdatedUtc datetime2(0) NOT NULL CONSTRAINT DF_StoryIntelligenceIllustrationDemands_UpdatedUtc DEFAULT SYSUTCDATETIME() + ); +END; +GO + +IF NOT EXISTS (SELECT 1 FROM sys.indexes WHERE name = N'UX_StoryIntelligenceIllustrationDemands_ProjectDemand' AND object_id = OBJECT_ID(N'dbo.StoryIntelligenceIllustrationDemands')) +BEGIN + CREATE UNIQUE INDEX UX_StoryIntelligenceIllustrationDemands_ProjectDemand + ON dbo.StoryIntelligenceIllustrationDemands(ProjectID, DemandKey); +END; +GO diff --git a/PlotLine/ViewModels/StoryIntelligenceExperiencePrototypeViewModels.cs b/PlotLine/ViewModels/StoryIntelligenceExperiencePrototypeViewModels.cs index 0f335d2..eac16c4 100644 --- a/PlotLine/ViewModels/StoryIntelligenceExperiencePrototypeViewModels.cs +++ b/PlotLine/ViewModels/StoryIntelligenceExperiencePrototypeViewModels.cs @@ -126,6 +126,12 @@ public sealed class StoryIntelligenceExperienceImageResolution public string? FinalImageUrl { get; init; } public string FallbackImageUrl { get; init; } = string.Empty; public bool FallbackUsed { get; init; } = true; + public decimal? AssignmentConfidence { get; init; } + public int? MatchingScore { get; init; } + public string? RejectedCandidates { get; init; } + public string? ReasonChosen { get; init; } + public string? ReasonReassigned { get; init; } + public string? DemandStatus { get; init; } } public sealed class StoryIntelligenceExperienceRelationshipState diff --git a/PlotLine/wwwroot/js/story-intelligence-experience-prototype.js b/PlotLine/wwwroot/js/story-intelligence-experience-prototype.js index c667c1d..feacf10 100644 --- a/PlotLine/wwwroot/js/story-intelligence-experience-prototype.js +++ b/PlotLine/wwwroot/js/story-intelligence-experience-prototype.js @@ -263,7 +263,13 @@ recordId: resolution.illustrationLibraryItemId || null, status: resolution.status || "fallback", finalImageUrl: resolution.finalImageUrl || item?.imagePath || "", - fallbackUsed: !!resolution.fallbackUsed + fallbackUsed: !!resolution.fallbackUsed, + assignmentConfidence: resolution.assignmentConfidence || null, + matchingScore: resolution.matchingScore || null, + reasonChosen: resolution.reasonChosen || "", + reasonReassigned: resolution.reasonReassigned || "", + rejectedCandidates: resolution.rejectedCandidates || "", + demandStatus: resolution.demandStatus || "" }); };