PlotDirector/PlotLine/Services/StoryIntelligenceIllustrationMatchingService.cs

509 lines
25 KiB
C#

using System.Text.Json;
using System.Text.RegularExpressions;
using PlotLine.Data;
using PlotLine.Models;
using PlotLine.ViewModels;
namespace PlotLine.Services;
public interface IStoryIntelligenceIllustrationMatchingService
{
Task ApplyCharacterIllustrationsAsync(int projectId, StoryIntelligenceExperiencePrototypeViewModel prototype);
}
public sealed class StoryIntelligenceIllustrationMatchingService(
IIllustrationLibraryRepository library,
IStoryIntelligenceIllustrationMatchingRepository assignments,
IUploadStorageService uploadStorage,
ILogger<StoryIntelligenceIllustrationMatchingService> logger) : IStoryIntelligenceIllustrationMatchingService
{
private const int SuitableScore = 48;
private const int ReassignmentMargin = 24;
private static readonly JsonSerializerOptions JsonOptions = new()
{
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
WriteIndented = false
};
public async Task ApplyCharacterIllustrationsAsync(int projectId, StoryIntelligenceExperiencePrototypeViewModel prototype)
{
if (projectId <= 0 || prototype.Scenes.Count == 0)
{
return;
}
var catalogue = (await library.ListAsync(new IllustrationLibraryFilter { Category = IllustrationLibraryCategories.Character }))
.Where(item => item.Status is IllustrationLibraryStatuses.Approved or IllustrationLibraryStatuses.Generated)
.Select(CharacterCandidate.From)
.Where(candidate => candidate is not null)
.Cast<CharacterCandidate>()
.Select(candidate => candidate with { PublicUrl = ResolveExistingPublicUploadUrl(candidate.Item.ThumbnailPath) ?? ResolveExistingPublicUploadUrl(candidate.Item.FilePath) })
.Where(candidate => !string.IsNullOrWhiteSpace(candidate.PublicUrl))
.ToList();
if (catalogue.Count == 0)
{
logger.LogWarning("No generated character illustrations are available for Story Intelligence matching.");
return;
}
var existingAssignments = (await assignments.ListAssignmentsAsync(projectId))
.ToDictionary(item => item.CharacterKey, StringComparer.OrdinalIgnoreCase);
var assignedItemIds = existingAssignments.Values
.Where(item => item.IllustrationLibraryItemID.HasValue)
.Select(item => item.IllustrationLibraryItemID!.Value)
.ToHashSet();
var observations = BuildCharacterObservations(prototype);
foreach (var observation in observations.Values.OrderByDescending(item => item.Significance).ThenBy(item => item.Name))
{
var existing = existingAssignments.GetValueOrDefault(observation.CharacterKey);
var ownExistingItemId = existing?.IllustrationLibraryItemID;
var allScores = catalogue
.Select(candidate => Score(
candidate,
observation,
assignedItemIds.Contains(candidate.Item.IllustrationLibraryItemID)
&& candidate.Item.IllustrationLibraryItemID != ownExistingItemId))
.OrderByDescending(score => score.Score)
.ThenBy(score => score.IsAlreadyAssignedToAnotherSignificantCharacter)
.ThenByDescending(score => score.Candidate.Item.Status == IllustrationLibraryStatuses.Approved)
.ThenByDescending(score => score.Candidate.Item.UpdatedUtc)
.ToList();
var existingScore = existing?.IllustrationLibraryItemID is int existingItemId
? allScores.FirstOrDefault(score => score.Candidate.Item.IllustrationLibraryItemID == existingItemId)
: null;
var best = allScores
.Where(score => observation.IsSignificant ? !score.IsAlreadyAssignedToAnotherSignificantCharacter : true)
.FirstOrDefault(score => score.Score >= SuitableScore);
var chosen = ChooseAssignment(observation, existing, existingScore, best);
if (chosen is null)
{
await RecordDemandAsync(projectId, observation, allScores);
ApplyDemandDiagnostics(prototype, observation, allScores);
continue;
}
var reasonReassigned = existingScore is not null && chosen.Candidate.Item.IllustrationLibraryItemID != existing?.IllustrationLibraryItemID
? ReassignmentReason(observation, existingScore, chosen)
: null;
var reason = reasonReassigned ?? (existing is null
? $"Assigned {chosen.Candidate.Item.StableCode} from structured character evidence."
: "Preserved existing Story Intelligence character illustration assignment.");
var candidateDiagnostics = JsonSerializer.Serialize(allScores.Take(6).Select(MatchDiagnostics.From), JsonOptions);
var evidenceJson = JsonSerializer.Serialize(observation.Evidence, JsonOptions);
var saved = await assignments.UpsertAssignmentAsync(new StoryIntelligenceCharacterIllustrationAssignmentSave(
projectId,
observation.CharacterKey,
observation.Name,
chosen.Candidate.Item.IllustrationLibraryItemID,
chosen.Candidate.Item.StableCode,
Confidence(chosen.Score),
chosen.Score,
evidenceJson,
candidateDiagnostics,
reason,
reasonReassigned));
assignedItemIds.Add(chosen.Candidate.Item.IllustrationLibraryItemID);
ApplyAssignment(prototype, observation, chosen, saved, reason, reasonReassigned, candidateDiagnostics);
}
}
private MatchScore? ChooseAssignment(
CharacterObservation observation,
StoryIntelligenceCharacterIllustrationAssignment? existing,
MatchScore? existingScore,
MatchScore? best)
{
if (existingScore is not null && existingScore.Candidate.PublicUrl is not null)
{
if (best is null)
{
return existingScore;
}
var materiallyWrong = MateriallyWrong(existingScore.Candidate.Metadata, observation.Evidence);
if (!materiallyWrong || best.Score < existingScore.Score + ReassignmentMargin)
{
return existingScore;
}
}
if (best is not null)
{
return best;
}
return existing is null ? null : existingScore;
}
private static bool MateriallyWrong(CharacterIllustrationMetadata candidate, CharacterEvidence evidence)
=> KnownMismatch(evidence.AgeBand, candidate.AgeBand)
|| KnownMismatch(evidence.Presentation, candidate.Presentation)
|| KnownMismatch(evidence.HairColour, candidate.HairColour)
|| KnownMismatch(evidence.SkinTone, candidate.SkinTone);
private static bool KnownMismatch(string value, string candidate)
=> !IsUnknown(value) && !IsUnknown(candidate) && !string.Equals(value, candidate, StringComparison.OrdinalIgnoreCase);
private async Task RecordDemandAsync(int projectId, CharacterObservation observation, IReadOnlyList<MatchScore> allScores)
{
var evidence = observation.Evidence;
var request = new StoryIntelligenceIllustrationDemandRequest(
projectId,
DemandKey(evidence),
evidence.AgeBand,
evidence.Presentation,
evidence.HairColour,
evidence.SkinTone,
evidence.HairLength,
evidence.Glasses,
evidence.FacialHair);
var demand = await assignments.UpsertDemandAsync(request);
logger.LogInformation(
"Recorded Story Intelligence illustration demand {DemandKey} for project {ProjectId}. Request count {RequestCount}. Best score {BestScore}.",
demand.DemandKey,
projectId,
demand.RequestCount,
allScores.FirstOrDefault()?.Score ?? 0);
}
private static string DemandKey(CharacterEvidence evidence)
=> string.Join('|', evidence.AgeBand, evidence.Presentation, evidence.HairColour, evidence.SkinTone, evidence.HairLength, evidence.Glasses, evidence.FacialHair).ToLowerInvariant();
private static Dictionary<string, CharacterObservation> BuildCharacterObservations(StoryIntelligenceExperiencePrototypeViewModel prototype)
{
var observations = new Dictionary<string, CharacterObservation>(StringComparer.OrdinalIgnoreCase);
foreach (var scene in prototype.Scenes)
{
foreach (var character in scene.Characters)
{
if (string.IsNullOrWhiteSpace(character.Id) || string.IsNullOrWhiteSpace(character.Name))
{
continue;
}
if (!observations.TryGetValue(character.Id, out var observation))
{
observation = new CharacterObservation(character.Id, character.Name);
observations[character.Id] = observation;
}
observation.Significance = Math.Max(observation.Significance, character.Weight);
observation.TextEvidence.Add(character.Name);
observation.TextEvidence.Add(character.Role);
observation.TextEvidence.Add(character.Relevance);
foreach (var relationship in scene.Relationships.Where(item => string.Equals(item.SourceId, character.Id, StringComparison.OrdinalIgnoreCase)
|| string.Equals(item.TargetId, character.Id, StringComparison.OrdinalIgnoreCase)))
{
observation.TextEvidence.Add(relationship.Label);
observation.TextEvidence.Add(relationship.State);
}
}
}
foreach (var observation in observations.Values)
{
observation.Evidence = InferEvidence(observation.TextEvidence);
}
return observations;
}
private static CharacterEvidence InferEvidence(IEnumerable<string> values)
{
var text = string.Join(' ', values.Where(value => !string.IsNullOrWhiteSpace(value))).ToLowerInvariant();
var age = ContainsAny(text, "grandfather", "grandmother", "elderly", "old man", "old woman", "older woman", "older man") ? "Elderly"
: ContainsAny(text, "middle aged", "middle-aged", "mother", "father", "parent", "aunt", "uncle") ? "Middle Aged"
: ContainsAny(text, "young teen", "child", "boy", "girl") ? "Child"
: ContainsAny(text, "older teen", "teenage", "teenager", "fifteen", "sixteen", "seventeen") ? "Older Teen"
: ContainsAny(text, "young adult", "student", "young woman", "young man") ? "Young Adult"
: "Unknown";
var presentation = ContainsAny(text, "she ", " her ", "herself", "woman", "girl", "mother", "daughter", "sister", "feminine") ? "Feminine"
: ContainsAny(text, " he ", " him ", "himself", "man", "boy", "father", "son", "brother", "masculine") ? "Masculine"
: "Androgynous";
var hairColour = ContainsAny(text, "black hair", "dark hair") ? "Black"
: ContainsAny(text, "brown hair", "brunette") ? "Brown"
: ContainsAny(text, "blonde", "fair hair") ? "Blonde"
: ContainsAny(text, "red hair", "red-haired", "auburn", "ginger") ? "Red"
: ContainsAny(text, "grey hair", "gray hair") ? "Grey"
: ContainsAny(text, "white hair") ? "White"
: "Unknown";
var skinTone = ContainsAny(text, "light skin", "pale skin", "fair skin") ? "Light"
: ContainsAny(text, "medium skin", "olive skin", "brown skin") ? "Medium"
: ContainsAny(text, "dark skin", "deep skin", "black skin") ? "Dark"
: "Unknown";
var hairLength = ContainsAny(text, "bald", "shaved head") ? "Bald"
: ContainsAny(text, "short hair") ? "Short"
: ContainsAny(text, "long hair") ? "Long"
: ContainsAny(text, "medium hair", "shoulder length") ? "Medium"
: "Unknown";
var glasses = ContainsAny(text, "glasses", "spectacles") ? "Yes" : "Unknown";
var facialHair = ContainsAny(text, "moustache", "mustache") ? "Moustache"
: ContainsAny(text, "beard", "bearded") ? "Beard"
: "Unknown";
return new(age, presentation, hairColour, skinTone, hairLength, glasses, facialHair);
}
private static bool ContainsAny(string text, params string[] needles)
=> needles.Any(needle => text.Contains(needle, StringComparison.OrdinalIgnoreCase));
private static MatchScore Score(CharacterCandidate candidate, CharacterObservation observation, bool alreadyAssigned)
{
var score = 0;
var rejected = new List<string>();
AddScore(observation.Evidence.AgeBand, candidate.Metadata.AgeBand, 25, "age band");
AddScore(observation.Evidence.Presentation, candidate.Metadata.Presentation, 25, "presentation");
AddScore(observation.Evidence.HairColour, candidate.Metadata.HairColour, 20, "hair colour");
AddScore(observation.Evidence.SkinTone, candidate.Metadata.SkinTone, 20, "skin tone");
AddScore(observation.Evidence.HairLength, candidate.Metadata.HairLength, 8, "hair length");
AddScore(observation.Evidence.Glasses, candidate.Metadata.Glasses, 7, "glasses");
AddScore(observation.Evidence.FacialHair, candidate.Metadata.FacialHair, 7, "facial hair");
if (candidate.Item.Status == IllustrationLibraryStatuses.Approved)
{
score += 3;
}
if (alreadyAssigned && observation.IsSignificant)
{
score -= 12;
rejected.Add("already assigned to another significant character");
}
return new(candidate, Math.Clamp(score, 0, 100), alreadyAssigned && observation.IsSignificant, rejected);
void AddScore(string evidence, string value, int weight, string label)
{
if (IsUnknown(evidence))
{
score += weight / 2;
return;
}
if (string.Equals(evidence, value, StringComparison.OrdinalIgnoreCase))
{
score += weight;
return;
}
if (IsUnknown(value))
{
score += weight / 3;
rejected.Add($"{label} unknown in illustration metadata");
return;
}
rejected.Add($"{label} mismatch: wanted {evidence}, candidate {value}");
}
}
private static decimal Confidence(int score)
=> Math.Round(Math.Clamp(score, 0, 100) / 100m, 2);
private static string ReassignmentReason(CharacterObservation observation, MatchScore existing, MatchScore chosen)
=> $"Reassigned {observation.Name} because new structured evidence made {existing.Candidate.Item.StableCode} materially weaker than {chosen.Candidate.Item.StableCode}.";
private void ApplyAssignment(
StoryIntelligenceExperiencePrototypeViewModel prototype,
CharacterObservation observation,
MatchScore chosen,
StoryIntelligenceCharacterIllustrationAssignment saved,
string reason,
string? reasonReassigned,
string candidateDiagnostics)
{
foreach (var character in prototype.Scenes.SelectMany(scene => scene.Characters).Where(character => string.Equals(character.Id, observation.CharacterKey, StringComparison.OrdinalIgnoreCase)))
{
var fallbackImageUrl = string.IsNullOrWhiteSpace(character.ImageResolution.FallbackImageUrl)
? character.ImagePath
: character.ImageResolution.FallbackImageUrl;
character.LibraryCode = chosen.Candidate.Item.StableCode;
character.ImagePath = chosen.Candidate.PublicUrl!;
character.ImageResolution = new StoryIntelligenceExperienceImageResolution
{
EntityId = character.Id,
StableCode = chosen.Candidate.Item.StableCode,
IllustrationLibraryItemId = chosen.Candidate.Item.IllustrationLibraryItemID,
Status = chosen.Candidate.Item.Status,
FinalImageUrl = chosen.Candidate.PublicUrl,
FallbackImageUrl = fallbackImageUrl,
FallbackUsed = false,
AssignmentConfidence = saved.MatchConfidence,
MatchingScore = saved.MatchingScore,
RejectedCandidates = candidateDiagnostics,
ReasonChosen = reason,
ReasonReassigned = reasonReassigned,
DemandStatus = "Satisfied"
};
}
}
private static void ApplyDemandDiagnostics(StoryIntelligenceExperiencePrototypeViewModel prototype, CharacterObservation observation, IReadOnlyList<MatchScore> allScores)
{
var diagnostics = JsonSerializer.Serialize(allScores.Take(6).Select(MatchDiagnostics.From), JsonOptions);
foreach (var character in prototype.Scenes.SelectMany(scene => scene.Characters).Where(character => string.Equals(character.Id, observation.CharacterKey, StringComparison.OrdinalIgnoreCase)))
{
character.ImageResolution = new StoryIntelligenceExperienceImageResolution
{
EntityId = character.Id,
StableCode = character.LibraryCode,
IllustrationLibraryItemId = character.ImageResolution.IllustrationLibraryItemId,
Status = character.ImageResolution.Status,
FinalImageUrl = character.ImageResolution.FinalImageUrl,
FallbackImageUrl = character.ImageResolution.FallbackImageUrl,
FallbackUsed = character.ImageResolution.FallbackUsed,
MatchingScore = allScores.FirstOrDefault()?.Score,
RejectedCandidates = diagnostics,
ReasonChosen = "No suitable unused illustration matched structured character evidence.",
DemandStatus = "Demand recorded"
};
}
}
private string? ResolveExistingPublicUploadUrl(string? storedPath)
{
var physicalPath = uploadStorage.TryResolvePublicPath(storedPath, "uploads/story-intelligence-library");
if (physicalPath is null || !File.Exists(physicalPath))
{
return null;
}
var normalised = storedPath!.Replace('\\', '/').Trim();
if (!normalised.StartsWith('/'))
{
normalised = "/" + normalised.TrimStart('/');
}
return EncodeUrlPath(normalised);
}
private static string EncodeUrlPath(string path)
=> string.Join(
'/',
path.Split('/', StringSplitOptions.None)
.Select((segment, index) => index == 0 && segment.Length == 0 ? string.Empty : Uri.EscapeDataString(segment)));
private static bool IsUnknown(string? value)
=> string.IsNullOrWhiteSpace(value) || string.Equals(value, "Unknown", StringComparison.OrdinalIgnoreCase);
private sealed class CharacterObservation(string characterKey, string name)
{
public string CharacterKey { get; } = characterKey;
public string Name { get; } = name;
public int Significance { get; set; }
public bool IsSignificant => Significance >= 50;
public List<string> TextEvidence { get; } = [];
public CharacterEvidence Evidence { get; set; } = CharacterEvidence.Unknown;
}
private sealed record CharacterEvidence(string AgeBand, string Presentation, string HairColour, string SkinTone, string HairLength, string Glasses, string FacialHair)
{
public static CharacterEvidence Unknown { get; } = new("Unknown", "Androgynous", "Unknown", "Unknown", "Unknown", "Unknown", "Unknown");
}
private sealed record CharacterCandidate(IllustrationLibraryItem Item, CharacterIllustrationMetadata Metadata, string? PublicUrl)
{
public static CharacterCandidate? From(IllustrationLibraryItem item)
{
var metadata = CharacterIllustrationMetadata.From(item);
return metadata is null ? null : new(item, metadata, null);
}
}
private sealed record CharacterIllustrationMetadata(string AgeBand, string Presentation, string HairColour, string SkinTone, string HairLength, string Glasses, string FacialHair)
{
public static CharacterIllustrationMetadata? From(IllustrationLibraryItem item)
{
IllustrationGenerationSpecification? specification = null;
if (!string.IsNullOrWhiteSpace(item.SpecificationJson))
{
try
{
specification = JsonSerializer.Deserialize<IllustrationGenerationSpecification>(item.SpecificationJson, new JsonSerializerOptions { PropertyNameCaseInsensitive = true });
}
catch (JsonException)
{
}
}
return new(
NormaliseAge(specification?.ApparentAgeBand, item.StableCode),
NormalisePresentation(specification?.Presentation, item.StableCode),
NormaliseHairColour(specification?.HairColour, item.StableCode),
NormaliseSkinTone(specification?.SkinTone, item.StableCode),
NormaliseHairLength(specification?.HairLength, item.StableCode),
FeatureValue(specification?.Features, "Glasses") ? "Yes" : "Unknown",
FeatureValue(specification?.Features, "Beard") ? "Beard" : FeatureValue(specification?.Features, "Moustache") ? "Moustache" : "Unknown");
}
private static bool FeatureValue(IReadOnlyList<string>? features, string value)
=> features?.Any(feature => feature.Contains(value, StringComparison.OrdinalIgnoreCase)) == true;
private static string NormaliseAge(string? value, string code)
{
var text = $"{value} {code}".ToLowerInvariant();
if (text.Contains("child")) return "Child";
if (text.Contains("youngteen")) return "Young Teen";
if (text.Contains("teen")) return "Older Teen";
if (text.Contains("youngadult") || text.Contains("young-adult")) return "Young Adult";
if (text.Contains("middleaged") || text.Contains("middle-aged")) return "Middle Aged";
if (text.Contains("elderly") || text.Contains("older")) return "Elderly";
return "Unknown";
}
private static string NormalisePresentation(string? value, string code)
{
var text = $"{value} {code}".ToLowerInvariant();
if (text.Contains("feminine") || text.Contains("woman") || text.Contains("female")) return "Feminine";
if (text.Contains("masculine") || text.Contains("man") || text.Contains("male")) return "Masculine";
if (text.Contains("androgynous")) return "Androgynous";
return "Androgynous";
}
private static string NormaliseHairColour(string? value, string code)
{
var text = $"{value} {code}".ToLowerInvariant();
if (text.Contains("black")) return "Black";
if (text.Contains("brown") || text.Contains("brunette") || text.Contains("darkbrown")) return "Brown";
if (text.Contains("blonde")) return "Blonde";
if (text.Contains("red") || text.Contains("auburn")) return "Red";
if (text.Contains("grey") || text.Contains("gray")) return "Grey";
if (text.Contains("white")) return "White";
return "Unknown";
}
private static string NormaliseSkinTone(string? value, string code)
{
var text = $"{value} {code}".ToLowerInvariant();
if (text.Contains("deep") || text.Contains("dark")) return "Dark";
if (text.Contains("medium") || text.Contains("lightmedium")) return "Medium";
if (text.Contains("light")) return "Light";
return "Unknown";
}
private static string NormaliseHairLength(string? value, string code)
{
var text = $"{value} {code}".ToLowerInvariant();
if (text.Contains("bald")) return "Bald";
if (text.Contains("short")) return "Short";
if (text.Contains("long")) return "Long";
if (text.Contains("medium")) return "Medium";
return "Unknown";
}
}
private sealed record MatchScore(CharacterCandidate Candidate, int Score, bool IsAlreadyAssignedToAnotherSignificantCharacter, IReadOnlyList<string> Rejections);
private sealed record MatchDiagnostics(int IllustrationLibraryItemID, string StableCode, int Score, IReadOnlyList<string> Rejections)
{
public static MatchDiagnostics From(MatchScore score)
=> new(score.Candidate.Item.IllustrationLibraryItemID, score.Candidate.Item.StableCode, score.Score, score.Rejections);
}
}