using System.Data; using System.Globalization; using System.Security.Cryptography; using System.Text; using System.Text.Json; using Dapper; using PlotLine.Data; using PlotLine.Models; using PlotLine.ViewModels; namespace PlotLine.Services; public interface INameIntelligenceService { Task EnsureReferenceDataImportedAsync(bool force = false, CancellationToken cancellationToken = default); Task ValidatePackageAsync(CancellationToken cancellationToken = default); Task> ListLocalesAsync(CancellationToken cancellationToken = default); Task RecogniseAsync(string rawName, string? localeCode = null, int? birthYear = null, CancellationToken cancellationToken = default); Task> SearchAsync(NameSearchFilter filter, CancellationToken cancellationToken = default); Task BuildLibraryAsync(NameSearchFilter filter, CancellationToken cancellationToken = default); } public sealed class NameIntelligenceService( INameIntelligenceRepository repository, ISqlConnectionFactory connectionFactory, IProjectRepository projects, IBookRepository books, ICurrentUserService currentUser, IWebHostEnvironment environment, ILogger logger) : INameIntelligenceService { private static readonly JsonSerializerOptions JsonOptions = new() { PropertyNameCaseInsensitive = true }; public async Task EnsureReferenceDataImportedAsync(bool force = false, CancellationToken cancellationToken = default) { var package = await ReadAndValidatePackageAsync(cancellationToken); if (package.Errors.Count > 0) { return package.ToResult(false, false, "Name reference package validation failed."); } var latest = await repository.GetLatestImportAsync(package.Manifest.PackageId, package.Manifest.Version, cancellationToken); if (!force && latest is not null && string.Equals(latest.PackageChecksum, package.Checksum, StringComparison.OrdinalIgnoreCase)) { return package.ToResult(true, true, "Name reference package is already imported."); } using var connection = connectionFactory.CreateConnection(); connection.Open(); using var transaction = connection.BeginTransaction(); var importId = 0; try { importId = await repository.StartImportAsync(package.Manifest.PackageId, package.Manifest.Version, package.Checksum, connection, transaction, cancellationToken); await repository.UpsertLocalesAsync(package.LocaleTable, connection, transaction, cancellationToken); await repository.UpsertGivenNamesAsync(package.GivenNameTable, connection, transaction, cancellationToken); await repository.UpsertRelationshipsAsync(package.RelationshipTable, connection, transaction, cancellationToken); foreach (var batch in package.UsageTables) { await repository.UpsertUsagesAsync(batch, connection, transaction, cancellationToken); } await repository.CompleteImportAsync(importId, true, package.LocaleRows, package.GivenNameRows, package.RelationshipRows, package.UsageRows, null, connection, transaction, cancellationToken); transaction.Commit(); return package.ToResult(true, false, "Name reference package imported."); } catch (Exception ex) when (ex is not OperationCanceledException) { logger.LogError(ex, "Name reference import failed for package {PackageId} {Version}.", package.Manifest.PackageId, package.Manifest.Version); try { if (importId > 0) { await repository.CompleteImportAsync(importId, false, 0, 0, 0, 0, ex.Message, connection, transaction, cancellationToken); } } finally { transaction.Rollback(); } return package.ToResult(false, false, "Name reference package import failed.", [ex.Message]); } } public async Task ValidatePackageAsync(CancellationToken cancellationToken = default) { var package = await ReadAndValidatePackageAsync(cancellationToken); return package.ToResult(package.Errors.Count == 0, false, package.Errors.Count == 0 ? "Name reference package validation succeeded." : "Name reference package validation failed."); } public Task> ListLocalesAsync(CancellationToken cancellationToken = default) => repository.ListLocalesAsync(cancellationToken); public Task RecogniseAsync(string rawName, string? localeCode = null, int? birthYear = null, CancellationToken cancellationToken = default) { var normalised = NormaliseName(rawName); return string.IsNullOrWhiteSpace(normalised) ? Task.FromResult(new NameRecognitionResult()) : repository.RecogniseAsync(normalised, localeCode, birthYear, cancellationToken); } public async Task> SearchAsync(NameSearchFilter filter, CancellationToken cancellationToken = default) { if (currentUser.UserId is not int userId || filter.ProjectID <= 0) { return []; } filter.Page = Math.Max(1, filter.Page); filter.PageSize = Math.Clamp(filter.PageSize, 1, 100); filter.UsedScope = string.IsNullOrWhiteSpace(filter.UsedScope) ? "Project" : filter.UsedScope.Trim(); if (!string.IsNullOrWhiteSpace(filter.GenderPreference) && !filter.GenderThreshold.HasValue) { filter.GenderThreshold = string.Equals(filter.GenderPreference, "Mixed", StringComparison.OrdinalIgnoreCase) ? 0.65m : 0.5m; } return await repository.SearchAsync(userId, filter, cancellationToken); } public async Task BuildLibraryAsync(NameSearchFilter filter, CancellationToken cancellationToken = default) { if (currentUser.UserId is not int userId || filter.ProjectID <= 0) { return null; } var project = await projects.GetForUserAsync(filter.ProjectID, userId); if (project is null) { return null; } var projectBooks = await books.ListByProjectAsync(filter.ProjectID); var locales = await ListLocalesAsync(cancellationToken); ApplyBookDefaults(filter, projectBooks, locales); var rows = await SearchAsync(filter, cancellationToken); return new NamesLibraryViewModel { Project = project, Filter = filter, Locales = locales, Books = projectBooks, Results = rows }; } private static void ApplyBookDefaults(NameSearchFilter filter, IReadOnlyList projectBooks, IReadOnlyList locales) { if (!filter.BookID.HasValue) { return; } var book = projectBooks.FirstOrDefault(item => item.BookID == filter.BookID.Value); if (book is null) { return; } if (string.IsNullOrWhiteSpace(filter.LocaleCode) && !string.IsNullOrWhiteSpace(book.PrimaryLocaleCode) && locales.Any(locale => string.Equals(locale.LocaleCode, book.PrimaryLocaleCode, StringComparison.OrdinalIgnoreCase))) { filter.LocaleCode = book.PrimaryLocaleCode; } if (!filter.BirthYear.HasValue) { filter.BirthYear = BookContextOptions.RepresentativeBirthYear(book.StoryEra); } } public static string NormaliseName(string? value) => string.IsNullOrWhiteSpace(value) ? string.Empty : value.Trim().Normalize(NormalizationForm.FormKC).ToUpperInvariant(); private async Task ReadAndValidatePackageAsync(CancellationToken cancellationToken) { var root = Path.Combine(environment.ContentRootPath, NameIntelligenceReferencePackage.RelativePath); var errors = new List(); var manifestPath = Path.Combine(root, "manifest.json"); if (!File.Exists(manifestPath)) { return ValidatedNamePackage.Empty(errors.Append($"Missing manifest: {manifestPath}").ToList()); } var manifest = JsonSerializer.Deserialize(await File.ReadAllTextAsync(manifestPath, cancellationToken), JsonOptions) ?? new NamePackageManifest(); if (!string.Equals(manifest.PackageId, NameIntelligenceReferencePackage.PackageId, StringComparison.Ordinal)) { errors.Add($"Unexpected package id '{manifest.PackageId}'."); } if (!string.Equals(manifest.Version, NameIntelligenceReferencePackage.Version, StringComparison.Ordinal)) { errors.Add($"Unexpected package version '{manifest.Version}'."); } var checksum = ComputePackageChecksum(root, manifest); var locales = ReadCsv(Path.Combine(root, "name_locales.csv"), ["locale_code", "display_name", "iso_country_code", "evidence_basis", "default_confidence", "sort_order", "is_active"], errors); var names = ReadCsv(Path.Combine(root, "given_names.csv"), ["given_name_key", "display_name", "normalised_name", "name_type", "is_word_ambiguous", "is_active", "notes"], errors); var relationships = ReadCsv(Path.Combine(root, "name_relationships.csv"), ["relationship_key", "from_given_name_key", "to_given_name_key", "relationship_type", "strength", "is_bidirectional", "is_active", "curation_source", "notes"], errors); var usages = ReadCsv(Path.Combine(root, "name_usages.csv"), ["usage_key", "given_name_key", "locale_code", "birth_year_from", "birth_year_to", "male_weight", "female_weight", "usage_weight", "confidence", "source_type", "source_reference", "is_active", "notes"], errors); ValidateCounts(manifest, locales.Count, names.Count, relationships.Count, usages.Count, errors); ValidatePackageRows(manifest, locales, names, relationships, usages, errors); return new ValidatedNamePackage( manifest, checksum, BuildLocaleTable(locales), BuildGivenNameTable(names), BuildRelationshipTable(relationships), BuildUsageTables(usages), locales.Count, names.Count, relationships.Count, usages.Count, errors); } private static void ValidateCounts(NamePackageManifest manifest, int localeRows, int nameRows, int relationshipRows, int usageRows, List errors) { if (manifest.Files.TryGetValue("name_locales", out var locales) && locales.Rows != localeRows) errors.Add($"Locale row count mismatch: manifest {locales.Rows}, file {localeRows}."); if (manifest.Files.TryGetValue("given_names", out var names) && names.Rows != nameRows) errors.Add($"Given name row count mismatch: manifest {names.Rows}, file {nameRows}."); if (manifest.Files.TryGetValue("name_relationships", out var relationships) && relationships.Rows != relationshipRows) errors.Add($"Relationship row count mismatch: manifest {relationships.Rows}, file {relationshipRows}."); if (manifest.Files.TryGetValue("name_usages", out var usages) && usages.Rows != usageRows) errors.Add($"Usage row count mismatch: manifest {usages.Rows}, file {usageRows}."); } private static void ValidatePackageRows(NamePackageManifest manifest, List> locales, List> names, List> relationships, List> usages, List errors) { var localeCodes = new HashSet(StringComparer.Ordinal); var nameKeys = new HashSet(StringComparer.Ordinal); var normalisedNames = new HashSet(StringComparer.OrdinalIgnoreCase); var relationshipKeys = new HashSet(StringComparer.Ordinal); var usageKeys = new HashSet(StringComparer.Ordinal); var usageBands = new Dictionary>(StringComparer.Ordinal); var nameTypes = manifest.ValueSets.GetValueOrDefault("nameType") ?? []; var relationshipTypes = manifest.ValueSets.GetValueOrDefault("relationshipType") ?? []; var sourceTypes = manifest.ValueSets.GetValueOrDefault("sourceType") ?? []; foreach (var row in locales) { RequireUnique(row["locale_code"], localeCodes, "locale_code", errors); ValidateDecimal(row["default_confidence"], "default_confidence", errors); ValidateBool(row["is_active"], "locale is_active", errors); } foreach (var row in names) { RequireUnique(row["given_name_key"], nameKeys, "given_name_key", errors); RequireUnique(row["normalised_name"], normalisedNames, "normalised_name", errors); RequireControlled(row["name_type"], nameTypes, "name_type", errors); ValidateBool(row["is_word_ambiguous"], "is_word_ambiguous", errors); ValidateBool(row["is_active"], "given name is_active", errors); } foreach (var row in relationships) { RequireUnique(row["relationship_key"], relationshipKeys, "relationship_key", errors); RequireForeign(row["from_given_name_key"], nameKeys, "relationship from_given_name_key", errors); RequireForeign(row["to_given_name_key"], nameKeys, "relationship to_given_name_key", errors); if (row["from_given_name_key"] == row["to_given_name_key"]) errors.Add($"Self relationship '{row["relationship_key"]}' is not allowed."); RequireControlled(row["relationship_type"], relationshipTypes, "relationship_type", errors); ValidateDecimal(row["strength"], "strength", errors); ValidateBool(row["is_bidirectional"], "is_bidirectional", errors); ValidateBool(row["is_active"], "relationship is_active", errors); } foreach (var row in usages) { RequireUnique(row["usage_key"], usageKeys, "usage_key", errors); RequireForeign(row["given_name_key"], nameKeys, "usage given_name_key", errors); RequireForeign(row["locale_code"], localeCodes, "usage locale_code", errors); var from = ValidateInt(row["birth_year_from"], "birth_year_from", errors); var to = ValidateInt(row["birth_year_to"], "birth_year_to", errors); if (from > to) errors.Add($"Usage '{row["usage_key"]}' has invalid year range."); var male = ValidateDecimal(row["male_weight"], "male_weight", errors); var female = ValidateDecimal(row["female_weight"], "female_weight", errors); ValidateDecimal(row["usage_weight"], "usage_weight", errors); ValidateDecimal(row["confidence"], "confidence", errors); if (Math.Abs((male + female) - 1m) > 0.0002m) errors.Add($"Usage '{row["usage_key"]}' gender weights do not total 1."); RequireControlled(row["source_type"], sourceTypes, "source_type", errors); ValidateBool(row["is_active"], "usage is_active", errors); var group = row["given_name_key"] + "|" + row["locale_code"]; if (!usageBands.TryGetValue(group, out var bands)) { bands = []; usageBands[group] = bands; } bands.Add((from, to)); } foreach (var pair in usageBands) { var bands = pair.Value.OrderBy(item => item.From).ToList(); if (bands.Count != 5 || bands[0].From != 1900 || bands[^1].To != 2026) { errors.Add($"Usage bands for {pair.Key} must contain five bands covering 1900-2026."); continue; } for (var i = 1; i < bands.Count; i++) { if (bands[i].From != bands[i - 1].To + 1) { errors.Add($"Usage bands for {pair.Key} are not contiguous."); break; } } } } private static List> ReadCsv(string path, IReadOnlyList expectedHeaders, List errors) { if (!File.Exists(path)) { errors.Add($"Missing CSV file: {path}"); return []; } using var reader = new StreamReader(path, Encoding.UTF8, detectEncodingFromByteOrderMarks: true); var headerLine = reader.ReadLine(); var headers = ParseCsvLine(headerLine ?? string.Empty); if (!headers.SequenceEqual(expectedHeaders, StringComparer.Ordinal)) { errors.Add($"Unexpected headers in {Path.GetFileName(path)}."); return []; } var rows = new List>(); string? line; while ((line = reader.ReadLine()) is not null) { if (string.IsNullOrWhiteSpace(line)) { continue; } var values = ParseCsvLine(line); if (values.Count != headers.Count) { errors.Add($"Invalid CSV column count in {Path.GetFileName(path)} at row {rows.Count + 2}."); continue; } rows.Add(headers.Zip(values, (key, value) => new { key, value }).ToDictionary(item => item.key, item => item.value, StringComparer.Ordinal)); } return rows; } private static List ParseCsvLine(string line) { var values = new List(); var builder = new StringBuilder(); var quoted = false; for (var i = 0; i < line.Length; i++) { var ch = line[i]; if (quoted) { if (ch == '"' && i + 1 < line.Length && line[i + 1] == '"') { builder.Append('"'); i++; } else if (ch == '"') { quoted = false; } else { builder.Append(ch); } } else if (ch == ',') { values.Add(builder.ToString()); builder.Clear(); } else if (ch == '"') { quoted = true; } else { builder.Append(ch); } } values.Add(builder.ToString()); return values; } private static DataTable BuildLocaleTable(IEnumerable> rows) { var table = new DataTable(); table.Columns.Add("LocaleCode", typeof(string)); table.Columns.Add("DisplayName", typeof(string)); table.Columns.Add("IsoCountryCode", typeof(string)); table.Columns.Add("EvidenceBasis", typeof(string)); table.Columns.Add("DefaultConfidence", typeof(decimal)); table.Columns.Add("SortOrder", typeof(int)); table.Columns.Add("IsActive", typeof(bool)); foreach (var row in rows) { table.Rows.Add(row["locale_code"], row["display_name"], row["iso_country_code"], row["evidence_basis"], Decimal(row["default_confidence"]), Int(row["sort_order"]), Bool(row["is_active"])); } return table; } private static DataTable BuildGivenNameTable(IEnumerable> rows) { var table = new DataTable(); table.Columns.Add("GivenNameKey", typeof(string)); table.Columns.Add("DisplayName", typeof(string)); table.Columns.Add("NormalisedName", typeof(string)); table.Columns.Add("NameType", typeof(string)); table.Columns.Add("IsWordAmbiguous", typeof(bool)); table.Columns.Add("IsActive", typeof(bool)); table.Columns.Add("Notes", typeof(string)); foreach (var row in rows) { table.Rows.Add(row["given_name_key"], row["display_name"], row["normalised_name"], row["name_type"], Bool(row["is_word_ambiguous"]), Bool(row["is_active"]), DbNull(row["notes"])); } return table; } private static DataTable BuildRelationshipTable(IEnumerable> rows) { var table = new DataTable(); table.Columns.Add("RelationshipKey", typeof(string)); table.Columns.Add("FromGivenNameKey", typeof(string)); table.Columns.Add("ToGivenNameKey", typeof(string)); table.Columns.Add("RelationshipType", typeof(string)); table.Columns.Add("Strength", typeof(decimal)); table.Columns.Add("IsBidirectional", typeof(bool)); table.Columns.Add("IsActive", typeof(bool)); table.Columns.Add("CurationSource", typeof(string)); table.Columns.Add("Notes", typeof(string)); foreach (var row in rows) { table.Rows.Add(row["relationship_key"], row["from_given_name_key"], row["to_given_name_key"], row["relationship_type"], Decimal(row["strength"]), Bool(row["is_bidirectional"]), Bool(row["is_active"]), DbNull(row["curation_source"]), DbNull(row["notes"])); } return table; } private static IReadOnlyList BuildUsageTables(IEnumerable> rows) { var tables = new List(); var table = NewUsageTable(); foreach (var row in rows) { if (table.Rows.Count >= 5000) { tables.Add(table); table = NewUsageTable(); } table.Rows.Add(row["usage_key"], row["given_name_key"], row["locale_code"], Int(row["birth_year_from"]), Int(row["birth_year_to"]), Decimal(row["male_weight"]), Decimal(row["female_weight"]), Decimal(row["usage_weight"]), Decimal(row["confidence"]), row["source_type"], DbNull(row["source_reference"]), Bool(row["is_active"]), DbNull(row["notes"])); } if (table.Rows.Count > 0) { tables.Add(table); } return tables; } private static DataTable NewUsageTable() { var table = new DataTable(); table.Columns.Add("UsageKey", typeof(string)); table.Columns.Add("GivenNameKey", typeof(string)); table.Columns.Add("LocaleCode", typeof(string)); table.Columns.Add("BirthYearFrom", typeof(int)); table.Columns.Add("BirthYearTo", typeof(int)); table.Columns.Add("MaleWeight", typeof(decimal)); table.Columns.Add("FemaleWeight", typeof(decimal)); table.Columns.Add("UsageWeight", typeof(decimal)); table.Columns.Add("Confidence", typeof(decimal)); table.Columns.Add("SourceType", typeof(string)); table.Columns.Add("SourceReference", typeof(string)); table.Columns.Add("IsActive", typeof(bool)); table.Columns.Add("Notes", typeof(string)); return table; } private static string ComputePackageChecksum(string root, NamePackageManifest manifest) { using var hash = SHA256.Create(); foreach (var fileName in manifest.ImportOrder) { var bytes = File.ReadAllBytes(Path.Combine(root, fileName)); hash.TransformBlock(bytes, 0, bytes.Length, null, 0); } hash.TransformFinalBlock([], 0, 0); return Convert.ToHexString(hash.Hash ?? []); } private static void RequireUnique(string value, HashSet values, string label, List errors) { if (string.IsNullOrWhiteSpace(value) || !values.Add(value)) { errors.Add($"Duplicate or empty {label}: '{value}'."); } } private static void RequireForeign(string value, HashSet values, string label, List errors) { if (!values.Contains(value)) { errors.Add($"Unknown {label}: '{value}'."); } } private static void RequireControlled(string value, IReadOnlyCollection allowed, string label, List errors) { if (!allowed.Contains(value, StringComparer.Ordinal)) { errors.Add($"Invalid {label}: '{value}'."); } } private static int ValidateInt(string value, string label, List errors) { if (int.TryParse(value, NumberStyles.Integer, CultureInfo.InvariantCulture, out var result)) { return result; } errors.Add($"Invalid integer {label}: '{value}'."); return 0; } private static decimal ValidateDecimal(string value, string label, List errors) { if (decimal.TryParse(value, NumberStyles.Number, CultureInfo.InvariantCulture, out var result) && result >= 0 && result <= 1) { return result; } errors.Add($"Invalid decimal {label}: '{value}'."); return 0; } private static void ValidateBool(string value, string label, List errors) { if (!bool.TryParse(value, out _)) { errors.Add($"Invalid boolean {label}: '{value}'."); } } private static int Int(string value) => int.Parse(value, CultureInfo.InvariantCulture); private static decimal Decimal(string value) => decimal.Parse(value, CultureInfo.InvariantCulture); private static bool Bool(string value) => bool.Parse(value); private static object DbNull(string value) => string.IsNullOrWhiteSpace(value) ? DBNull.Value : value; private sealed record ValidatedNamePackage( NamePackageManifest Manifest, string Checksum, DataTable LocaleTable, DataTable GivenNameTable, DataTable RelationshipTable, IReadOnlyList UsageTables, int LocaleRows, int GivenNameRows, int RelationshipRows, int UsageRows, IReadOnlyList Errors) { public static ValidatedNamePackage Empty(IReadOnlyList errors) => new(new(), string.Empty, new(), new(), new(), [], 0, 0, 0, 0, errors); public NameReferenceImportResult ToResult(bool succeeded, bool alreadyCurrent, string message, IReadOnlyList? extraErrors = null) => new(succeeded, alreadyCurrent, message, Manifest.PackageId, Manifest.Version, LocaleRows, GivenNameRows, RelationshipRows, UsageRows, Errors.Concat(extraErrors ?? []).ToList()); } private sealed class NamePackageManifest { public string PackageId { get; set; } = string.Empty; public string Version { get; set; } = string.Empty; public Dictionary Files { get; set; } = []; public Dictionary ValueSets { get; set; } = []; public string[] ImportOrder { get; set; } = []; } private sealed class NamePackageFile { public string File { get; set; } = string.Empty; public int Rows { get; set; } } }