292 lines
12 KiB
C#
292 lines
12 KiB
C#
using System.Globalization;
|
||
using System.Text.Json;
|
||
using System.Text.RegularExpressions;
|
||
using PlotLine.Data;
|
||
using PlotLine.Models;
|
||
|
||
namespace PlotLine.Services;
|
||
|
||
public sealed record CoreImportExistingLocationName(int LocationID, string Name);
|
||
|
||
public static partial class CoreImportBasicLocationDetector
|
||
{
|
||
private static readonly HashSet<string> ExplicitExcludedNames = new(StringComparer.OrdinalIgnoreCase)
|
||
{
|
||
"bar area", "bathroom", "bedroom", "corridor", "corner", "doorway", "garden",
|
||
"hall", "hallway", "inside", "kitchen", "office", "outside", "room", "stairs",
|
||
"toilet", "upstairs", "downstairs", "waiting room", "ward"
|
||
};
|
||
|
||
private static readonly HashSet<string> DirectionalStarts = new(StringComparer.OrdinalIgnoreCase)
|
||
{
|
||
"behind", "beside", "inside", "outside", "near", "under", "upstairs", "downstairs"
|
||
};
|
||
|
||
public static CoreImportBasicLocationDiscoveryResult Discover(
|
||
IReadOnlyList<SceneManuscriptSourceRecord> sourceScenes,
|
||
IReadOnlyList<CoreImportExistingLocationName> existingNames)
|
||
{
|
||
var existingIndex = existingNames
|
||
.Select(item => new { Key = StableKey(item.Name), item.LocationID })
|
||
.Where(item => !string.IsNullOrWhiteSpace(item.Key))
|
||
.GroupBy(item => item.Key, StringComparer.Ordinal)
|
||
.ToDictionary(group => group.Key, group => group.First().LocationID, StringComparer.Ordinal);
|
||
|
||
var orderedScenes = sourceScenes
|
||
.OrderBy(scene => scene.ChapterID)
|
||
.ThenBy(scene => scene.SourceStartParagraph ?? int.MaxValue)
|
||
.ThenBy(scene => scene.SceneID)
|
||
.ToList();
|
||
var chapterNumbers = orderedScenes
|
||
.GroupBy(scene => scene.ChapterID)
|
||
.Select((group, index) => new { group.Key, ChapterNumber = index + 1 })
|
||
.ToDictionary(item => item.Key, item => item.ChapterNumber);
|
||
var sceneNumbers = orderedScenes
|
||
.GroupBy(scene => scene.ChapterID)
|
||
.SelectMany(group => group.Select((scene, index) => new { scene.SceneID, SceneNumber = index + 1 }))
|
||
.ToDictionary(item => item.SceneID, item => item.SceneNumber);
|
||
|
||
var observations = new List<LocationObservation>();
|
||
var excluded = 0;
|
||
foreach (var scene in orderedScenes)
|
||
{
|
||
if (string.IsNullOrWhiteSpace(scene.SourceText))
|
||
{
|
||
continue;
|
||
}
|
||
|
||
var text = scene.SourceText;
|
||
foreach (var observation in ExtractObservations(text, scene, chapterNumbers.GetValueOrDefault(scene.ChapterID), sceneNumbers.GetValueOrDefault(scene.SceneID)))
|
||
{
|
||
if (IsExcluded(observation.Name))
|
||
{
|
||
excluded++;
|
||
continue;
|
||
}
|
||
|
||
observations.Add(observation);
|
||
}
|
||
}
|
||
|
||
var grouped = observations
|
||
.GroupBy(item => StableKey(item.Name), StringComparer.Ordinal)
|
||
.Where(group => !string.IsNullOrWhiteSpace(group.Key))
|
||
.Select(group => BuildCandidate(group.Key, group.ToList(), existingIndex))
|
||
.Where(candidate => !candidate.ExistingLocationID.HasValue)
|
||
.OrderBy(candidate => candidate.FirstChapterNumber)
|
||
.ThenBy(candidate => candidate.FirstSceneNumber)
|
||
.ThenBy(candidate => candidate.FirstOccurrenceIndex)
|
||
.Select((candidate, index) =>
|
||
{
|
||
candidate.SortOrder = (index + 1) * 10;
|
||
return candidate;
|
||
})
|
||
.ToList();
|
||
|
||
var keys = grouped.Select(candidate => candidate.StableKey).ToHashSet(StringComparer.Ordinal);
|
||
foreach (var candidate in grouped)
|
||
{
|
||
if (IsLinearLocation(candidate.ProposedName))
|
||
{
|
||
continue;
|
||
}
|
||
|
||
var parentKey = candidate.Evidence
|
||
.Select(evidence => InferParentKey(evidence.EvidenceText, candidate.ProposedName))
|
||
.FirstOrDefault(key => !string.IsNullOrWhiteSpace(key) && keys.Contains(key));
|
||
if (!string.IsNullOrWhiteSpace(parentKey) && !string.Equals(parentKey, candidate.StableKey, StringComparison.Ordinal))
|
||
{
|
||
candidate.ParentCandidateKey = parentKey;
|
||
}
|
||
}
|
||
|
||
return new CoreImportBasicLocationDiscoveryResult
|
||
{
|
||
RawObservationCount = observations.Count,
|
||
ExcludedObservationCount = excluded,
|
||
Candidates = grouped
|
||
};
|
||
}
|
||
|
||
public static string StableKey(string? name)
|
||
{
|
||
var normalized = NormalizeName(name);
|
||
if (string.IsNullOrWhiteSpace(normalized))
|
||
{
|
||
return string.Empty;
|
||
}
|
||
|
||
normalized = Regex.Replace(normalized, @"^the\s+", string.Empty, RegexOptions.IgnoreCase | RegexOptions.CultureInvariant);
|
||
normalized = Regex.Replace(normalized, @"[^\p{L}\p{N}]+", "-", RegexOptions.CultureInvariant).Trim('-').ToLowerInvariant();
|
||
return normalized;
|
||
}
|
||
|
||
public static string NormalizeName(string? name)
|
||
{
|
||
if (string.IsNullOrWhiteSpace(name))
|
||
{
|
||
return string.Empty;
|
||
}
|
||
|
||
var cleaned = name.Replace('’', '\'').Replace('–', '-').Replace('—', '-');
|
||
cleaned = Regex.Replace(cleaned, @"\s+", " ", RegexOptions.CultureInvariant).Trim();
|
||
cleaned = cleaned.Trim(' ', '.', ',', ';', ':', '!', '?', '"', '\'');
|
||
cleaned = Regex.Replace(cleaned, @"^(?:at|in|into|inside|outside|to|from|near|beside|behind|on)\s+", string.Empty, RegexOptions.IgnoreCase | RegexOptions.CultureInvariant);
|
||
return cleaned.Trim();
|
||
}
|
||
|
||
private static IEnumerable<LocationObservation> ExtractObservations(
|
||
string text,
|
||
SceneManuscriptSourceRecord scene,
|
||
int chapterNumber,
|
||
int sceneNumber)
|
||
{
|
||
var seen = new HashSet<string>(StringComparer.Ordinal);
|
||
foreach (var regex in DetectionRegexes())
|
||
{
|
||
foreach (Match match in regex.Matches(text))
|
||
{
|
||
var value = NormalizeName(match.Groups["name"].Success ? match.Groups["name"].Value : match.Value);
|
||
var key = $"{match.Index}:{StableKey(value)}";
|
||
if (string.IsNullOrWhiteSpace(value) || !seen.Add(key))
|
||
{
|
||
continue;
|
||
}
|
||
|
||
yield return new LocationObservation(
|
||
value,
|
||
scene.SceneID,
|
||
scene.ChapterID,
|
||
chapterNumber,
|
||
sceneNumber,
|
||
match.Index,
|
||
EvidenceWindow(text, match.Index, match.Length));
|
||
}
|
||
}
|
||
}
|
||
|
||
private static CoreImportBasicLocationCandidate BuildCandidate(
|
||
string key,
|
||
IReadOnlyList<LocationObservation> observations,
|
||
IReadOnlyDictionary<string, int> existingIndex)
|
||
{
|
||
var ordered = observations
|
||
.OrderBy(item => item.ChapterNumber)
|
||
.ThenBy(item => item.SceneNumber)
|
||
.ThenBy(item => item.OccurrenceIndex)
|
||
.ToList();
|
||
var names = ordered
|
||
.Select(item => NormalizeName(item.Name))
|
||
.Where(name => !string.IsNullOrWhiteSpace(name))
|
||
.Distinct(StringComparer.OrdinalIgnoreCase)
|
||
.OrderByDescending(name => ordered.Count(item => string.Equals(NormalizeName(item.Name), name, StringComparison.OrdinalIgnoreCase)))
|
||
.ThenBy(name => name, StringComparer.OrdinalIgnoreCase)
|
||
.ToList();
|
||
var preferred = names.FirstOrDefault() ?? ordered[0].Name;
|
||
var candidate = new CoreImportBasicLocationCandidate
|
||
{
|
||
StableKey = key,
|
||
ProposedName = preferred,
|
||
PreferredName = preferred,
|
||
KnownNamesJson = JsonSerializer.Serialize(names),
|
||
ExistingLocationID = existingIndex.GetValueOrDefault(key) == 0 ? null : existingIndex[key],
|
||
FirstChapterNumber = ordered[0].ChapterNumber,
|
||
FirstSceneNumber = ordered[0].SceneNumber,
|
||
FirstOccurrenceIndex = ordered[0].OccurrenceIndex,
|
||
ObservationCount = ordered.Count,
|
||
Evidence = ordered
|
||
.Take(5)
|
||
.Select((item, index) => new CoreImportBasicLocationEvidence
|
||
{
|
||
SceneID = item.SceneID,
|
||
ChapterID = item.ChapterID,
|
||
ChapterNumber = item.ChapterNumber,
|
||
SceneNumber = item.SceneNumber,
|
||
OccurrenceIndex = item.OccurrenceIndex,
|
||
EvidenceText = item.EvidenceText,
|
||
SortOrder = (index + 1) * 10
|
||
})
|
||
.ToList()
|
||
};
|
||
return candidate;
|
||
}
|
||
|
||
private static bool IsExcluded(string name)
|
||
{
|
||
var normalized = NormalizeName(name);
|
||
var key = StableKey(normalized).Replace('-', ' ');
|
||
if (string.IsNullOrWhiteSpace(key) || ExplicitExcludedNames.Contains(key))
|
||
{
|
||
return true;
|
||
}
|
||
|
||
var first = key.Split(' ', StringSplitOptions.RemoveEmptyEntries).FirstOrDefault();
|
||
if (first is not null && DirectionalStarts.Contains(first))
|
||
{
|
||
return true;
|
||
}
|
||
|
||
if (Regex.IsMatch(normalized, @"^(?:1[5-9]\d{2}|20\d{2}|21\d{2})\s+", RegexOptions.CultureInvariant))
|
||
{
|
||
return true;
|
||
}
|
||
|
||
return key.Length < 3 || Regex.IsMatch(normalized, @"^(?:Chapter|Scene)\s+\d+", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant);
|
||
}
|
||
|
||
private static bool IsLinearLocation(string name)
|
||
=> Regex.IsMatch(name, @"\b(?:Road|Street|Lane|Avenue|Drive|Way|Close|Crescent|Square|Place|Hill|Hills)\b$", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant);
|
||
|
||
private static string? InferParentKey(string evidenceText, string childName)
|
||
{
|
||
var child = Regex.Escape(childName);
|
||
var road = @"(?<parent>(?:The\s+)?[A-Z][\p{L}'-]+(?:\s+[A-Z][\p{L}'-]+){0,4}\s+(?:Road|Street|Lane|Avenue|Drive|Way|Close|Crescent|Square|Place|Hill|Hills))";
|
||
var pattern = $@"{child}.{{0,60}}\b(?:on|in|at)\s+{road}|{road}.{{0,60}}{child}";
|
||
var match = Regex.Match(evidenceText, pattern, RegexOptions.CultureInvariant);
|
||
return match.Success ? StableKey(match.Groups["parent"].Value) : null;
|
||
}
|
||
|
||
private static string EvidenceWindow(string text, int start, int length)
|
||
{
|
||
var left = Math.Max(0, start - 100);
|
||
var right = Math.Min(text.Length, start + length + 140);
|
||
var snippet = text[left..right];
|
||
snippet = Regex.Replace(snippet, @"\s+", " ", RegexOptions.CultureInvariant).Trim();
|
||
return snippet.Length <= 360 ? snippet : snippet[..360].Trim();
|
||
}
|
||
|
||
private static IReadOnlyList<Regex> DetectionRegexes()
|
||
=>
|
||
[
|
||
AddressRegex(),
|
||
PossessiveResidenceRegex(),
|
||
NamedRoadRegex(),
|
||
NamedVenueRegex(),
|
||
ThePlaceRegex()
|
||
];
|
||
|
||
[GeneratedRegex(@"\b(?<name>\d+\s+(?:The\s+)?[A-Z][\p{L}'-]+(?:\s+[A-Z][\p{L}'-]+){0,4}\s+(?:Road|Street|Lane|Avenue|Drive|Way|Close|Crescent|Square|Place|Hill|Hills))\b", RegexOptions.CultureInvariant)]
|
||
private static partial Regex AddressRegex();
|
||
|
||
[GeneratedRegex(@"\b(?<name>[A-Z][\p{L}'-]+(?:\s+[A-Z][\p{L}'-]+){0,2}['’]s\s+(?:house|home|flat|apartment|cottage))\b", RegexOptions.CultureInvariant)]
|
||
private static partial Regex PossessiveResidenceRegex();
|
||
|
||
[GeneratedRegex(@"\b(?<name>(?:The\s+)?[A-Z][\p{L}'-]+(?:\s+[A-Z][\p{L}'-]+){0,4}\s+(?:Road|Street|Lane|Avenue|Drive|Way|Close|Crescent|Square|Place|Hill|Hills))\b", RegexOptions.CultureInvariant)]
|
||
private static partial Regex NamedRoadRegex();
|
||
|
||
[GeneratedRegex(@"\b(?<name>(?:The\s+)?[A-Z][\p{L}'-]+(?:\s+[A-Z][\p{L}'-]+){0,4}\s+(?:Inn|Pub|Hotel|School|Hospital|Station|Park|Church|Cathedral|University|College|Library|Shop|Store|Cafe|Café|Restaurant|Club|Arcade|Centre|Center|House|Cottage|Farm|Hall|Manor|Estate|Gardens|Common|Bridge|Junction|Airport|Museum|Theatre|Theater))\b", RegexOptions.CultureInvariant)]
|
||
private static partial Regex NamedVenueRegex();
|
||
|
||
[GeneratedRegex(@"\b(?<name>The\s+[A-Z][\p{L}'-]+(?:ies|s))\b", RegexOptions.CultureInvariant)]
|
||
private static partial Regex ThePlaceRegex();
|
||
|
||
private sealed record LocationObservation(
|
||
string Name,
|
||
int SceneID,
|
||
int ChapterID,
|
||
int ChapterNumber,
|
||
int SceneNumber,
|
||
int OccurrenceIndex,
|
||
string EvidenceText);
|
||
}
|