429 lines
20 KiB
C#
429 lines
20 KiB
C#
using System.Diagnostics;
|
|
using System.Text;
|
|
using System.Text.Json;
|
|
using System.Text.Json.Serialization;
|
|
using Microsoft.Extensions.Options;
|
|
using PlotLine.Data;
|
|
using PlotLine.Models;
|
|
using PlotLine.ViewModels;
|
|
|
|
namespace PlotLine.Services;
|
|
|
|
public interface ICharacterEnrichmentService
|
|
{
|
|
Task<CharacterEnrichmentRun?> QueueAfterCharacterResolutionAsync(int projectId, int bookId, int userId);
|
|
Task<CharacterEnrichmentRun?> QueueForCharacterAsync(int characterId, int userId, int? bookId = null);
|
|
Task<CharacterEnrichmentRun?> RetryAsync(int bookId, int userId);
|
|
Task<CharacterEnrichmentRun?> GetCurrentAsync(int bookId);
|
|
Task<CharacterEnrichmentRun?> GetCurrentForCharacterAsync(int characterId, int userId);
|
|
Task<CharacterEnrichmentCharacterStatus?> GetCharacterStatusAsync(int characterId, int userId);
|
|
Task<bool> ProcessNextAsync(CancellationToken cancellationToken);
|
|
string BuildPromptForTest(CharacterEnrichmentContext context);
|
|
StoryIntelligenceResponseContract BuildResponseContractForTest();
|
|
}
|
|
|
|
public sealed class CharacterEnrichmentService(
|
|
ICharacterEnrichmentRepository repository,
|
|
IStoryIntelligencePipelineRepository pipelines,
|
|
IStoryIntelligenceClient client,
|
|
IOptions<StoryIntelligenceOptions> options,
|
|
IStoryIntelligenceProgressNotifier notifier,
|
|
ILogger<CharacterEnrichmentService> logger) : ICharacterEnrichmentService
|
|
{
|
|
private const string PromptVersion = "Character-Enrichment-V1";
|
|
private const int BatchSize = 8;
|
|
private const int MaxPromptCharacters = 350_000;
|
|
private const decimal SummaryMinimumConfidence = 0.65m;
|
|
private const decimal FactMinimumConfidence = 0.9m;
|
|
private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web)
|
|
{
|
|
WriteIndented = true
|
|
};
|
|
private readonly StoryIntelligenceOptions settings = options.Value;
|
|
|
|
public async Task<CharacterEnrichmentRun?> QueueAfterCharacterResolutionAsync(int projectId, int bookId, int userId)
|
|
{
|
|
var pipeline = await pipelines.GetByBookForUserAsync(bookId, userId);
|
|
return await repository.QueueAsync(new CharacterEnrichmentQueueRequest
|
|
{
|
|
ProjectID = projectId,
|
|
BookID = bookId,
|
|
UserID = userId,
|
|
StoryIntelligenceBookPipelineID = pipeline?.StoryIntelligenceBookPipelineID,
|
|
CharacterResolutionVersion = $"CharacterImport:{bookId}"
|
|
});
|
|
}
|
|
|
|
public async Task<CharacterEnrichmentRun?> QueueForCharacterAsync(int characterId, int userId, int? bookId = null)
|
|
{
|
|
var run = await repository.QueueCharacterAsync(new CharacterEnrichmentCharacterQueueRequest
|
|
{
|
|
CharacterID = characterId,
|
|
UserID = userId,
|
|
BookID = bookId
|
|
});
|
|
if (run.WasCoalesced)
|
|
{
|
|
logger.LogInformation(
|
|
"Manual character enrichment request ignored because an active run already exists. RunID={RunID} ProjectID={ProjectID} BookID={BookID} CharacterID={CharacterID} Status={Status}",
|
|
run.CharacterEnrichmentRunID,
|
|
run.ProjectID,
|
|
run.BookID,
|
|
run.CharacterID,
|
|
run.Status);
|
|
}
|
|
else
|
|
{
|
|
logger.LogInformation(
|
|
"Manual character enrichment queued. RunID={RunID} ProjectID={ProjectID} BookID={BookID} CharacterID={CharacterID} Status={Status}",
|
|
run.CharacterEnrichmentRunID,
|
|
run.ProjectID,
|
|
run.BookID,
|
|
run.CharacterID,
|
|
run.Status);
|
|
}
|
|
return run;
|
|
}
|
|
|
|
public Task<CharacterEnrichmentRun?> RetryAsync(int bookId, int userId)
|
|
=> repository.RetryAsync(bookId, userId);
|
|
|
|
public Task<CharacterEnrichmentRun?> GetCurrentAsync(int bookId)
|
|
=> repository.GetCurrentAsync(bookId);
|
|
|
|
public Task<CharacterEnrichmentRun?> GetCurrentForCharacterAsync(int characterId, int userId)
|
|
=> repository.GetCurrentForCharacterAsync(characterId, userId);
|
|
|
|
public Task<CharacterEnrichmentCharacterStatus?> GetCharacterStatusAsync(int characterId, int userId)
|
|
=> repository.GetCharacterStatusAsync(characterId, userId);
|
|
|
|
public async Task<bool> ProcessNextAsync(CancellationToken cancellationToken)
|
|
{
|
|
var run = await repository.ClaimNextAsync(Math.Max(15, settings.ClaimLeaseMinutes));
|
|
if (run is null)
|
|
{
|
|
return false;
|
|
}
|
|
|
|
var stopwatch = Stopwatch.StartNew();
|
|
var results = new List<CharacterEnrichmentCharacterResult>();
|
|
var inputTokens = 0;
|
|
var outputTokens = 0;
|
|
var model = settings.EffectiveWholeBookPlotSynthesisModel;
|
|
try
|
|
{
|
|
logger.LogInformation(
|
|
"Character enrichment starting. RunID={RunID} ProjectID={ProjectID} BookID={BookID} CharacterID={CharacterID}",
|
|
run.CharacterEnrichmentRunID,
|
|
run.ProjectID,
|
|
run.BookID,
|
|
run.CharacterID);
|
|
await PublishProgressAsync(run, 0, 0, "Preparing character analysis", "Preparing character analysis...", cancellationToken);
|
|
|
|
var context = await repository.BuildContextAsync(run.ProjectID, run.BookID, run.CharacterID);
|
|
var characters = context.Characters
|
|
.Where(character => character.Scenes.Count > 0)
|
|
.OrderBy(character => character.CharacterName, StringComparer.OrdinalIgnoreCase)
|
|
.ToList();
|
|
logger.LogInformation(
|
|
"Character enrichment context prepared. RunID={RunID} ProjectID={ProjectID} BookID={BookID} CharacterID={CharacterID} Characters={CharacterCount}",
|
|
run.CharacterEnrichmentRunID,
|
|
run.ProjectID,
|
|
run.BookID,
|
|
run.CharacterID,
|
|
characters.Count);
|
|
|
|
await PublishProgressAsync(run, characters.Count, 0, "Analysing characters", "Analysing characters...", cancellationToken);
|
|
foreach (var batch in characters.Chunk(BatchSize))
|
|
{
|
|
cancellationToken.ThrowIfCancellationRequested();
|
|
var batchContext = new CharacterEnrichmentContext
|
|
{
|
|
ProjectID = context.ProjectID,
|
|
BookID = context.BookID,
|
|
BookTitle = context.BookTitle,
|
|
StoryEra = context.StoryEra,
|
|
SeriesStartDate = context.SeriesStartDate,
|
|
Characters = batch
|
|
};
|
|
var prompt = BuildPrompt(batchContext);
|
|
if (prompt.Length > MaxPromptCharacters)
|
|
{
|
|
throw new InvalidOperationException("Character enrichment prompt would exceed the configured safe context limit. No manuscript text was silently truncated.");
|
|
}
|
|
|
|
logger.LogInformation(
|
|
"Character enrichment batch starting. RunID={RunID} CharacterID={CharacterID} BatchCharacters={BatchCharacters} Processed={Processed}/{Total}",
|
|
run.CharacterEnrichmentRunID,
|
|
run.CharacterID,
|
|
batch.Length,
|
|
results.Count,
|
|
characters.Count);
|
|
await PublishProgressAsync(run, characters.Count, results.Count, "Creating summaries and extracting character details", "Creating summaries and extracting character details...", cancellationToken);
|
|
|
|
var clientResult = await client.ExecutePromptAsync(
|
|
prompt,
|
|
PromptVersion,
|
|
cancellationToken,
|
|
settings.EffectiveWholeBookPlotSynthesisModel,
|
|
settings.WholeBookPlotSynthesisMaxOutputTokens.GetValueOrDefault(settings.MaxOutputTokens),
|
|
responseContract: CharacterEnrichmentStructuredOutputSchema.Contract);
|
|
model = clientResult.Model;
|
|
inputTokens += clientResult.InputTokens ?? 0;
|
|
outputTokens += clientResult.OutputTokens ?? 0;
|
|
var parsed = CharacterEnrichmentResult.FromJson(ExtractOutputText(clientResult.RawResponseText));
|
|
results.AddRange(FilterResults(parsed.Characters, batch.Select(character => character.CharacterID).ToHashSet()));
|
|
await PublishProgressAsync(run, characters.Count, Math.Min(characters.Count, results.Count), "Saving character analysis", "Saving character analysis...", cancellationToken);
|
|
}
|
|
|
|
stopwatch.Stop();
|
|
await repository.CompleteAsync(new CharacterEnrichmentCompletionRequest
|
|
{
|
|
CharacterEnrichmentRunID = run.CharacterEnrichmentRunID,
|
|
Model = model,
|
|
InputTokens = inputTokens == 0 ? null : inputTokens,
|
|
OutputTokens = outputTokens == 0 ? null : outputTokens,
|
|
TotalTokens = inputTokens + outputTokens == 0 ? null : inputTokens + outputTokens,
|
|
DurationMs = stopwatch.ElapsedMilliseconds
|
|
}, results);
|
|
await notifier.PublishCharacterEnrichmentAsync(ToProgress(run, CharacterEnrichmentStatuses.Completed, results.Count, results.Count, "Complete", "Character summaries are ready."));
|
|
logger.LogInformation(
|
|
"Character enrichment completed. RunID={RunID} ProjectID={ProjectID} BookID={BookID} CharacterID={CharacterID} Characters={CharacterCount} DurationMs={DurationMs}",
|
|
run.CharacterEnrichmentRunID,
|
|
run.ProjectID,
|
|
run.BookID,
|
|
run.CharacterID,
|
|
results.Count,
|
|
stopwatch.ElapsedMilliseconds);
|
|
return true;
|
|
}
|
|
catch (Exception ex) when (ex is not OperationCanceledException)
|
|
{
|
|
stopwatch.Stop();
|
|
await repository.FailAsync(run.CharacterEnrichmentRunID, ex.Message, ex.ToString(), stopwatch.ElapsedMilliseconds);
|
|
await notifier.PublishCharacterEnrichmentAsync(ToProgress(run, CharacterEnrichmentStatuses.Failed, run.TotalCharacters, run.ProcessedCharacters, "Failed", "Character enrichment failed.", ex.Message));
|
|
logger.LogError(ex, "Character enrichment failed. RunID={RunID} ProjectID={ProjectID} BookID={BookID} CharacterID={CharacterID}", run.CharacterEnrichmentRunID, run.ProjectID, run.BookID, run.CharacterID);
|
|
return true;
|
|
}
|
|
}
|
|
|
|
public string BuildPromptForTest(CharacterEnrichmentContext context)
|
|
=> BuildPrompt(context);
|
|
|
|
public StoryIntelligenceResponseContract BuildResponseContractForTest()
|
|
=> CharacterEnrichmentStructuredOutputSchema.Contract;
|
|
|
|
private async Task PublishProgressAsync(CharacterEnrichmentRun run, int total, int processed, string stage, string message, CancellationToken cancellationToken)
|
|
{
|
|
await repository.UpdateProgressAsync(new CharacterEnrichmentProgressUpdate
|
|
{
|
|
CharacterEnrichmentRunID = run.CharacterEnrichmentRunID,
|
|
TotalCharacters = total,
|
|
ProcessedCharacters = processed,
|
|
CurrentStage = stage,
|
|
CurrentMessage = message
|
|
});
|
|
await notifier.PublishCharacterEnrichmentAsync(ToProgress(run, CharacterEnrichmentStatuses.Running, total, processed, stage, message));
|
|
}
|
|
|
|
private static IReadOnlyList<CharacterEnrichmentCharacterResult> FilterResults(
|
|
IReadOnlyList<CharacterEnrichmentCharacterResult> results,
|
|
HashSet<int> allowedCharacterIds)
|
|
=> results
|
|
.Where(result => allowedCharacterIds.Contains(result.CharacterID))
|
|
.Select(result => new CharacterEnrichmentCharacterResult
|
|
{
|
|
CharacterID = result.CharacterID,
|
|
Summary = Clamp(result.Confidence) >= SummaryMinimumConfidence ? Clean(result.Summary, 2500) : string.Empty,
|
|
Confidence = Clamp(result.Confidence),
|
|
DateOfBirth = StrongFact(result.DateOfBirth),
|
|
AgeAtStartOfSeries = StrongFact(result.AgeAtStartOfSeries),
|
|
Height = StrongFact(result.Height),
|
|
EyeColour = StrongFact(result.EyeColour),
|
|
EvidenceSceneIDs = result.EvidenceSceneIDs.Where(id => id > 0).Distinct().Take(20).ToList()
|
|
})
|
|
.Where(result => !string.IsNullOrWhiteSpace(result.Summary)
|
|
|| !string.IsNullOrWhiteSpace(result.DateOfBirth.Value)
|
|
|| !string.IsNullOrWhiteSpace(result.AgeAtStartOfSeries.Value)
|
|
|| !string.IsNullOrWhiteSpace(result.Height.Value)
|
|
|| !string.IsNullOrWhiteSpace(result.EyeColour.Value))
|
|
.ToList();
|
|
|
|
private static CharacterEnrichmentFactResult StrongFact(CharacterEnrichmentFactResult fact)
|
|
=> Clamp(fact.Confidence) >= FactMinimumConfidence && !string.IsNullOrWhiteSpace(fact.Value) && fact.SceneID is > 0
|
|
? new CharacterEnrichmentFactResult
|
|
{
|
|
Value = Clean(fact.Value, 120),
|
|
Confidence = Clamp(fact.Confidence),
|
|
Evidence = Clean(fact.Evidence, 700),
|
|
SceneID = fact.SceneID
|
|
}
|
|
: new CharacterEnrichmentFactResult();
|
|
|
|
private static string BuildPrompt(CharacterEnrichmentContext context)
|
|
{
|
|
var builder = new StringBuilder();
|
|
builder.AppendLine("You are PlotDirector's Character Enrichment engine.");
|
|
builder.AppendLine("Create concise author-facing character summaries and extract only strongly evidenced character facts.");
|
|
builder.AppendLine("Use only the supplied manuscript scenes. Do not invent, infer from behaviour alone, speculate about future events, or list every appearance.");
|
|
builder.AppendLine("For dateOfBirth, ageAtStartOfSeries, height, and eyeColour, return a value only when direct manuscript wording strongly supports it. Otherwise return null value and confidence 0.");
|
|
builder.AppendLine("If a character already has a stored value, preserve it by returning null unless the manuscript contains stronger direct evidence; the save layer will still refuse to overwrite existing author data.");
|
|
builder.AppendLine();
|
|
builder.AppendLine("[BOOK]");
|
|
builder.AppendLine($"BookID: {context.BookID}");
|
|
builder.AppendLine($"Title: {context.BookTitle}");
|
|
builder.AppendLine($"Story era: {context.StoryEra}");
|
|
builder.AppendLine($"Series start date: {context.SeriesStartDate:yyyy-MM-dd}");
|
|
builder.AppendLine();
|
|
builder.AppendLine("[CHARACTERS]");
|
|
foreach (var character in context.Characters)
|
|
{
|
|
builder.AppendLine();
|
|
builder.AppendLine($"CharacterID: {character.CharacterID}");
|
|
builder.AppendLine($"Name: {character.CharacterName}");
|
|
builder.AppendLine($"Aliases: {string.Join(", ", character.Aliases)}");
|
|
builder.AppendLine($"Existing birth date: {character.BirthDate:yyyy-MM-dd}");
|
|
builder.AppendLine($"Existing age at series start: {character.AgeAtSeriesStart}");
|
|
builder.AppendLine($"Existing height: {character.Height}");
|
|
builder.AppendLine($"Existing eye colour: {character.EyeColour}");
|
|
builder.AppendLine($"Existing description: {character.DefaultDescription}");
|
|
builder.AppendLine("[SCENES]");
|
|
foreach (var scene in character.Scenes.Take(60))
|
|
{
|
|
builder.AppendLine($"SceneID: {scene.SceneID}; Chapter {scene.ChapterNumber:0.##} {scene.ChapterTitle}; Scene {scene.SceneNumber:0.##} {scene.SceneTitle}");
|
|
builder.AppendLine("<text>");
|
|
builder.AppendLine(scene.SourceText);
|
|
builder.AppendLine("</text>");
|
|
}
|
|
}
|
|
|
|
return builder.ToString();
|
|
}
|
|
|
|
private static CharacterEnrichmentProgressEvent ToProgress(CharacterEnrichmentRun run, string status, int total, int processed, string stage, string message, string? error = null)
|
|
=> new()
|
|
{
|
|
CharacterEnrichmentRunID = run.CharacterEnrichmentRunID,
|
|
UserID = run.UserID,
|
|
ProjectID = run.ProjectID,
|
|
BookID = run.BookID,
|
|
CharacterID = run.CharacterID,
|
|
Status = status,
|
|
TotalCharacters = total,
|
|
ProcessedCharacters = processed,
|
|
CurrentStage = stage,
|
|
CurrentMessage = message,
|
|
ErrorMessage = error,
|
|
UpdatedUtc = DateTime.UtcNow
|
|
};
|
|
|
|
private static string ExtractOutputText(string rawResponseText)
|
|
{
|
|
using var response = JsonDocument.Parse(rawResponseText);
|
|
foreach (var output in response.RootElement.GetProperty("output").EnumerateArray())
|
|
{
|
|
if (!output.TryGetProperty("content", out var content))
|
|
{
|
|
continue;
|
|
}
|
|
|
|
foreach (var item in content.EnumerateArray())
|
|
{
|
|
if (item.TryGetProperty("type", out var type)
|
|
&& string.Equals(type.GetString(), "output_text", StringComparison.OrdinalIgnoreCase)
|
|
&& item.TryGetProperty("text", out var text)
|
|
&& !string.IsNullOrWhiteSpace(text.GetString()))
|
|
{
|
|
return text.GetString()!.Trim();
|
|
}
|
|
}
|
|
}
|
|
|
|
throw new JsonException("OpenAI response did not contain output_text content.");
|
|
}
|
|
|
|
private static decimal Clamp(decimal value)
|
|
=> Math.Clamp(value, 0m, 1m);
|
|
|
|
private static string Clean(string? value, int maxLength)
|
|
{
|
|
var clean = value?.Trim();
|
|
if (string.IsNullOrWhiteSpace(clean))
|
|
{
|
|
return string.Empty;
|
|
}
|
|
|
|
return clean.Length <= maxLength ? clean : clean[..maxLength].TrimEnd();
|
|
}
|
|
}
|
|
|
|
public static class CharacterEnrichmentStructuredOutputSchema
|
|
{
|
|
public static readonly StoryIntelligenceResponseContract Contract = new()
|
|
{
|
|
Name = "character_enrichment",
|
|
Strict = true,
|
|
Schema = new Dictionary<string, object>
|
|
{
|
|
["type"] = "object",
|
|
["additionalProperties"] = false,
|
|
["required"] = new[] { "schemaVersion", "characters" },
|
|
["properties"] = new Dictionary<string, object>
|
|
{
|
|
["schemaVersion"] = StringSchema(),
|
|
["characters"] = ArraySchema(ObjectSchema(new Dictionary<string, object>
|
|
{
|
|
["characterID"] = IntegerSchema(),
|
|
["summary"] = StringSchema(),
|
|
["confidence"] = NumberSchema(),
|
|
["dateOfBirth"] = FactSchema(),
|
|
["ageAtStartOfSeries"] = FactSchema(),
|
|
["height"] = FactSchema(),
|
|
["eyeColour"] = FactSchema(),
|
|
["evidenceSceneIDs"] = ArraySchema(IntegerSchema())
|
|
}, "characterID", "summary", "confidence", "dateOfBirth", "ageAtStartOfSeries", "height", "eyeColour", "evidenceSceneIDs"))
|
|
}
|
|
}
|
|
};
|
|
|
|
private static Dictionary<string, object> FactSchema()
|
|
=> ObjectSchema(new Dictionary<string, object>
|
|
{
|
|
["value"] = NullableStringSchema(),
|
|
["confidence"] = NumberSchema(),
|
|
["evidence"] = NullableStringSchema(),
|
|
["sceneID"] = NullableIntegerSchema()
|
|
}, "value", "confidence", "evidence", "sceneID");
|
|
|
|
private static Dictionary<string, object> ObjectSchema(Dictionary<string, object> properties, params string[] required)
|
|
=> new()
|
|
{
|
|
["type"] = "object",
|
|
["additionalProperties"] = false,
|
|
["required"] = required,
|
|
["properties"] = properties
|
|
};
|
|
|
|
private static Dictionary<string, object> ArraySchema(object items)
|
|
=> new()
|
|
{
|
|
["type"] = "array",
|
|
["items"] = items
|
|
};
|
|
|
|
private static Dictionary<string, object> StringSchema()
|
|
=> new() { ["type"] = "string" };
|
|
|
|
private static Dictionary<string, object> NullableStringSchema()
|
|
=> new() { ["type"] = new[] { "string", "null" } };
|
|
|
|
private static Dictionary<string, object> IntegerSchema()
|
|
=> new() { ["type"] = "integer" };
|
|
|
|
private static Dictionary<string, object> NullableIntegerSchema()
|
|
=> new() { ["type"] = new[] { "integer", "null" } };
|
|
|
|
private static Dictionary<string, object> NumberSchema()
|
|
=> new() { ["type"] = "number", ["minimum"] = 0, ["maximum"] = 1 };
|
|
}
|