474 lines
22 KiB
C#
474 lines
22 KiB
C#
using System.Diagnostics;
|
|
using System.Text;
|
|
using System.Text.Json;
|
|
using System.Text.Json.Serialization;
|
|
using Microsoft.Extensions.Options;
|
|
using PlotLine.Data;
|
|
using PlotLine.Models;
|
|
using PlotLine.ViewModels;
|
|
|
|
namespace PlotLine.Services;
|
|
|
|
public interface IWholeBookPlotIntelligenceService
|
|
{
|
|
Task<StoryIntelligenceAuthorPlotContextViewModel?> GetAuthorPlotContextAsync(int? bookId = null);
|
|
Task SaveAuthorPlotContextAsync(int projectId, int bookId, IReadOnlyList<string> descriptions);
|
|
Task<WholeBookPlotSynthesisRun?> QueueAfterCharacterResolutionAsync(int projectId, int bookId, int userId);
|
|
Task<WholeBookPlotSynthesisRun?> RetryAsync(int bookId, int userId);
|
|
Task<WholeBookPlotSynthesisRun?> GetCurrentAsync(int bookId);
|
|
Task<bool> ProcessNextAsync(CancellationToken cancellationToken);
|
|
string BuildPromptForTest(WholeBookPlotSynthesisContext context);
|
|
StoryIntelligenceResponseContract BuildResponseContractForTest();
|
|
}
|
|
|
|
public sealed class WholeBookPlotIntelligenceService(
|
|
IWholeBookPlotIntelligenceRepository repository,
|
|
IStoryIntelligencePipelineRepository pipelines,
|
|
IBookRepository books,
|
|
IStoryIntelligenceClient client,
|
|
IOptions<StoryIntelligenceOptions> options,
|
|
IStoryIntelligenceProgressNotifier notifier,
|
|
ICurrentUserService currentUser,
|
|
ILogger<WholeBookPlotIntelligenceService> logger) : IWholeBookPlotIntelligenceService
|
|
{
|
|
private const string PromptVersion = "Whole-Book-Plot-Synthesis-V1";
|
|
private const int MaxPromptCharacters = 900_000;
|
|
private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web)
|
|
{
|
|
WriteIndented = true
|
|
};
|
|
private readonly StoryIntelligenceOptions settings = options.Value;
|
|
|
|
public async Task<StoryIntelligenceAuthorPlotContextViewModel?> GetAuthorPlotContextAsync(int? bookId = null)
|
|
{
|
|
var userId = currentUser.UserId;
|
|
if (!userId.HasValue)
|
|
{
|
|
return null;
|
|
}
|
|
|
|
var book = bookId.HasValue ? await books.GetAsync(bookId.Value) : null;
|
|
if (book is null && bookId.HasValue)
|
|
{
|
|
return null;
|
|
}
|
|
|
|
var descriptions = book is null
|
|
? new List<string>()
|
|
: (await repository.ListAuthorSeedsAsync(book.BookID)).Select(seed => seed.Description).ToList();
|
|
|
|
while (descriptions.Count < 3)
|
|
{
|
|
descriptions.Add(string.Empty);
|
|
}
|
|
|
|
return new StoryIntelligenceAuthorPlotContextViewModel
|
|
{
|
|
ProjectID = book?.ProjectID ?? 0,
|
|
BookID = book?.BookID ?? 0,
|
|
BookTitle = book?.BookDisplayTitle ?? string.Empty,
|
|
Descriptions = descriptions
|
|
};
|
|
}
|
|
|
|
public Task SaveAuthorPlotContextAsync(int projectId, int bookId, IReadOnlyList<string> descriptions)
|
|
=> repository.SaveAuthorSeedsAsync(new AuthorPlotSeedSaveRequest
|
|
{
|
|
ProjectID = projectId,
|
|
BookID = bookId,
|
|
Descriptions = descriptions
|
|
});
|
|
|
|
public async Task<WholeBookPlotSynthesisRun?> QueueAfterCharacterResolutionAsync(int projectId, int bookId, int userId)
|
|
{
|
|
var pipeline = await pipelines.GetByBookForUserAsync(bookId, userId);
|
|
var latestRunId = pipeline?.LastRunID ?? (await pipelines.ListCommittedRunsByBookAsync(bookId, userId))
|
|
.OrderByDescending(run => run.StoryIntelligenceRunID)
|
|
.FirstOrDefault()
|
|
?.StoryIntelligenceRunID;
|
|
return await repository.QueueAsync(new WholeBookPlotSynthesisQueueRequest
|
|
{
|
|
ProjectID = projectId,
|
|
BookID = bookId,
|
|
UserID = userId,
|
|
SourceStoryIntelligenceRunID = latestRunId,
|
|
CharacterResolutionVersion = $"CharacterImport:{bookId}"
|
|
});
|
|
}
|
|
|
|
public Task<WholeBookPlotSynthesisRun?> RetryAsync(int bookId, int userId)
|
|
=> repository.RetryAsync(bookId, userId);
|
|
|
|
public Task<WholeBookPlotSynthesisRun?> GetCurrentAsync(int bookId)
|
|
=> repository.GetCurrentAsync(bookId);
|
|
|
|
public async Task<bool> ProcessNextAsync(CancellationToken cancellationToken)
|
|
{
|
|
var run = await repository.ClaimNextAsync(Math.Max(15, settings.ClaimLeaseMinutes));
|
|
if (run is null)
|
|
{
|
|
return false;
|
|
}
|
|
|
|
var stopwatch = Stopwatch.StartNew();
|
|
try
|
|
{
|
|
var context = await repository.BuildContextAsync(run.ProjectID, run.BookID);
|
|
if (!context.SourceAudit.CanReconstruct)
|
|
{
|
|
throw new InvalidOperationException(
|
|
$"Scene manuscript source audit failed. Scenes={context.SourceAudit.SceneCount}, sources={context.SourceAudit.SourceRowCount}, missing={context.SourceAudit.MissingSourceCount}, duplicateSourceScenes={context.SourceAudit.DuplicateSourceSceneCount}.");
|
|
}
|
|
|
|
var prompt = BuildPrompt(context);
|
|
if (prompt.Length > MaxPromptCharacters)
|
|
{
|
|
prompt = BuildPrompt(context, includeOptionalContext: false);
|
|
}
|
|
|
|
if (prompt.Length > MaxPromptCharacters)
|
|
{
|
|
throw new InvalidOperationException("Whole-book Plot Intelligence prompt would exceed the configured safe context limit. No manuscript text was silently truncated.");
|
|
}
|
|
|
|
var result = await client.ExecutePromptAsync(
|
|
prompt,
|
|
PromptVersion,
|
|
cancellationToken,
|
|
settings.EffectiveWholeBookPlotSynthesisModel,
|
|
WholeBookPlotSynthesisMaxOutputTokens(),
|
|
responseContract: WholeBookPlotSynthesisStructuredOutputSchema.Contract);
|
|
var outputJson = ExtractOutputText(result.RawResponseText);
|
|
var parsed = PreserveUnmatchedAuthorSeedsForReview(
|
|
WholeBookPlotSynthesisResult.FromJson(outputJson),
|
|
context.AuthorSeeds);
|
|
stopwatch.Stop();
|
|
|
|
await repository.CompleteAsync(new WholeBookPlotSynthesisCompletionRequest
|
|
{
|
|
WholeBookPlotSynthesisRunID = run.WholeBookPlotSynthesisRunID,
|
|
Model = result.Model,
|
|
OpenAIResponseID = ExtractResponseId(result.RawResponseText),
|
|
InputTokens = result.InputTokens,
|
|
OutputTokens = result.OutputTokens,
|
|
CachedInputTokens = ExtractCachedTokens(result.RawResponseText),
|
|
TotalTokens = (result.InputTokens ?? 0) + (result.OutputTokens ?? 0) > 0
|
|
? (result.InputTokens ?? 0) + (result.OutputTokens ?? 0)
|
|
: null,
|
|
DurationMs = stopwatch.ElapsedMilliseconds,
|
|
ResultJson = outputJson
|
|
}, parsed);
|
|
await notifier.PublishGlobalAsync(run.UserID);
|
|
logger.LogInformation(
|
|
"Whole-book Plot Intelligence completed. RunID={RunID} ProjectID={ProjectID} BookID={BookID} PlotLines={PlotLineCount} DurationMs={DurationMs}",
|
|
run.WholeBookPlotSynthesisRunID,
|
|
run.ProjectID,
|
|
run.BookID,
|
|
parsed.PlotLines.Count,
|
|
stopwatch.ElapsedMilliseconds);
|
|
return true;
|
|
}
|
|
catch (Exception ex) when (ex is not OperationCanceledException)
|
|
{
|
|
stopwatch.Stop();
|
|
await repository.FailAsync(run.WholeBookPlotSynthesisRunID, ex.Message, ex.ToString(), stopwatch.ElapsedMilliseconds);
|
|
await notifier.PublishGlobalAsync(run.UserID);
|
|
logger.LogError(ex, "Whole-book Plot Intelligence failed. RunID={RunID} BookID={BookID}", run.WholeBookPlotSynthesisRunID, run.BookID);
|
|
return true;
|
|
}
|
|
}
|
|
|
|
public string BuildPromptForTest(WholeBookPlotSynthesisContext context)
|
|
=> BuildPrompt(context);
|
|
|
|
public StoryIntelligenceResponseContract BuildResponseContractForTest()
|
|
=> WholeBookPlotSynthesisStructuredOutputSchema.Contract;
|
|
|
|
private int WholeBookPlotSynthesisMaxOutputTokens()
|
|
=> settings.WholeBookPlotSynthesisMaxOutputTokens.GetValueOrDefault(settings.MaxOutputTokens) > 0
|
|
? settings.WholeBookPlotSynthesisMaxOutputTokens.GetValueOrDefault(settings.MaxOutputTokens)
|
|
: settings.MaxOutputTokens;
|
|
|
|
public static WholeBookPlotSynthesisResult PreserveUnmatchedAuthorSeedsForReviewForTest(
|
|
WholeBookPlotSynthesisResult result,
|
|
IReadOnlyList<AuthorPlotSeed> authorSeeds)
|
|
=> PreserveUnmatchedAuthorSeedsForReview(result, authorSeeds);
|
|
|
|
private static WholeBookPlotSynthesisResult PreserveUnmatchedAuthorSeedsForReview(
|
|
WholeBookPlotSynthesisResult result,
|
|
IReadOnlyList<AuthorPlotSeed> authorSeeds)
|
|
{
|
|
var matchedSeedIds = result.PlotLines
|
|
.Where(plotLine => plotLine.AuthorPlotSeedID.HasValue)
|
|
.Select(plotLine => plotLine.AuthorPlotSeedID!.Value)
|
|
.ToHashSet();
|
|
var unmatchedSeeds = authorSeeds
|
|
.Where(seed => !matchedSeedIds.Contains(seed.AuthorPlotSeedID))
|
|
.ToList();
|
|
if (unmatchedSeeds.Count == 0)
|
|
{
|
|
return result;
|
|
}
|
|
|
|
var plotLines = result.PlotLines.ToList();
|
|
foreach (var seed in unmatchedSeeds)
|
|
{
|
|
plotLines.Add(new WholeBookPlotSynthesisPlotLineResult
|
|
{
|
|
Key = $"author-seed-{seed.AuthorPlotSeedID}",
|
|
ProposedTitle = "Author-provided Plot Line",
|
|
Description = seed.Description,
|
|
Significance = "Author intent",
|
|
BookStatus = "Limited current-book evidence",
|
|
ExistingPlotLineID = seed.MatchedPlotLineID,
|
|
ExistingPlotLineMatchConfidence = seed.MatchedPlotLineID.HasValue ? 1m : null,
|
|
ExistingPlotLineMatchReason = seed.MatchedPlotLineID.HasValue
|
|
? "Author seed is already linked to an existing Project Plot Line."
|
|
: null,
|
|
AuthorPlotSeedID = seed.AuthorPlotSeedID,
|
|
SeedMatchConfidence = null,
|
|
SeedMatchReason = "Author-provided seed retained for review; no detected whole-book Plot Line referenced this seed.",
|
|
Confidence = null,
|
|
Threads = []
|
|
});
|
|
}
|
|
|
|
return new WholeBookPlotSynthesisResult
|
|
{
|
|
Metadata = result.Metadata,
|
|
PlotLines = plotLines
|
|
};
|
|
}
|
|
|
|
private static string BuildPrompt(WholeBookPlotSynthesisContext context, bool includeOptionalContext = true)
|
|
{
|
|
var builder = new StringBuilder();
|
|
builder.AppendLine("You are PlotDirector's Whole-Book Plot Intelligence engine.");
|
|
builder.AppendLine("Read the complete manuscript and identify sustained plot lines, subordinate threads, and every meaningful scene touch.");
|
|
builder.AppendLine("Return only data that conforms to the strict schema. Reference only SceneIDs, PlotLineIDs, PlotThreadIDs, and AuthorPlotSeedIDs supplied in this request.");
|
|
builder.AppendLine("A major Plot Line is a sustained narrative arc, mystery, conflict, goal, hidden truth, character struggle, investigation, relationship arc, or continuing question.");
|
|
builder.AppendLine("Do not promote isolated incidents, simple presence, locations, or recurring props without narrative development into major Plot Lines.");
|
|
builder.AppendLine("A Thread is a meaningful subordinate strand within a Plot Line. Do not fragment every scene development into a separate Thread.");
|
|
builder.AppendLine("Create an event for a scene when the scene meaningfully mentions, reinforces, advances, complicates, foreshadows, questions, misdirects, reveals, pays off, contradicts, or resolves the thread.");
|
|
builder.AppendLine("Do not attach a scene merely because associated characters are present.");
|
|
builder.AppendLine("Use Mentioned only as the fallback for genuine reinforcement when no more informative event type applies.");
|
|
builder.AppendLine();
|
|
builder.AppendLine("[BOOK]");
|
|
builder.AppendLine($"BookID: {context.BookID}");
|
|
builder.AppendLine($"Title: {context.BookTitle}");
|
|
builder.AppendLine();
|
|
|
|
if (includeOptionalContext)
|
|
{
|
|
builder.AppendLine("[CANONICAL CHARACTERS]");
|
|
foreach (var character in context.Characters)
|
|
{
|
|
builder.AppendLine($"CharacterID: {character.CharacterID}; CanonicalName: {character.CanonicalName}; Aliases: {string.Join(", ", character.Aliases)}");
|
|
}
|
|
builder.AppendLine();
|
|
|
|
builder.AppendLine("[EXISTING PROJECT PLOT LINES]");
|
|
foreach (var plotLine in context.ExistingPlotLines)
|
|
{
|
|
builder.AppendLine($"PlotLineID: {plotLine.PlotLineID}; Name: {plotLine.Name}; Importance: {plotLine.Importance}; BookID: {plotLine.BookID?.ToString() ?? "series"}");
|
|
if (!string.IsNullOrWhiteSpace(plotLine.Description))
|
|
{
|
|
builder.AppendLine($"Description: {plotLine.Description}");
|
|
}
|
|
|
|
foreach (var thread in plotLine.Threads)
|
|
{
|
|
builder.AppendLine($" ThreadID: {thread.PlotThreadID}; Title: {thread.Title}; Status: {thread.Status}");
|
|
}
|
|
}
|
|
builder.AppendLine();
|
|
|
|
builder.AppendLine("[AUTHOR PLOT SEEDS]");
|
|
foreach (var seed in context.AuthorSeeds)
|
|
{
|
|
builder.AppendLine($"AuthorPlotSeedID: {seed.AuthorPlotSeedID}; Description: {seed.Description}");
|
|
}
|
|
builder.AppendLine();
|
|
}
|
|
|
|
builder.AppendLine("[MANUSCRIPT]");
|
|
foreach (var chapter in context.Chapters)
|
|
{
|
|
builder.AppendLine();
|
|
builder.AppendLine($"[CHAPTER {chapter.ChapterNumber:0.##}]");
|
|
builder.AppendLine($"ChapterID: {chapter.ChapterID}");
|
|
builder.AppendLine($"Title: {chapter.ChapterTitle}");
|
|
foreach (var scene in chapter.Scenes)
|
|
{
|
|
builder.AppendLine();
|
|
builder.AppendLine("[SCENE]");
|
|
builder.AppendLine($"SceneID: {scene.SceneID}");
|
|
builder.AppendLine($"SceneNumber: {scene.SceneNumber:0.##}");
|
|
builder.AppendLine($"Title: {scene.SceneTitle}");
|
|
builder.AppendLine($"POV: {scene.PovCharacterName}");
|
|
builder.AppendLine($"Setting: {scene.StructuralSetting}");
|
|
builder.AppendLine("<text>");
|
|
builder.AppendLine(scene.SourceText);
|
|
builder.AppendLine("</text>");
|
|
}
|
|
}
|
|
|
|
return builder.ToString();
|
|
}
|
|
|
|
private static string ExtractOutputText(string rawResponseText)
|
|
{
|
|
var response = JsonSerializer.Deserialize<WholeBookOpenAIResponseEnvelope>(rawResponseText, JsonOptions)
|
|
?? throw new JsonException("OpenAI response envelope was empty.");
|
|
return response.Output?
|
|
.SelectMany(item => item.Content ?? [])
|
|
.FirstOrDefault(content => string.Equals(content.Type, "output_text", StringComparison.OrdinalIgnoreCase)
|
|
&& !string.IsNullOrWhiteSpace(content.Text))
|
|
?.Text
|
|
?.Trim()
|
|
?? throw new JsonException("OpenAI response did not contain output_text content.");
|
|
}
|
|
|
|
private static string? ExtractResponseId(string rawResponseText)
|
|
{
|
|
try
|
|
{
|
|
return JsonSerializer.Deserialize<WholeBookOpenAIResponseEnvelope>(rawResponseText, JsonOptions)?.Id;
|
|
}
|
|
catch (JsonException)
|
|
{
|
|
return null;
|
|
}
|
|
}
|
|
|
|
private static int? ExtractCachedTokens(string rawResponseText)
|
|
{
|
|
try
|
|
{
|
|
return JsonSerializer.Deserialize<WholeBookOpenAIResponseEnvelope>(rawResponseText, JsonOptions)?.Usage?.InputTokenDetails?.CachedTokens;
|
|
}
|
|
catch (JsonException)
|
|
{
|
|
return null;
|
|
}
|
|
}
|
|
|
|
private sealed class WholeBookOpenAIResponseEnvelope
|
|
{
|
|
public string? Id { get; init; }
|
|
public List<WholeBookOpenAIResponseOutputItem>? Output { get; init; }
|
|
public WholeBookOpenAIResponseUsage? Usage { get; init; }
|
|
}
|
|
|
|
private sealed class WholeBookOpenAIResponseOutputItem
|
|
{
|
|
public List<WholeBookOpenAIResponseContentItem>? Content { get; init; }
|
|
}
|
|
|
|
private sealed class WholeBookOpenAIResponseContentItem
|
|
{
|
|
public string? Type { get; init; }
|
|
public string? Text { get; init; }
|
|
}
|
|
|
|
private sealed class WholeBookOpenAIResponseUsage
|
|
{
|
|
[JsonPropertyName("input_tokens")]
|
|
public int? InputTokens { get; init; }
|
|
|
|
[JsonPropertyName("output_tokens")]
|
|
public int? OutputTokens { get; init; }
|
|
|
|
[JsonPropertyName("input_tokens_details")]
|
|
public WholeBookOpenAIInputTokenDetails? InputTokenDetails { get; init; }
|
|
}
|
|
|
|
private sealed class WholeBookOpenAIInputTokenDetails
|
|
{
|
|
[JsonPropertyName("cached_tokens")]
|
|
public int? CachedTokens { get; init; }
|
|
}
|
|
}
|
|
|
|
public static class WholeBookPlotSynthesisStructuredOutputSchema
|
|
{
|
|
public static readonly StoryIntelligenceResponseContract Contract = new()
|
|
{
|
|
Name = "whole_book_plot_synthesis",
|
|
Strict = true,
|
|
Schema = new Dictionary<string, object>
|
|
{
|
|
["type"] = "object",
|
|
["additionalProperties"] = false,
|
|
["required"] = new[] { "metadata", "plotLines" },
|
|
["properties"] = new Dictionary<string, object>
|
|
{
|
|
["metadata"] = ObjectSchema(new Dictionary<string, object>
|
|
{
|
|
["bookStatus"] = StringSchema(),
|
|
["summary"] = StringSchema()
|
|
}, "bookStatus", "summary"),
|
|
["plotLines"] = ArraySchema(ObjectSchema(new Dictionary<string, object>
|
|
{
|
|
["key"] = StringSchema(),
|
|
["proposedTitle"] = StringSchema(),
|
|
["description"] = StringSchema(),
|
|
["significance"] = StringSchema(),
|
|
["bookStatus"] = StringSchema(),
|
|
["existingPlotLineID"] = NullableIntegerSchema(),
|
|
["existingPlotLineMatchConfidence"] = NullableNumberSchema(),
|
|
["existingPlotLineMatchReason"] = NullableStringSchema(),
|
|
["authorPlotSeedID"] = NullableIntegerSchema(),
|
|
["seedMatchConfidence"] = NullableNumberSchema(),
|
|
["seedMatchReason"] = NullableStringSchema(),
|
|
["confidence"] = NullableNumberSchema(),
|
|
["threads"] = ArraySchema(ObjectSchema(new Dictionary<string, object>
|
|
{
|
|
["key"] = StringSchema(),
|
|
["proposedTitle"] = StringSchema(),
|
|
["description"] = StringSchema(),
|
|
["status"] = StringSchema(),
|
|
["existingPlotThreadID"] = NullableIntegerSchema(),
|
|
["existingThreadMatchConfidence"] = NullableNumberSchema(),
|
|
["matchReason"] = NullableStringSchema(),
|
|
["confidence"] = NullableNumberSchema(),
|
|
["events"] = ArraySchema(ObjectSchema(new Dictionary<string, object>
|
|
{
|
|
["key"] = StringSchema(),
|
|
["sceneID"] = IntegerSchema(),
|
|
["eventType"] = EnumSchema("Introduced", "Mentioned", "Developed", "Complicated", "Clue Planted", "Question Raised", "Question Answered", "False Lead", "Revealed", "Payoff", "Resolved", "Contradicted", "Retconned", "Note"),
|
|
["title"] = StringSchema(),
|
|
["description"] = StringSchema(),
|
|
["evidence"] = StringSchema(),
|
|
["confidence"] = NullableNumberSchema()
|
|
}, "key", "sceneID", "eventType", "title", "description", "evidence", "confidence"))
|
|
}, "key", "proposedTitle", "description", "status", "existingPlotThreadID", "existingThreadMatchConfidence", "matchReason", "confidence", "events"))
|
|
}, "key", "proposedTitle", "description", "significance", "bookStatus", "existingPlotLineID", "existingPlotLineMatchConfidence", "existingPlotLineMatchReason", "authorPlotSeedID", "seedMatchConfidence", "seedMatchReason", "confidence", "threads"))
|
|
}
|
|
}
|
|
};
|
|
|
|
private static Dictionary<string, object> ObjectSchema(Dictionary<string, object> properties, params string[] required)
|
|
=> new()
|
|
{
|
|
["type"] = "object",
|
|
["additionalProperties"] = false,
|
|
["required"] = required,
|
|
["properties"] = properties
|
|
};
|
|
|
|
private static Dictionary<string, object> ArraySchema(object items)
|
|
=> new()
|
|
{
|
|
["type"] = "array",
|
|
["items"] = items
|
|
};
|
|
|
|
private static Dictionary<string, object> StringSchema() => new() { ["type"] = "string" };
|
|
private static Dictionary<string, object> IntegerSchema() => new() { ["type"] = "integer" };
|
|
private static Dictionary<string, object> NullableIntegerSchema() => new() { ["type"] = new[] { "integer", "null" } };
|
|
private static Dictionary<string, object> NullableNumberSchema() => new() { ["type"] = new[] { "number", "null" }, ["minimum"] = 0, ["maximum"] = 1 };
|
|
private static Dictionary<string, object> NullableStringSchema() => new() { ["type"] = new[] { "string", "null" } };
|
|
private static Dictionary<string, object> EnumSchema(params string[] values) => new() { ["type"] = "string", ["enum"] = values };
|
|
}
|