759 lines
30 KiB
C#
759 lines
30 KiB
C#
using System.Diagnostics;
|
|
using System.Net;
|
|
using System.Net.Http.Headers;
|
|
using System.Text;
|
|
using System.Text.Json;
|
|
using System.Text.Json.Serialization;
|
|
using Microsoft.Extensions.Caching.Memory;
|
|
using Microsoft.Extensions.Options;
|
|
using PlotLine.Models;
|
|
using PlotLine.ViewModels;
|
|
|
|
namespace PlotLine.Services;
|
|
|
|
public interface IStoryPromptRepository
|
|
{
|
|
Task<string> LoadPromptAsync(string fileName, CancellationToken cancellationToken);
|
|
}
|
|
|
|
public interface IStoryPromptBuilder
|
|
{
|
|
string BuildPrompt(string promptTemplate, string sceneContextJson, string sceneText);
|
|
}
|
|
|
|
public interface IStoryPromptVersionService
|
|
{
|
|
string GetPromptVersion(string fileName);
|
|
}
|
|
|
|
public interface IStoryIntelligenceClient
|
|
{
|
|
Task<StoryIntelligenceClientResult> ExecutePromptAsync(
|
|
string completedPrompt,
|
|
string promptVersion,
|
|
CancellationToken cancellationToken,
|
|
string? modelOverride = null,
|
|
int? maxOutputTokensOverride = null);
|
|
StoryIntelligenceClientConfigurationStatus GetConfigurationStatus();
|
|
}
|
|
|
|
public interface IStoryIntelligenceDiagnosticsService
|
|
{
|
|
Task<StoryIntelligenceDiagnosticsViewModel> GetDiagnosticsAsync(CancellationToken cancellationToken);
|
|
}
|
|
|
|
public interface IStoryIntelligenceDryRunService
|
|
{
|
|
StoryIntelligenceDryRunViewModel GetDefault();
|
|
Task<StoryIntelligenceDryRunViewModel> ExecuteAsync(StoryIntelligenceDryRunForm form, CancellationToken cancellationToken);
|
|
}
|
|
|
|
public interface IChapterStructureDryRunService
|
|
{
|
|
ChapterStructureDryRunViewModel GetDefault();
|
|
Task<ChapterStructureDryRunViewModel> ExecuteAsync(ChapterStructureDryRunForm form, CancellationToken cancellationToken);
|
|
}
|
|
|
|
public sealed class StoryPromptRepository(
|
|
IWebHostEnvironment environment,
|
|
IMemoryCache cache,
|
|
ILogger<StoryPromptRepository> logger) : IStoryPromptRepository
|
|
{
|
|
private const string PromptFolder = "Docs/AI";
|
|
private const string RepositoryPromptFolder = "PlotLine/Docs/AI";
|
|
|
|
public async Task<string> LoadPromptAsync(string fileName, CancellationToken cancellationToken)
|
|
{
|
|
if (string.IsNullOrWhiteSpace(fileName)
|
|
|| fileName.IndexOfAny(Path.GetInvalidFileNameChars()) >= 0
|
|
|| fileName.Contains(Path.DirectorySeparatorChar)
|
|
|| fileName.Contains(Path.AltDirectorySeparatorChar))
|
|
{
|
|
throw new InvalidOperationException("Prompt filename is invalid.");
|
|
}
|
|
|
|
var cacheKey = $"story-prompt:{fileName}";
|
|
if (cache.TryGetValue(cacheKey, out string? cachedPrompt) && !string.IsNullOrWhiteSpace(cachedPrompt))
|
|
{
|
|
return cachedPrompt;
|
|
}
|
|
|
|
var promptPath = ResolvePromptPath(fileName);
|
|
if (promptPath is null)
|
|
{
|
|
throw new FileNotFoundException($"Story Intelligence prompt file '{fileName}' was not found in /{PromptFolder}/ or /{RepositoryPromptFolder}/.", Path.Combine(environment.ContentRootPath, PromptFolder, fileName));
|
|
}
|
|
|
|
var prompt = await File.ReadAllTextAsync(promptPath, cancellationToken);
|
|
if (string.IsNullOrWhiteSpace(prompt))
|
|
{
|
|
throw new InvalidOperationException($"Story Intelligence prompt file '{fileName}' is empty.");
|
|
}
|
|
|
|
cache.Set(cacheKey, prompt, TimeSpan.FromMinutes(15));
|
|
logger.LogInformation("Loaded Story Intelligence prompt {PromptFile} from {PromptPath}.", fileName, promptPath);
|
|
return prompt;
|
|
}
|
|
|
|
private string? ResolvePromptPath(string fileName)
|
|
{
|
|
var promptPath = Path.Combine(environment.ContentRootPath, PromptFolder, fileName);
|
|
if (File.Exists(promptPath))
|
|
{
|
|
return promptPath;
|
|
}
|
|
|
|
var repositoryPromptPath = Path.Combine(environment.ContentRootPath, RepositoryPromptFolder, fileName);
|
|
if (File.Exists(repositoryPromptPath))
|
|
{
|
|
return repositoryPromptPath;
|
|
}
|
|
|
|
return null;
|
|
}
|
|
}
|
|
|
|
public sealed class StoryPromptBuilder : IStoryPromptBuilder
|
|
{
|
|
public string BuildPrompt(string promptTemplate, string sceneContextJson, string sceneText)
|
|
{
|
|
if (string.IsNullOrWhiteSpace(promptTemplate))
|
|
{
|
|
throw new InvalidOperationException("Prompt template is empty.");
|
|
}
|
|
|
|
return promptTemplate
|
|
.Replace("{{SCENE_CONTEXT_JSON}}", sceneContextJson ?? string.Empty, StringComparison.Ordinal)
|
|
.Replace("{{SCENE_TEXT}}", sceneText ?? string.Empty, StringComparison.Ordinal);
|
|
}
|
|
}
|
|
|
|
public sealed class StoryPromptVersionService : IStoryPromptVersionService
|
|
{
|
|
public string GetPromptVersion(string fileName)
|
|
{
|
|
if (string.IsNullOrWhiteSpace(fileName))
|
|
{
|
|
throw new InvalidOperationException("Prompt filename is required.");
|
|
}
|
|
|
|
var safeName = Path.GetFileName(fileName);
|
|
return Path.GetFileNameWithoutExtension(safeName);
|
|
}
|
|
}
|
|
|
|
public sealed class StoryIntelligenceClient(
|
|
HttpClient httpClient,
|
|
IOptions<StoryIntelligenceOptions> options,
|
|
ILogger<StoryIntelligenceClient> logger) : IStoryIntelligenceClient
|
|
{
|
|
private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web);
|
|
private readonly StoryIntelligenceOptions settings = options.Value;
|
|
|
|
public StoryIntelligenceClientConfigurationStatus GetConfigurationStatus()
|
|
=> new()
|
|
{
|
|
ApiKeyConfigured = !string.IsNullOrWhiteSpace(settings.ApiKey),
|
|
Model = settings.EffectiveModel,
|
|
ChapterStructureModel = settings.EffectiveChapterStructureModel,
|
|
SceneIntelligenceModel = settings.EffectiveSceneIntelligenceModel,
|
|
MaxOutputTokens = settings.MaxOutputTokens,
|
|
TimeoutSeconds = settings.TimeoutSeconds,
|
|
ClientConstructed = true,
|
|
ConnectionStatus = "Not checked on diagnostics page. Only the admin dry-run page makes a live OpenAI request."
|
|
};
|
|
|
|
public async Task<StoryIntelligenceClientResult> ExecutePromptAsync(
|
|
string completedPrompt,
|
|
string promptVersion,
|
|
CancellationToken cancellationToken,
|
|
string? modelOverride = null,
|
|
int? maxOutputTokensOverride = null)
|
|
{
|
|
if (string.IsNullOrWhiteSpace(settings.ApiKey))
|
|
{
|
|
throw new InvalidOperationException("OpenAI API key is not configured. Add OpenAI:ApiKey to User Secrets or environment variables.");
|
|
}
|
|
|
|
var model = string.IsNullOrWhiteSpace(modelOverride)
|
|
? settings.EffectiveModel
|
|
: modelOverride.Trim();
|
|
|
|
if (string.IsNullOrWhiteSpace(model))
|
|
{
|
|
throw new InvalidOperationException("OpenAI model is not configured. Set OpenAI:Model or StoryIntelligence stage models in configuration.");
|
|
}
|
|
|
|
if (string.IsNullOrWhiteSpace(completedPrompt))
|
|
{
|
|
throw new InvalidOperationException("Completed prompt is empty.");
|
|
}
|
|
|
|
var maxOutputTokens = maxOutputTokensOverride.GetValueOrDefault(settings.MaxOutputTokens);
|
|
if (maxOutputTokens <= 0)
|
|
{
|
|
maxOutputTokens = settings.MaxOutputTokens;
|
|
}
|
|
|
|
var requestBody = JsonSerializer.Serialize(new
|
|
{
|
|
model = model,
|
|
input = completedPrompt,
|
|
max_output_tokens = maxOutputTokens,
|
|
store = false,
|
|
reasoning = new
|
|
{
|
|
effort = "minimal"
|
|
},
|
|
text = new
|
|
{
|
|
verbosity = "low"
|
|
}
|
|
}, JsonOptions);
|
|
|
|
var stopwatch = Stopwatch.StartNew();
|
|
try
|
|
{
|
|
var (responseText, retryCount) = await SendWithRetryAsync(requestBody, cancellationToken);
|
|
stopwatch.Stop();
|
|
logger.LogInformation(
|
|
"Story Intelligence prompt executed. PromptVersion={PromptVersion} Model={Model} DurationMs={DurationMs} RetryCount={RetryCount} InputTokens={InputTokens} OutputTokens={OutputTokens} EstimatedCost={EstimatedCost} Success={Success}",
|
|
promptVersion,
|
|
model,
|
|
stopwatch.ElapsedMilliseconds,
|
|
retryCount,
|
|
null,
|
|
null,
|
|
null,
|
|
true);
|
|
|
|
return new StoryIntelligenceClientResult
|
|
{
|
|
RawResponseText = responseText,
|
|
Model = model,
|
|
Duration = stopwatch.Elapsed,
|
|
RetryCount = retryCount,
|
|
InputTokens = GetTokenUsage(responseText).InputTokens,
|
|
OutputTokens = GetTokenUsage(responseText).OutputTokens
|
|
};
|
|
}
|
|
catch (Exception ex) when (ex is not OperationCanceledException)
|
|
{
|
|
stopwatch.Stop();
|
|
logger.LogError(
|
|
ex,
|
|
"Story Intelligence prompt failed. PromptVersion={PromptVersion} Model={Model} DurationMs={DurationMs} RetryCount={RetryCount} InputTokens={InputTokens} OutputTokens={OutputTokens} EstimatedCost={EstimatedCost} Success={Success}",
|
|
promptVersion,
|
|
model,
|
|
stopwatch.ElapsedMilliseconds,
|
|
null,
|
|
null,
|
|
null,
|
|
null,
|
|
false);
|
|
throw;
|
|
}
|
|
}
|
|
|
|
private async Task<(string ResponseText, int RetryCount)> SendWithRetryAsync(string requestBody, CancellationToken cancellationToken)
|
|
{
|
|
const int maxAttempts = 3;
|
|
var retryCount = 0;
|
|
for (var attempt = 1; attempt <= maxAttempts; attempt++)
|
|
{
|
|
using var request = new HttpRequestMessage(HttpMethod.Post, "https://api.openai.com/v1/responses")
|
|
{
|
|
Content = new StringContent(requestBody, Encoding.UTF8, "application/json")
|
|
};
|
|
request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", settings.ApiKey);
|
|
|
|
try
|
|
{
|
|
using var timeout = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
|
|
timeout.CancelAfter(TimeSpan.FromSeconds(Math.Max(1, settings.TimeoutSeconds)));
|
|
using var response = await httpClient.SendAsync(request, timeout.Token);
|
|
var responseText = await response.Content.ReadAsStringAsync(cancellationToken);
|
|
|
|
if (response.IsSuccessStatusCode)
|
|
{
|
|
return (responseText, retryCount);
|
|
}
|
|
|
|
if (IsInsufficientQuota(responseText))
|
|
{
|
|
throw new InvalidOperationException($"OpenAI billing quota exhausted; request was not retried: {TrimForLog(responseText)}");
|
|
}
|
|
|
|
if (!ShouldRetry(response.StatusCode) || attempt == maxAttempts)
|
|
{
|
|
throw new InvalidOperationException($"OpenAI request failed with {(int)response.StatusCode} {response.StatusCode}: {TrimForLog(responseText)}");
|
|
}
|
|
|
|
retryCount++;
|
|
}
|
|
catch (TaskCanceledException) when (!cancellationToken.IsCancellationRequested)
|
|
{
|
|
if (attempt == maxAttempts)
|
|
{
|
|
throw new InvalidOperationException($"OpenAI request timed out after {settings.TimeoutSeconds} seconds.");
|
|
}
|
|
|
|
retryCount++;
|
|
await DelayForRetryAsync(attempt, cancellationToken);
|
|
continue;
|
|
}
|
|
catch (HttpRequestException) when (attempt < maxAttempts)
|
|
{
|
|
retryCount++;
|
|
await DelayForRetryAsync(attempt, cancellationToken);
|
|
continue;
|
|
}
|
|
|
|
await DelayForRetryAsync(attempt, cancellationToken);
|
|
}
|
|
|
|
throw new InvalidOperationException("OpenAI request failed after retry attempts.");
|
|
}
|
|
|
|
private static bool ShouldRetry(HttpStatusCode statusCode)
|
|
=> statusCode is HttpStatusCode.RequestTimeout
|
|
or HttpStatusCode.TooManyRequests
|
|
or HttpStatusCode.InternalServerError
|
|
or HttpStatusCode.BadGateway
|
|
or HttpStatusCode.ServiceUnavailable
|
|
or HttpStatusCode.GatewayTimeout;
|
|
|
|
internal static bool IsInsufficientQuota(string responseText)
|
|
=> responseText.Contains("\"insufficient_quota\"", StringComparison.OrdinalIgnoreCase)
|
|
|| responseText.Contains("exceeded your current quota", StringComparison.OrdinalIgnoreCase);
|
|
|
|
private static Task DelayForRetryAsync(int attempt, CancellationToken cancellationToken)
|
|
=> Task.Delay(TimeSpan.FromSeconds(Math.Pow(2, attempt - 1)), cancellationToken);
|
|
|
|
private static string TrimForLog(string value)
|
|
=> string.IsNullOrWhiteSpace(value) || value.Length <= 500 ? value : value[..500];
|
|
|
|
private static (int? InputTokens, int? OutputTokens) GetTokenUsage(string responseText)
|
|
{
|
|
try
|
|
{
|
|
var response = JsonSerializer.Deserialize<OpenAIResponseEnvelope>(responseText, JsonOptions);
|
|
return (response?.Usage?.InputTokens, response?.Usage?.OutputTokens);
|
|
}
|
|
catch (JsonException)
|
|
{
|
|
return (null, null);
|
|
}
|
|
}
|
|
}
|
|
|
|
public sealed class StoryIntelligenceDiagnosticsService(
|
|
IStoryPromptRepository prompts,
|
|
IStoryPromptVersionService versions,
|
|
IStoryPromptBuilder builder,
|
|
IStoryIntelligenceClient client) : IStoryIntelligenceDiagnosticsService
|
|
{
|
|
private const string ScenePromptFile = "Scene-Prompt-V2.md";
|
|
|
|
public async Task<StoryIntelligenceDiagnosticsViewModel> GetDiagnosticsAsync(CancellationToken cancellationToken)
|
|
{
|
|
var clientStatus = client.GetConfigurationStatus();
|
|
var model = new StoryIntelligenceDiagnosticsViewModel
|
|
{
|
|
Model = clientStatus.Model,
|
|
ChapterStructureModel = clientStatus.ChapterStructureModel,
|
|
SceneIntelligenceModel = clientStatus.SceneIntelligenceModel,
|
|
ApiKeyConfigured = clientStatus.ApiKeyConfigured,
|
|
ClientConstructed = clientStatus.ClientConstructed,
|
|
ConnectionStatus = clientStatus.ConnectionStatus,
|
|
PromptFileName = ScenePromptFile,
|
|
PromptVersion = versions.GetPromptVersion(ScenePromptFile),
|
|
MaxOutputTokens = clientStatus.MaxOutputTokens,
|
|
TimeoutSeconds = clientStatus.TimeoutSeconds
|
|
};
|
|
|
|
try
|
|
{
|
|
var prompt = await prompts.LoadPromptAsync(ScenePromptFile, cancellationToken);
|
|
model.PromptRepositoryHealthy = true;
|
|
model.ScenePromptLoaded = true;
|
|
model.ScenePromptLength = prompt.Length;
|
|
var completed = builder.BuildPrompt(prompt, "{\"sceneId\":0}", "Diagnostic scene text.");
|
|
model.RuntimeSubstitutionWorks = completed.Contains("\"sceneId\":0", StringComparison.Ordinal)
|
|
&& completed.Contains("Diagnostic scene text.", StringComparison.Ordinal)
|
|
&& !completed.Contains("{{SCENE_CONTEXT_JSON}}", StringComparison.Ordinal)
|
|
&& !completed.Contains("{{SCENE_TEXT}}", StringComparison.Ordinal);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
model.PromptRepositoryHealthy = false;
|
|
model.ScenePromptLoaded = false;
|
|
model.ErrorMessage = ex.Message;
|
|
}
|
|
|
|
return model;
|
|
}
|
|
}
|
|
|
|
public sealed class StoryIntelligenceDryRunService(
|
|
IStoryPromptRepository prompts,
|
|
IStoryPromptVersionService versions,
|
|
IStoryPromptBuilder builder,
|
|
IStoryIntelligenceClient client,
|
|
IStorySceneValidator validator,
|
|
ILogger<StoryIntelligenceDryRunService> logger) : IStoryIntelligenceDryRunService
|
|
{
|
|
private const string ScenePromptFile = "Scene-Prompt-V2.md";
|
|
private static readonly JsonSerializerOptions JsonOptions = new()
|
|
{
|
|
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
|
|
PropertyNameCaseInsensitive = true,
|
|
WriteIndented = true
|
|
};
|
|
|
|
public StoryIntelligenceDryRunViewModel GetDefault()
|
|
=> new();
|
|
|
|
public async Task<StoryIntelligenceDryRunViewModel> ExecuteAsync(StoryIntelligenceDryRunForm form, CancellationToken cancellationToken)
|
|
{
|
|
var model = new StoryIntelligenceDryRunViewModel { Form = form };
|
|
var promptVersion = versions.GetPromptVersion(ScenePromptFile);
|
|
var clientStatus = client.GetConfigurationStatus();
|
|
var contextJson = BuildSceneContextJson(form);
|
|
|
|
try
|
|
{
|
|
if (string.IsNullOrWhiteSpace(form.SceneText))
|
|
{
|
|
throw new InvalidOperationException("Enter short fictional scene text before running the dry run.");
|
|
}
|
|
|
|
var template = await prompts.LoadPromptAsync(ScenePromptFile, cancellationToken);
|
|
var completedPrompt = builder.BuildPrompt(template, contextJson, form.SceneText);
|
|
var result = await client.ExecutePromptAsync(
|
|
completedPrompt,
|
|
promptVersion,
|
|
cancellationToken,
|
|
clientStatus.SceneIntelligenceModel);
|
|
var sceneJson = ExtractSceneJson(result.RawResponseText);
|
|
var parsedScene = DeserializeScene(sceneJson);
|
|
var validation = validator.Validate(parsedScene);
|
|
|
|
model.Result = new StoryIntelligenceDryRunResultViewModel
|
|
{
|
|
Success = true,
|
|
PromptVersion = promptVersion,
|
|
Model = result.Model,
|
|
SceneContextJson = contextJson,
|
|
Duration = FormatDuration(result.Duration),
|
|
RetryCount = result.RetryCount,
|
|
InputTokens = result.InputTokens,
|
|
OutputTokens = result.OutputTokens,
|
|
RawResponseText = result.RawResponseText,
|
|
SceneJsonText = sceneJson,
|
|
ParsedScene = parsedScene,
|
|
Validation = validation
|
|
};
|
|
|
|
logger.LogInformation(
|
|
"Admin Story Intelligence dry run completed. PromptVersion={PromptVersion} Model={Model} DurationMs={DurationMs} RetryCount={RetryCount} InputTokens={InputTokens} OutputTokens={OutputTokens} ValidationPassed={ValidationPassed} ValidationWarnings={ValidationWarnings} ValidationErrors={ValidationErrors} Success={Success}",
|
|
promptVersion,
|
|
result.Model,
|
|
result.Duration.TotalMilliseconds,
|
|
result.RetryCount,
|
|
result.InputTokens,
|
|
result.OutputTokens,
|
|
validation.IsValid,
|
|
validation.Warnings.Count,
|
|
validation.Errors.Count,
|
|
true);
|
|
}
|
|
catch (JsonException ex)
|
|
{
|
|
model.Result = new StoryIntelligenceDryRunResultViewModel
|
|
{
|
|
Success = false,
|
|
PromptVersion = promptVersion,
|
|
Model = clientStatus.SceneIntelligenceModel,
|
|
SceneContextJson = contextJson,
|
|
ErrorMessage = $"Scene Intelligence JSON could not be deserialised: {ex.Message}",
|
|
Validation = validator.Validate(null)
|
|
};
|
|
|
|
logger.LogWarning(
|
|
ex,
|
|
"Admin Story Intelligence dry run deserialisation failed. PromptVersion={PromptVersion} Model={Model} Success={Success}",
|
|
promptVersion,
|
|
clientStatus.SceneIntelligenceModel,
|
|
false);
|
|
}
|
|
catch (Exception ex) when (ex is not OperationCanceledException)
|
|
{
|
|
model.Result = new StoryIntelligenceDryRunResultViewModel
|
|
{
|
|
Success = false,
|
|
PromptVersion = promptVersion,
|
|
Model = clientStatus.SceneIntelligenceModel,
|
|
SceneContextJson = contextJson,
|
|
ErrorMessage = ex.Message
|
|
};
|
|
|
|
logger.LogWarning(
|
|
ex,
|
|
"Admin Story Intelligence dry run failed. PromptVersion={PromptVersion} Model={Model} Success={Success}",
|
|
promptVersion,
|
|
clientStatus.SceneIntelligenceModel,
|
|
false);
|
|
}
|
|
|
|
return model;
|
|
}
|
|
|
|
private static string BuildSceneContextJson(StoryIntelligenceDryRunForm form)
|
|
=> JsonSerializer.Serialize(new
|
|
{
|
|
projectId = form.ProjectID,
|
|
bookId = form.BookID,
|
|
chapterId = form.ChapterID,
|
|
sceneId = form.SceneID,
|
|
chapterNumber = form.ChapterNumber,
|
|
sceneNumber = form.SceneNumber,
|
|
sourceLabel = string.IsNullOrWhiteSpace(form.SourceLabel) ? "Admin test scene" : form.SourceLabel.Trim()
|
|
}, JsonOptions);
|
|
|
|
private static string FormatDuration(TimeSpan duration)
|
|
=> duration.TotalSeconds < 1
|
|
? $"{duration.TotalMilliseconds:0} ms"
|
|
: $"{duration.TotalSeconds:0.00} sec";
|
|
|
|
private static string ExtractSceneJson(string rawResponseText)
|
|
{
|
|
var response = JsonSerializer.Deserialize<OpenAIResponseEnvelope>(rawResponseText, JsonOptions)
|
|
?? throw new JsonException("OpenAI response envelope was empty.");
|
|
var outputText = response.Output?
|
|
.SelectMany(item => item.Content ?? [])
|
|
.FirstOrDefault(content => string.Equals(content.Type, "output_text", StringComparison.OrdinalIgnoreCase)
|
|
&& !string.IsNullOrWhiteSpace(content.Text))
|
|
?.Text;
|
|
|
|
if (string.IsNullOrWhiteSpace(outputText))
|
|
{
|
|
throw new JsonException("OpenAI response did not contain output_text content.");
|
|
}
|
|
|
|
return outputText.Trim();
|
|
}
|
|
|
|
private static SceneIntelligenceScene? DeserializeScene(string sceneJson)
|
|
=> JsonSerializer.Deserialize<SceneIntelligenceScene>(sceneJson, JsonOptions);
|
|
}
|
|
|
|
public sealed class ChapterStructureDryRunService(
|
|
IStoryPromptRepository prompts,
|
|
IStoryPromptVersionService versions,
|
|
IStoryIntelligenceClient client,
|
|
IChapterStructureValidator validator,
|
|
ILogger<ChapterStructureDryRunService> logger) : IChapterStructureDryRunService
|
|
{
|
|
private const string ChapterPromptFile = "Chapter-Structure-Prompt-V2.md";
|
|
private static readonly JsonSerializerOptions JsonOptions = new()
|
|
{
|
|
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
|
|
PropertyNameCaseInsensitive = true,
|
|
WriteIndented = true
|
|
};
|
|
|
|
public ChapterStructureDryRunViewModel GetDefault()
|
|
=> new();
|
|
|
|
public async Task<ChapterStructureDryRunViewModel> ExecuteAsync(ChapterStructureDryRunForm form, CancellationToken cancellationToken)
|
|
{
|
|
var model = new ChapterStructureDryRunViewModel { Form = form };
|
|
var promptVersion = versions.GetPromptVersion(ChapterPromptFile);
|
|
var clientStatus = client.GetConfigurationStatus();
|
|
var contextJson = BuildChapterContextJson(form);
|
|
var paragraphs = StoryIntelligenceParagraphs.Split(form.ChapterText);
|
|
var paragraphCount = paragraphs.Count;
|
|
|
|
try
|
|
{
|
|
if (string.IsNullOrWhiteSpace(form.ChapterText))
|
|
{
|
|
throw new InvalidOperationException("Enter fictional chapter text before running the chapter structure test.");
|
|
}
|
|
|
|
var template = await prompts.LoadPromptAsync(ChapterPromptFile, cancellationToken);
|
|
var numberedChapterText = StoryIntelligenceParagraphs.Number(paragraphs);
|
|
var completedPrompt = BuildChapterPrompt(template, contextJson, numberedChapterText);
|
|
var result = await client.ExecutePromptAsync(
|
|
completedPrompt,
|
|
promptVersion,
|
|
cancellationToken,
|
|
clientStatus.ChapterStructureModel);
|
|
var chapterJson = ExtractOutputText(result.RawResponseText);
|
|
var parsedChapter = DeserializeChapter(chapterJson);
|
|
var chapterNormalisation = ChapterStructureBoundaryNormaliser.Normalise(parsedChapter, paragraphCount);
|
|
var chapterForValidation = chapterNormalisation.ChapterStructure;
|
|
var validation = validator.Validate(chapterForValidation, paragraphCount);
|
|
ChapterStructureBoundaryNormaliser.AddIssuesTo(validation, chapterNormalisation);
|
|
|
|
model.Result = new ChapterStructureDryRunResultViewModel
|
|
{
|
|
Success = true,
|
|
PromptVersion = promptVersion,
|
|
Model = result.Model,
|
|
ChapterContextJson = contextJson,
|
|
ParagraphCount = paragraphCount,
|
|
Duration = FormatDuration(result.Duration),
|
|
RetryCount = result.RetryCount,
|
|
InputTokens = result.InputTokens,
|
|
OutputTokens = result.OutputTokens,
|
|
RawResponseText = result.RawResponseText,
|
|
ChapterJsonText = chapterJson,
|
|
ParsedChapter = chapterForValidation,
|
|
Validation = validation
|
|
};
|
|
|
|
logger.LogInformation(
|
|
"Admin Chapter Structure dry run completed. PromptVersion={PromptVersion} Model={Model} DurationMs={DurationMs} RetryCount={RetryCount} InputTokens={InputTokens} OutputTokens={OutputTokens} ValidationPassed={ValidationPassed} ValidationWarnings={ValidationWarnings} ValidationErrors={ValidationErrors} Success={Success}",
|
|
promptVersion,
|
|
result.Model,
|
|
result.Duration.TotalMilliseconds,
|
|
result.RetryCount,
|
|
result.InputTokens,
|
|
result.OutputTokens,
|
|
validation.IsValid,
|
|
validation.Warnings.Count,
|
|
validation.Errors.Count,
|
|
true);
|
|
}
|
|
catch (JsonException ex)
|
|
{
|
|
model.Result = new ChapterStructureDryRunResultViewModel
|
|
{
|
|
Success = false,
|
|
PromptVersion = promptVersion,
|
|
Model = clientStatus.ChapterStructureModel,
|
|
ChapterContextJson = contextJson,
|
|
ParagraphCount = paragraphCount,
|
|
ErrorMessage = $"Chapter Structure JSON could not be deserialised: {ex.Message}",
|
|
Validation = validator.Validate(null, paragraphCount)
|
|
};
|
|
|
|
logger.LogWarning(
|
|
ex,
|
|
"Admin Chapter Structure dry run deserialisation failed. PromptVersion={PromptVersion} Model={Model} Success={Success}",
|
|
promptVersion,
|
|
clientStatus.ChapterStructureModel,
|
|
false);
|
|
}
|
|
catch (Exception ex) when (ex is not OperationCanceledException)
|
|
{
|
|
model.Result = new ChapterStructureDryRunResultViewModel
|
|
{
|
|
Success = false,
|
|
PromptVersion = promptVersion,
|
|
Model = clientStatus.ChapterStructureModel,
|
|
ChapterContextJson = contextJson,
|
|
ParagraphCount = paragraphCount,
|
|
ErrorMessage = ex.Message
|
|
};
|
|
|
|
logger.LogWarning(
|
|
ex,
|
|
"Admin Chapter Structure dry run failed. PromptVersion={PromptVersion} Model={Model} Success={Success}",
|
|
promptVersion,
|
|
clientStatus.ChapterStructureModel,
|
|
false);
|
|
}
|
|
|
|
return model;
|
|
}
|
|
|
|
private static string BuildChapterContextJson(ChapterStructureDryRunForm form)
|
|
=> JsonSerializer.Serialize(new
|
|
{
|
|
projectId = form.ProjectID,
|
|
bookId = form.BookID,
|
|
chapterId = form.ChapterID,
|
|
chapterNumber = form.ChapterNumber,
|
|
sourceLabel = string.IsNullOrWhiteSpace(form.SourceLabel) ? "Admin test chapter" : form.SourceLabel.Trim()
|
|
}, JsonOptions);
|
|
|
|
private static string BuildChapterPrompt(string promptTemplate, string chapterContextJson, string chapterText)
|
|
{
|
|
if (string.IsNullOrWhiteSpace(promptTemplate))
|
|
{
|
|
throw new InvalidOperationException("Prompt template is empty.");
|
|
}
|
|
|
|
return promptTemplate
|
|
.Replace("{{CHAPTER_CONTEXT_JSON}}", chapterContextJson ?? string.Empty, StringComparison.Ordinal)
|
|
.Replace("{{CHAPTER_TEXT}}", chapterText ?? string.Empty, StringComparison.Ordinal);
|
|
}
|
|
|
|
private static string ExtractOutputText(string rawResponseText)
|
|
{
|
|
var response = JsonSerializer.Deserialize<OpenAIResponseEnvelope>(rawResponseText, JsonOptions)
|
|
?? throw new JsonException("OpenAI response envelope was empty.");
|
|
var outputText = response.Output?
|
|
.SelectMany(item => item.Content ?? [])
|
|
.FirstOrDefault(content => string.Equals(content.Type, "output_text", StringComparison.OrdinalIgnoreCase)
|
|
&& !string.IsNullOrWhiteSpace(content.Text))
|
|
?.Text;
|
|
|
|
if (string.IsNullOrWhiteSpace(outputText))
|
|
{
|
|
throw new JsonException("OpenAI response did not contain output_text content.");
|
|
}
|
|
|
|
return outputText.Trim();
|
|
}
|
|
|
|
private static ChapterStructureModel? DeserializeChapter(string chapterJson)
|
|
=> JsonSerializer.Deserialize<ChapterStructureModel>(chapterJson, JsonOptions);
|
|
|
|
private static string FormatDuration(TimeSpan duration)
|
|
=> duration.TotalSeconds < 1
|
|
? $"{duration.TotalMilliseconds:0} ms"
|
|
: $"{duration.TotalSeconds:0.00} sec";
|
|
}
|
|
|
|
internal sealed class OpenAIResponseEnvelope
|
|
{
|
|
public List<OpenAIResponseOutputItem>? Output { get; init; }
|
|
public OpenAIResponseUsage? Usage { get; init; }
|
|
}
|
|
|
|
internal sealed class OpenAIResponseOutputItem
|
|
{
|
|
public List<OpenAIResponseContentItem>? Content { get; init; }
|
|
}
|
|
|
|
internal sealed class OpenAIResponseContentItem
|
|
{
|
|
public string? Type { get; init; }
|
|
public string? Text { get; init; }
|
|
}
|
|
|
|
internal sealed class OpenAIResponseUsage
|
|
{
|
|
[JsonPropertyName("input_tokens")]
|
|
public int? InputTokens { get; init; }
|
|
|
|
[JsonPropertyName("output_tokens")]
|
|
public int? OutputTokens { get; init; }
|
|
}
|
|
|
|
public sealed class StoryIntelligenceClientConfigurationStatus
|
|
{
|
|
public bool ApiKeyConfigured { get; init; }
|
|
public string Model { get; init; } = string.Empty;
|
|
public string ChapterStructureModel { get; init; } = string.Empty;
|
|
public string SceneIntelligenceModel { get; init; } = string.Empty;
|
|
public int MaxOutputTokens { get; init; }
|
|
public int TimeoutSeconds { get; init; }
|
|
public bool ClientConstructed { get; init; }
|
|
public string ConnectionStatus { get; init; } = string.Empty;
|
|
}
|