Chunk Word Companion manuscript scans
This commit is contained in:
parent
d2449a4683
commit
d32de17dcc
@ -1,8 +1,10 @@
|
||||
using System.Text.Json;
|
||||
using System.Net;
|
||||
using System.Text;
|
||||
using System.IO.Compression;
|
||||
using Microsoft.AspNetCore.Http.Metadata;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using PlotLine.Controllers;
|
||||
using PlotLine.Data;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
@ -150,6 +152,9 @@ var tests = new (string Name, Action Test)[]
|
||||
("Character discovery rejects sentence-start words", CharacterDiscoveryRejectsSentenceStartWords),
|
||||
("Character discovery rejects fragments and location names", CharacterDiscoveryRejectsFragmentsAndLocationNames),
|
||||
("Manuscript scan preview keeps only auto-selected character candidates", ManuscriptScanPreviewKeepsOnlyAutoSelectedCharacterCandidates),
|
||||
("Word Companion scan completion uses chunked transport", WordCompanionScanCompletionUsesChunkedTransport),
|
||||
("Word Companion scan chunks assemble full novel payloads", WordCompanionScanChunksAssembleFullNovelPayloads),
|
||||
("Word Companion scan chunks reject partial and conflicting payloads", WordCompanionScanChunksRejectPartialAndConflictingPayloads),
|
||||
("Onboarding wizard keeps character pages inside one main step", OnboardingWizardKeepsCharacterPagesInsideOneMainStep),
|
||||
("Scan review aliases feed Story Intelligence known character context", ScanReviewAliasesFeedStoryIntelligenceKnownCharacterContext),
|
||||
("Name Intelligence package contract is present", NameIntelligencePackageContractIsPresent),
|
||||
@ -3624,6 +3629,108 @@ static void ManuscriptScanPreviewKeepsOnlyAutoSelectedCharacterCandidates()
|
||||
Assert(candidates[0].Name == "Colin", "Scan preview should keep probable character candidates.");
|
||||
}
|
||||
|
||||
static void WordCompanionScanCompletionUsesChunkedTransport()
|
||||
{
|
||||
var root = Path.Combine(AppContext.BaseDirectory, "../../../../PlotLine");
|
||||
var hostScript = File.ReadAllText(Path.Combine(root, "wwwroot/js/word-companion-host.js"));
|
||||
var hub = File.ReadAllText(Path.Combine(root, "Hubs/WordCompanionFollowHub.cs"));
|
||||
var models = File.ReadAllText(Path.Combine(root, "Models/ManuscriptScanModels.cs"));
|
||||
var store = File.ReadAllText(Path.Combine(root, "Services/ManuscriptScanPreviewStore.cs"));
|
||||
var monitor = File.ReadAllText(Path.Combine(root, "Services/WordCompanionPresenceMonitor.cs"));
|
||||
var presence = File.ReadAllText(Path.Combine(root, "Services/WordCompanionPresenceService.cs"));
|
||||
var program = File.ReadAllText(Path.Combine(root, "Program.cs"));
|
||||
|
||||
Assert(hostScript.Contains("const onboardingScanChunkSize = 128 * 1024", StringComparison.Ordinal), "Word Companion scan result chunks should remain well below the SignalR receive limit.");
|
||||
Assert(hostScript.Contains("completeOnboardingScanInChunks(command, preview)", StringComparison.Ordinal), "Word Companion should upload scan completion through chunked transport.");
|
||||
Assert(!hostScript.Contains("invoke(\"CompleteOnboardingScan\", preview)", StringComparison.Ordinal), "Word Companion should not send full manuscript previews as one SignalR message.");
|
||||
Assert(hub.Contains("ReceiveOnboardingScanChunk", StringComparison.Ordinal), "Follow hub should expose a chunk receive method.");
|
||||
Assert(models.Contains("public sealed class ManuscriptScanChunk", StringComparison.Ordinal), "Scan chunk model should be explicit.");
|
||||
Assert(models.Contains("public Guid? ScanID", StringComparison.Ordinal), "Scan state should expose a scan ID for correlation.");
|
||||
Assert(store.Contains("A duplicate scan result chunk did not match the original payload.", StringComparison.Ordinal), "Chunk store should reject conflicting duplicate chunks.");
|
||||
Assert(store.Contains("HasRecentRunningScanForUserAsync", StringComparison.Ordinal), "Scan store should expose active scan state for connection semantics.");
|
||||
Assert(monitor.Contains("ActiveScanHeartbeatGrace", StringComparison.Ordinal), "Presence monitor should treat long-running scans as a distinct active state.");
|
||||
Assert(monitor.Contains("scanStore.HasRecentRunningScanForUserAsync", StringComparison.Ordinal), "Presence monitor should not mark an active scanning companion offline just because normal heartbeat is quiet.");
|
||||
Assert(presence.Contains("Func<int, Task<bool>>? keepConnected", StringComparison.Ordinal), "Presence stale checks should support scan-aware keep-connected semantics.");
|
||||
Assert(program.Contains("options.MaximumReceiveMessageSize = 1024 * 1024", StringComparison.Ordinal), "The fix should not depend on globally raising the SignalR receive limit.");
|
||||
}
|
||||
|
||||
static void WordCompanionScanChunksAssembleFullNovelPayloads()
|
||||
{
|
||||
var preview = BuildSyntheticScanPreview(chapters: 42, paragraphsPerChapter: 58, wordsPerParagraph: 82, includeUnicode: true);
|
||||
var json = JsonSerializer.Serialize(preview, ManuscriptScanJsonOptions());
|
||||
var compressedSize = GZip(Encoding.UTF8.GetBytes(json)).Length;
|
||||
Assert(preview.TotalWordCount >= 200_000, $"Synthetic full-novel fixture should exceed 200k words; got {preview.TotalWordCount}.");
|
||||
Assert(json.Length > 1024 * 1024, $"Synthetic full-novel scan JSON should exceed the old 1 MB SignalR cap; got {json.Length} chars.");
|
||||
Assert(compressedSize > 0 && compressedSize < Encoding.UTF8.GetByteCount(json), "Compressed size measurement should be valid and smaller than raw JSON.");
|
||||
|
||||
var store = BuildScanStore();
|
||||
var state = store.StartAsync(preview.UserID, preview.OnboardingID, preview.ProjectID, preview.BookID).GetAwaiter().GetResult();
|
||||
var chunks = SplitScanChunks(state.ScanID!.Value, preview.OnboardingID, json, 128 * 1024);
|
||||
foreach (var chunk in chunks)
|
||||
{
|
||||
state = store.AddChunkAsync(preview.UserID, chunk).GetAwaiter().GetResult();
|
||||
}
|
||||
|
||||
Assert(state.Status == ManuscriptScanStatuses.Complete, "Final chunk should complete the full-novel scan.");
|
||||
Assert(state.TotalWordCount == preview.TotalWordCount, "Full-novel scan should preserve word count.");
|
||||
Assert(state.ChapterCount == preview.ChapterCount, "Full-novel scan should preserve chapter count.");
|
||||
var stored = store.GetLatestPreviewAsync(preview.UserID, preview.OnboardingID).GetAwaiter().GetResult();
|
||||
Assert(stored is not null, "Completed chunked scan should persist a preview.");
|
||||
Assert(stored!.Chapters.Count == preview.Chapters.Count, "Chunked scan should preserve all chapters.");
|
||||
Assert(stored.Chapters.Sum(chapter => chapter.SourceParagraphs.Count) == preview.Chapters.Sum(chapter => chapter.SourceParagraphs.Count), "Chunked scan should preserve all source paragraphs.");
|
||||
Assert(stored.Chapters.SelectMany(chapter => chapter.SourceParagraphs).Any(paragraph => paragraph.Text.Contains("Rosie's", StringComparison.Ordinal)), "Chunked scan should preserve Unicode punctuation.");
|
||||
|
||||
var longParagraphPreview = BuildSyntheticScanPreview(chapters: 1, paragraphsPerChapter: 1, wordsPerParagraph: 200_000, includeUnicode: true);
|
||||
var longParagraphJson = JsonSerializer.Serialize(longParagraphPreview, ManuscriptScanJsonOptions());
|
||||
var longParagraphStore = BuildScanStore();
|
||||
var longParagraphState = longParagraphStore.StartAsync(longParagraphPreview.UserID, longParagraphPreview.OnboardingID, longParagraphPreview.ProjectID, longParagraphPreview.BookID).GetAwaiter().GetResult();
|
||||
foreach (var chunk in SplitScanChunks(longParagraphState.ScanID!.Value, longParagraphPreview.OnboardingID, longParagraphJson, 128 * 1024))
|
||||
{
|
||||
longParagraphState = longParagraphStore.AddChunkAsync(longParagraphPreview.UserID, chunk).GetAwaiter().GetResult();
|
||||
}
|
||||
|
||||
Assert(longParagraphState.Status == ManuscriptScanStatuses.Complete, "Chunked scan should support a large individual paragraph.");
|
||||
Assert(longParagraphState.TotalWordCount >= 200_000, "Large individual paragraph fixture should preserve full word count.");
|
||||
}
|
||||
|
||||
static void WordCompanionScanChunksRejectPartialAndConflictingPayloads()
|
||||
{
|
||||
var preview = BuildSyntheticScanPreview(chapters: 6, paragraphsPerChapter: 20, wordsPerParagraph: 70, includeUnicode: false);
|
||||
var json = JsonSerializer.Serialize(preview, ManuscriptScanJsonOptions());
|
||||
var store = BuildScanStore();
|
||||
var state = store.StartAsync(preview.UserID, preview.OnboardingID, preview.ProjectID, preview.BookID).GetAwaiter().GetResult();
|
||||
var chunks = SplitScanChunks(state.ScanID!.Value, preview.OnboardingID, json, 4096);
|
||||
Assert(chunks.Count > 2, "Six-chapter synthetic fixture should span multiple small test chunks.");
|
||||
|
||||
var partialState = store.AddChunkAsync(preview.UserID, chunks[0]).GetAwaiter().GetResult();
|
||||
Assert(partialState.Status == ManuscriptScanStatuses.Running, "A partial chunk set must not be accepted as complete.");
|
||||
var duplicateState = store.AddChunkAsync(preview.UserID, chunks[0]).GetAwaiter().GetResult();
|
||||
Assert(duplicateState.Status == ManuscriptScanStatuses.Running, "An exact duplicate chunk retry should be idempotent.");
|
||||
|
||||
var conflicting = new ManuscriptScanChunk
|
||||
{
|
||||
ScanID = chunks[0].ScanID,
|
||||
UserID = chunks[0].UserID,
|
||||
OnboardingID = chunks[0].OnboardingID,
|
||||
ChunkIndex = chunks[0].ChunkIndex,
|
||||
TotalChunks = chunks[0].TotalChunks,
|
||||
TotalLength = chunks[0].TotalLength,
|
||||
Payload = chunks[0].Payload + "x"
|
||||
};
|
||||
AssertThrows(() => store.AddChunkAsync(preview.UserID, conflicting).GetAwaiter().GetResult(), "Conflicting duplicate chunks should be rejected.");
|
||||
var staleScanChunk = new ManuscriptScanChunk
|
||||
{
|
||||
ScanID = Guid.NewGuid(),
|
||||
UserID = chunks[1].UserID,
|
||||
OnboardingID = chunks[1].OnboardingID,
|
||||
ChunkIndex = chunks[1].ChunkIndex,
|
||||
TotalChunks = chunks[1].TotalChunks,
|
||||
TotalLength = chunks[1].TotalLength,
|
||||
Payload = chunks[1].Payload
|
||||
};
|
||||
AssertThrows(() => store.AddChunkAsync(preview.UserID, staleScanChunk).GetAwaiter().GetResult(), "Chunks from a stale scan ID should be rejected.");
|
||||
}
|
||||
|
||||
static void CharacterDiscoveryRejectsFragmentsAndLocationNames()
|
||||
{
|
||||
var method = typeof(WordCompanionService).GetMethod("DiscoverCharacterCandidates", System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Static);
|
||||
@ -5909,6 +6016,20 @@ static void Assert(bool condition, string message)
|
||||
}
|
||||
}
|
||||
|
||||
static void AssertThrows(Action action, string message)
|
||||
{
|
||||
try
|
||||
{
|
||||
action();
|
||||
}
|
||||
catch
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
throw new InvalidOperationException(message);
|
||||
}
|
||||
|
||||
static string ExtractMethod(string source, string methodName)
|
||||
{
|
||||
var signature = $" {methodName}(";
|
||||
@ -5937,6 +6058,126 @@ static string ExtractMethod(string source, string methodName)
|
||||
throw new InvalidOperationException($"{methodName} body was not closed.");
|
||||
}
|
||||
|
||||
static ManuscriptScanPreview BuildSyntheticScanPreview(int chapters, int paragraphsPerChapter, int wordsPerParagraph, bool includeUnicode)
|
||||
{
|
||||
var chapterPreviews = new List<ManuscriptScanChapterPreview>();
|
||||
var paragraphIndex = 0;
|
||||
var totalWords = 0;
|
||||
for (var chapterIndex = 1; chapterIndex <= chapters; chapterIndex++)
|
||||
{
|
||||
var paragraphTexts = new List<string>();
|
||||
var paragraphMap = new List<ManuscriptScanParagraphMap>();
|
||||
var sourceParagraphs = new List<ManuscriptScanSourceParagraph>
|
||||
{
|
||||
new()
|
||||
{
|
||||
WordParagraphIndex = paragraphIndex,
|
||||
Text = $"Chapter {chapterIndex}",
|
||||
IsChapterHeading = true,
|
||||
IncludeInSourceText = false
|
||||
}
|
||||
};
|
||||
var chapterStart = paragraphIndex;
|
||||
paragraphIndex++;
|
||||
|
||||
for (var paragraphOrdinal = 1; paragraphOrdinal <= paragraphsPerChapter; paragraphOrdinal++)
|
||||
{
|
||||
var sentence = includeUnicode && paragraphOrdinal % 17 == 0
|
||||
? "Rosie's mum Mary said, \"Don't panic\" as the rain tapped against the window."
|
||||
: "Maggie watched the road while Beth remembered Grace and Rebecca counted the passing houses.";
|
||||
var text = string.Join(' ', Enumerable.Repeat(sentence, Math.Max(1, wordsPerParagraph / 13)));
|
||||
var words = text.Split(' ', StringSplitOptions.RemoveEmptyEntries).Length;
|
||||
totalWords += words;
|
||||
paragraphTexts.Add(text);
|
||||
paragraphMap.Add(new ManuscriptScanParagraphMap
|
||||
{
|
||||
StructuralParagraphOrdinal = paragraphOrdinal,
|
||||
WordParagraphIndex = paragraphIndex,
|
||||
Text = text
|
||||
});
|
||||
sourceParagraphs.Add(new ManuscriptScanSourceParagraph
|
||||
{
|
||||
WordParagraphIndex = paragraphIndex,
|
||||
Text = text,
|
||||
IncludeInSourceText = true
|
||||
});
|
||||
paragraphIndex++;
|
||||
}
|
||||
|
||||
chapterPreviews.Add(new ManuscriptScanChapterPreview
|
||||
{
|
||||
TemporaryChapterKey = $"chapter-{chapterIndex}",
|
||||
ChapterNumber = chapterIndex,
|
||||
Title = $"Chapter {chapterIndex}",
|
||||
WordCount = paragraphTexts.Sum(text => text.Split(' ', StringSplitOptions.RemoveEmptyEntries).Length),
|
||||
ChapterText = string.Join(Environment.NewLine + Environment.NewLine, paragraphTexts),
|
||||
StartPosition = chapterStart,
|
||||
ParagraphMap = paragraphMap,
|
||||
SourceParagraphs = sourceParagraphs
|
||||
});
|
||||
}
|
||||
|
||||
return new ManuscriptScanPreview
|
||||
{
|
||||
UserID = 1,
|
||||
OnboardingID = 100,
|
||||
ProjectID = 200,
|
||||
BookID = 300,
|
||||
Source = ManuscriptScanSources.WordCompanion,
|
||||
DocumentTitle = "Synthetic full novel",
|
||||
CompanionDocumentIdentifier = Guid.NewGuid().ToString("N"),
|
||||
TotalWordCount = totalWords,
|
||||
ChapterCount = chapterPreviews.Count,
|
||||
CreatedUtc = DateTime.UtcNow,
|
||||
Chapters = chapterPreviews
|
||||
};
|
||||
}
|
||||
|
||||
static List<ManuscriptScanChunk> SplitScanChunks(Guid scanId, int onboardingId, string payload, int chunkSize)
|
||||
{
|
||||
var chunks = new List<ManuscriptScanChunk>();
|
||||
var totalChunks = Math.Max(1, (int)Math.Ceiling(payload.Length / (double)chunkSize));
|
||||
for (var chunkIndex = 0; chunkIndex < totalChunks; chunkIndex++)
|
||||
{
|
||||
var start = chunkIndex * chunkSize;
|
||||
chunks.Add(new ManuscriptScanChunk
|
||||
{
|
||||
ScanID = scanId,
|
||||
UserID = 1,
|
||||
OnboardingID = onboardingId,
|
||||
ChunkIndex = chunkIndex,
|
||||
TotalChunks = totalChunks,
|
||||
TotalLength = payload.Length,
|
||||
Payload = payload.Substring(start, Math.Min(chunkSize, payload.Length - start))
|
||||
});
|
||||
}
|
||||
|
||||
return chunks;
|
||||
}
|
||||
|
||||
static ManuscriptScanPreviewStore BuildScanStore()
|
||||
{
|
||||
var services = new ServiceCollection().BuildServiceProvider();
|
||||
return new ManuscriptScanPreviewStore(services.GetRequiredService<IServiceScopeFactory>(), NullLogger<ManuscriptScanPreviewStore>.Instance);
|
||||
}
|
||||
|
||||
static byte[] GZip(byte[] payload)
|
||||
{
|
||||
using var output = new MemoryStream();
|
||||
using (var gzip = new GZipStream(output, CompressionLevel.Fastest))
|
||||
{
|
||||
gzip.Write(payload, 0, payload.Length);
|
||||
}
|
||||
|
||||
return output.ToArray();
|
||||
}
|
||||
|
||||
static JsonSerializerOptions ManuscriptScanJsonOptions()
|
||||
=> new()
|
||||
{
|
||||
PropertyNamingPolicy = JsonNamingPolicy.CamelCase
|
||||
};
|
||||
|
||||
static StoryIntelligenceClient BuildStoryIntelligenceClient(Func<HttpRequestMessage, HttpResponseMessage> respond)
|
||||
=> new(
|
||||
new HttpClient(new CaptureImageRequestHandler(respond)),
|
||||
|
||||
@ -154,8 +154,17 @@ public sealed class WordCompanionFollowHub(
|
||||
|
||||
var scanState = await scanStore.StartAsync(userId, state.UserOnboardingStateID, state.ProjectID.Value, state.BookID.Value);
|
||||
await Clients.Group(PresenceGroup(userId)).SendAsync("OnboardingScanStateChanged", scanState);
|
||||
logger.LogInformation(
|
||||
"Word Companion onboarding scan requested. UserID={UserID} OnboardingID={OnboardingID} ProjectID={ProjectID} BookID={BookID} ScanID={ScanID} CompanionConnectionID={CompanionConnectionID}",
|
||||
userId,
|
||||
state.UserOnboardingStateID,
|
||||
state.ProjectID.Value,
|
||||
state.BookID.Value,
|
||||
scanState.ScanID,
|
||||
companionConnectionId);
|
||||
await Clients.Client(companionConnectionId).SendAsync("ScanCurrentDocument", new ManuscriptScanCommand
|
||||
{
|
||||
ScanID = scanState.ScanID ?? Guid.NewGuid(),
|
||||
UserID = userId,
|
||||
OnboardingID = state.UserOnboardingStateID,
|
||||
ProjectID = state.ProjectID.Value,
|
||||
@ -165,6 +174,31 @@ public sealed class WordCompanionFollowHub(
|
||||
return scanState;
|
||||
}
|
||||
|
||||
public async Task<ManuscriptScanState> ReceiveOnboardingScanChunk(ManuscriptScanChunk chunk)
|
||||
{
|
||||
var userId = RequireUserId();
|
||||
var scanState = await scanStore.AddChunkAsync(userId, chunk);
|
||||
if (string.Equals(scanState.Status, ManuscriptScanStatuses.Complete, StringComparison.Ordinal))
|
||||
{
|
||||
logger.LogInformation(
|
||||
"Word Companion onboarding scan completed from chunked payload. UserID={UserID} OnboardingID={OnboardingID} ScanID={ScanID} Chunks={Chunks} PayloadCharacters={PayloadCharacters} Words={Words} Chapters={Chapters}",
|
||||
userId,
|
||||
chunk.OnboardingID,
|
||||
chunk.ScanID,
|
||||
chunk.TotalChunks,
|
||||
chunk.TotalLength,
|
||||
scanState.TotalWordCount,
|
||||
scanState.ChapterCount);
|
||||
await Clients.Group(PresenceGroup(userId)).SendAsync("OnboardingScanCompleted", scanState);
|
||||
}
|
||||
else
|
||||
{
|
||||
await Clients.Group(PresenceGroup(userId)).SendAsync("OnboardingScanProgress", scanState);
|
||||
}
|
||||
|
||||
return scanState;
|
||||
}
|
||||
|
||||
public async Task<ManuscriptScanState> ReportOnboardingScanProgress(ManuscriptScanProgress progress)
|
||||
{
|
||||
var userId = RequireUserId();
|
||||
|
||||
@ -24,6 +24,7 @@ public static class ManuscriptScanReviewStatuses
|
||||
|
||||
public sealed class ManuscriptScanCommand
|
||||
{
|
||||
public Guid ScanID { get; init; }
|
||||
public int UserID { get; init; }
|
||||
public int OnboardingID { get; init; }
|
||||
public int ProjectID { get; init; }
|
||||
@ -50,6 +51,17 @@ public sealed class ManuscriptScanFailure
|
||||
public string Message { get; init; } = string.Empty;
|
||||
}
|
||||
|
||||
public sealed class ManuscriptScanChunk
|
||||
{
|
||||
public Guid ScanID { get; init; }
|
||||
public int? UserID { get; init; }
|
||||
public int OnboardingID { get; init; }
|
||||
public int ChunkIndex { get; init; }
|
||||
public int TotalChunks { get; init; }
|
||||
public int TotalLength { get; init; }
|
||||
public string Payload { get; init; } = string.Empty;
|
||||
}
|
||||
|
||||
public sealed class ManuscriptScanPreview
|
||||
{
|
||||
public Guid PreviewID { get; init; } = Guid.NewGuid();
|
||||
@ -138,6 +150,7 @@ public sealed class ManuscriptScanCharacterEvidenceScene
|
||||
|
||||
public sealed class ManuscriptScanState
|
||||
{
|
||||
public Guid? ScanID { get; init; }
|
||||
public string Status { get; init; } = ManuscriptScanStatuses.NotStarted;
|
||||
public string ReviewStatus { get; init; } = ManuscriptScanReviewStatuses.ScanComplete;
|
||||
public string Message { get; init; } = string.Empty;
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
using System.Collections.Concurrent;
|
||||
using System.Text.Json;
|
||||
using PlotLine.Models;
|
||||
|
||||
namespace PlotLine.Services;
|
||||
@ -8,9 +9,11 @@ public interface IManuscriptScanPreviewStore
|
||||
Task<ManuscriptScanState> StartAsync(int userId, int onboardingId, int projectId, int bookId);
|
||||
Task ClearAsync(int userId, int onboardingId);
|
||||
Task<ManuscriptScanState> ProgressAsync(int userId, ManuscriptScanProgress progress);
|
||||
Task<ManuscriptScanState> AddChunkAsync(int userId, ManuscriptScanChunk chunk);
|
||||
Task<ManuscriptScanState> CompleteAsync(int userId, ManuscriptScanPreview preview);
|
||||
Task<ManuscriptScanState> FailAsync(int userId, int onboardingId, string message);
|
||||
Task<IReadOnlyList<ManuscriptScanState>> FailRunningForUserAsync(int userId, string message);
|
||||
Task<bool> HasRecentRunningScanForUserAsync(int userId, TimeSpan maxAge);
|
||||
Task<ManuscriptScanState> GetStateAsync(int userId, int onboardingId);
|
||||
Task<ManuscriptScanPreview?> GetLatestPreviewAsync(int userId, int onboardingId);
|
||||
Task<ManuscriptScanPreview?> GetPreviewAsync(int userId, Guid previewId);
|
||||
@ -36,6 +39,7 @@ public sealed class ManuscriptScanPreviewStore(
|
||||
{
|
||||
var session = new ScanSession
|
||||
{
|
||||
ScanID = Guid.NewGuid(),
|
||||
UserID = userId,
|
||||
OnboardingID = onboardingId,
|
||||
ProjectID = projectId,
|
||||
@ -65,6 +69,11 @@ public sealed class ManuscriptScanPreviewStore(
|
||||
Status = ManuscriptScanStatuses.Running
|
||||
});
|
||||
|
||||
if (string.Equals(session.Status, ManuscriptScanStatuses.Complete, StringComparison.Ordinal))
|
||||
{
|
||||
return Task.FromResult(ToState(session));
|
||||
}
|
||||
|
||||
session.Status = ManuscriptScanStatuses.Running;
|
||||
session.Message = Clean(progress.Message, "Scanning manuscript");
|
||||
session.PercentComplete = progress.PercentComplete;
|
||||
@ -76,6 +85,73 @@ public sealed class ManuscriptScanPreviewStore(
|
||||
return Task.FromResult(ToState(session));
|
||||
}
|
||||
|
||||
public Task<ManuscriptScanState> AddChunkAsync(int userId, ManuscriptScanChunk chunk)
|
||||
{
|
||||
if (chunk.ScanID == Guid.Empty
|
||||
|| chunk.OnboardingID <= 0
|
||||
|| chunk.ChunkIndex < 0
|
||||
|| chunk.TotalChunks <= 0
|
||||
|| chunk.ChunkIndex >= chunk.TotalChunks
|
||||
|| chunk.TotalLength <= 0
|
||||
|| string.IsNullOrEmpty(chunk.Payload))
|
||||
{
|
||||
throw new InvalidOperationException("The scan result chunk was invalid.");
|
||||
}
|
||||
|
||||
if (!sessions.TryGetValue((userId, chunk.OnboardingID), out var session)
|
||||
|| !string.Equals(session.Status, ManuscriptScanStatuses.Running, StringComparison.Ordinal)
|
||||
|| session.ScanID != chunk.ScanID)
|
||||
{
|
||||
throw new InvalidOperationException("The scan result no longer matches an active scan.");
|
||||
}
|
||||
|
||||
if (session.ExpectedChunkCount.HasValue && session.ExpectedChunkCount.Value != chunk.TotalChunks
|
||||
|| session.ExpectedPayloadLength.HasValue && session.ExpectedPayloadLength.Value != chunk.TotalLength)
|
||||
{
|
||||
throw new InvalidOperationException("The scan result chunk metadata changed during upload.");
|
||||
}
|
||||
|
||||
session.ExpectedChunkCount = chunk.TotalChunks;
|
||||
session.ExpectedPayloadLength = chunk.TotalLength;
|
||||
if (session.Chunks.TryGetValue(chunk.ChunkIndex, out var existingPayload))
|
||||
{
|
||||
if (!string.Equals(existingPayload, chunk.Payload, StringComparison.Ordinal))
|
||||
{
|
||||
throw new InvalidOperationException("A duplicate scan result chunk did not match the original payload.");
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
session.Chunks[chunk.ChunkIndex] = chunk.Payload;
|
||||
}
|
||||
|
||||
session.Message = $"Receiving manuscript scan result ({session.Chunks.Count:N0}/{chunk.TotalChunks:N0})";
|
||||
session.PercentComplete = Math.Min(99, 95 + (int)Math.Floor(session.Chunks.Count * 4m / chunk.TotalChunks));
|
||||
session.UpdatedUtc = DateTime.UtcNow;
|
||||
if (session.Chunks.Count < chunk.TotalChunks)
|
||||
{
|
||||
return Task.FromResult(ToState(session));
|
||||
}
|
||||
|
||||
for (var index = 0; index < chunk.TotalChunks; index++)
|
||||
{
|
||||
if (!session.Chunks.ContainsKey(index))
|
||||
{
|
||||
throw new InvalidOperationException("The scan result is missing one or more chunks.");
|
||||
}
|
||||
}
|
||||
|
||||
var payload = string.Concat(Enumerable.Range(0, chunk.TotalChunks).Select(index => session.Chunks[index]));
|
||||
if (payload.Length != chunk.TotalLength)
|
||||
{
|
||||
throw new InvalidOperationException("The assembled scan result length did not match the expected length.");
|
||||
}
|
||||
|
||||
var preview = JsonSerializer.Deserialize<ManuscriptScanPreview>(payload, JsonOptions)
|
||||
?? throw new InvalidOperationException("The assembled scan result could not be read.");
|
||||
return CompleteAsync(userId, preview);
|
||||
}
|
||||
|
||||
public Task<ManuscriptScanState> CompleteAsync(int userId, ManuscriptScanPreview preview)
|
||||
{
|
||||
var stored = preview.WithUser(userId);
|
||||
@ -98,6 +174,7 @@ public sealed class ManuscriptScanPreviewStore(
|
||||
session.TotalWordCount = stored.TotalWordCount;
|
||||
session.PreviewID = stored.PreviewID;
|
||||
session.UpdatedUtc = stored.CreatedUtc;
|
||||
session.Chunks.Clear();
|
||||
return Task.FromResult(ToState(session));
|
||||
}
|
||||
|
||||
@ -111,6 +188,7 @@ public sealed class ManuscriptScanPreviewStore(
|
||||
session.Status = ManuscriptScanStatuses.Failed;
|
||||
session.Message = Clean(message, "The scan could not finish. Try reconnecting the Word Companion.");
|
||||
session.UpdatedUtc = DateTime.UtcNow;
|
||||
session.Chunks.Clear();
|
||||
return Task.FromResult(ToState(session));
|
||||
}
|
||||
|
||||
@ -127,12 +205,23 @@ public sealed class ManuscriptScanPreviewStore(
|
||||
pair.Value.Status = ManuscriptScanStatuses.Failed;
|
||||
pair.Value.Message = Clean(message, "The Word Companion disconnected before the scan finished. Reopen Word and try again.");
|
||||
pair.Value.UpdatedUtc = DateTime.UtcNow;
|
||||
pair.Value.Chunks.Clear();
|
||||
failed.Add(ToState(pair.Value));
|
||||
}
|
||||
|
||||
return Task.FromResult<IReadOnlyList<ManuscriptScanState>>(failed);
|
||||
}
|
||||
|
||||
public Task<bool> HasRecentRunningScanForUserAsync(int userId, TimeSpan maxAge)
|
||||
{
|
||||
var now = DateTime.UtcNow;
|
||||
return Task.FromResult(sessions.Any(pair =>
|
||||
pair.Key.UserId == userId
|
||||
&& string.Equals(pair.Value.Status, ManuscriptScanStatuses.Running, StringComparison.Ordinal)
|
||||
&& pair.Value.UpdatedUtc.HasValue
|
||||
&& now - pair.Value.UpdatedUtc.Value <= maxAge));
|
||||
}
|
||||
|
||||
public Task<ManuscriptScanState> GetStateAsync(int userId, int onboardingId)
|
||||
=> Task.FromResult(sessions.TryGetValue((userId, onboardingId), out var session)
|
||||
? ToState(session)
|
||||
@ -277,6 +366,7 @@ public sealed class ManuscriptScanPreviewStore(
|
||||
private static ManuscriptScanState ToState(ScanSession session)
|
||||
=> new()
|
||||
{
|
||||
ScanID = session.ScanID,
|
||||
Status = session.Status,
|
||||
ReviewStatus = session.ReviewStatus,
|
||||
Message = session.Message,
|
||||
@ -349,6 +439,7 @@ public sealed class ManuscriptScanPreviewStore(
|
||||
|
||||
private sealed class ScanSession
|
||||
{
|
||||
public Guid ScanID { get; set; }
|
||||
public int UserID { get; init; }
|
||||
public int OnboardingID { get; init; }
|
||||
public int ProjectID { get; init; }
|
||||
@ -363,7 +454,15 @@ public sealed class ManuscriptScanPreviewStore(
|
||||
public int TotalWordCount { get; set; }
|
||||
public Guid? PreviewID { get; set; }
|
||||
public DateTime? UpdatedUtc { get; set; }
|
||||
public int? ExpectedChunkCount { get; set; }
|
||||
public int? ExpectedPayloadLength { get; set; }
|
||||
public SortedDictionary<int, string> Chunks { get; } = [];
|
||||
}
|
||||
|
||||
private static readonly JsonSerializerOptions JsonOptions = new()
|
||||
{
|
||||
PropertyNameCaseInsensitive = true
|
||||
};
|
||||
}
|
||||
|
||||
file static class ManuscriptScanPreviewExtensions
|
||||
|
||||
@ -10,6 +10,8 @@ public sealed class WordCompanionPresenceMonitor(
|
||||
IManuscriptScanPreviewStore scanStore,
|
||||
IHubContext<WordCompanionFollowHub> hub) : BackgroundService
|
||||
{
|
||||
private static readonly TimeSpan ActiveScanHeartbeatGrace = TimeSpan.FromMinutes(30);
|
||||
|
||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||
{
|
||||
using var timer = new PeriodicTimer(TimeSpan.FromSeconds(10));
|
||||
@ -17,7 +19,8 @@ public sealed class WordCompanionPresenceMonitor(
|
||||
{
|
||||
while (await timer.WaitForNextTickAsync(stoppingToken))
|
||||
{
|
||||
var offlineStatuses = await presence.MarkStaleOfflineAsync();
|
||||
var offlineStatuses = await presence.MarkStaleOfflineAsync(userId =>
|
||||
scanStore.HasRecentRunningScanForUserAsync(userId, ActiveScanHeartbeatGrace));
|
||||
foreach (var status in offlineStatuses)
|
||||
{
|
||||
await hub.Clients
|
||||
|
||||
@ -11,7 +11,7 @@ public interface IWordCompanionPresenceService
|
||||
Task<WordCompanionPresenceStatus> GetStatusAsync(int userId);
|
||||
Task<WordCompanionPresenceStatus> UpdateCurrentStoryPositionAsync(int userId, StoryPosition? position);
|
||||
Task<bool> IsConnectedAsync(int userId);
|
||||
Task<IReadOnlyList<WordCompanionPresenceStatus>> MarkStaleOfflineAsync();
|
||||
Task<IReadOnlyList<WordCompanionPresenceStatus>> MarkStaleOfflineAsync(Func<int, Task<bool>>? keepConnected = null);
|
||||
Task<string?> GetCompanionConnectionIdAsync(int userId);
|
||||
}
|
||||
|
||||
@ -141,7 +141,7 @@ public sealed class WordCompanionPresenceService : IWordCompanionPresenceService
|
||||
: null);
|
||||
}
|
||||
|
||||
public Task<IReadOnlyList<WordCompanionPresenceStatus>> MarkStaleOfflineAsync()
|
||||
public async Task<IReadOnlyList<WordCompanionPresenceStatus>> MarkStaleOfflineAsync(Func<int, Task<bool>>? keepConnected = null)
|
||||
{
|
||||
var now = DateTime.UtcNow;
|
||||
var changed = new List<WordCompanionPresenceStatus>();
|
||||
@ -154,6 +154,11 @@ public sealed class WordCompanionPresenceService : IWordCompanionPresenceService
|
||||
continue;
|
||||
}
|
||||
|
||||
if (keepConnected is not null && await keepConnected(record.UserID))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
record.Status = WordCompanionPresenceStatuses.Offline;
|
||||
if (!string.IsNullOrWhiteSpace(record.ConnectionID))
|
||||
{
|
||||
@ -164,7 +169,7 @@ public sealed class WordCompanionPresenceService : IWordCompanionPresenceService
|
||||
changed.Add(ToStatus(record, now));
|
||||
}
|
||||
|
||||
return Task.FromResult<IReadOnlyList<WordCompanionPresenceStatus>>(changed);
|
||||
return changed;
|
||||
}
|
||||
|
||||
private static WordCompanionPresenceStatus ToStatus(PresenceRecord record, DateTime now)
|
||||
|
||||
@ -235,6 +235,7 @@
|
||||
let runtimeGeneration = 0;
|
||||
let lastCurrentSceneSnapshot = null;
|
||||
const runtimeIdleDelayMs = 1200;
|
||||
const onboardingScanChunkSize = 128 * 1024;
|
||||
let selectionHandlerRegistered = false;
|
||||
let isAutoRefreshingScene = false;
|
||||
let sceneWordCountSyncTimer = null;
|
||||
@ -2487,6 +2488,51 @@
|
||||
});
|
||||
};
|
||||
|
||||
const completeOnboardingScanInChunks = async (command, preview) => {
|
||||
if (!companionPresenceConnection || companionPresenceConnection.state !== signalR.HubConnectionState.Connected) {
|
||||
throw new Error("The Word Companion connection is not available.");
|
||||
}
|
||||
|
||||
const payload = JSON.stringify(preview);
|
||||
const totalChunks = Math.max(1, Math.ceil(payload.length / onboardingScanChunkSize));
|
||||
const scanID = scanCommandValue(command, "scanID", "ScanID", "scanId");
|
||||
if (!scanID) {
|
||||
throw new Error("The scan request did not include a scan ID.");
|
||||
}
|
||||
|
||||
console.info("[Word Companion Scan] Uploading chunked scan result.", {
|
||||
scanID,
|
||||
totalChunks,
|
||||
payloadCharacters: payload.length,
|
||||
payloadBytes: new Blob([payload]).size
|
||||
});
|
||||
|
||||
for (let chunkIndex = 0; chunkIndex < totalChunks; chunkIndex += 1) {
|
||||
const start = chunkIndex * onboardingScanChunkSize;
|
||||
const chunkPayload = payload.slice(start, start + onboardingScanChunkSize);
|
||||
if (chunkIndex === 0 || chunkIndex === totalChunks - 1 || chunkIndex % 10 === 0) {
|
||||
await reportOnboardingScanProgress(
|
||||
command,
|
||||
`Uploading scan result (${chunkIndex + 1}/${totalChunks})`,
|
||||
Math.min(99, 95 + Math.floor(((chunkIndex + 1) * 4) / totalChunks)),
|
||||
preview);
|
||||
}
|
||||
|
||||
const state = await companionPresenceConnection.invoke("ReceiveOnboardingScanChunk", {
|
||||
scanID,
|
||||
userID: scanCommandValue(command, "userID", "UserID", "userId"),
|
||||
onboardingID: scanCommandValue(command, "onboardingID", "OnboardingID", "onboardingId"),
|
||||
chunkIndex,
|
||||
totalChunks,
|
||||
totalLength: payload.length,
|
||||
payload: chunkPayload
|
||||
});
|
||||
if (state?.status === "Failed" || state?.Status === "Failed") {
|
||||
throw new Error(state?.message || state?.Message || "The scan result upload failed.");
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const runOnboardingDocumentScan = async (command) => {
|
||||
if (!wordHostAvailable || !window.Word) {
|
||||
await failOnboardingScan(command, "Open your manuscript in Microsoft Word, then try the scan again.");
|
||||
@ -2494,7 +2540,13 @@
|
||||
}
|
||||
|
||||
try {
|
||||
const scanStartedAt = Date.now();
|
||||
console.info("[Word Companion Scan] Starting onboarding document scan.", {
|
||||
scanID: scanCommandValue(command, "scanID", "ScanID", "scanId"),
|
||||
onboardingID: scanCommandValue(command, "onboardingID", "OnboardingID", "onboardingId")
|
||||
});
|
||||
await reportOnboardingScanProgress(command, "Preparing document scan", 5);
|
||||
const extractionStartedAt = Date.now();
|
||||
const paragraphs = await window.Word.run(async (context) => {
|
||||
const bodyParagraphs = context.document.body.paragraphs;
|
||||
bodyParagraphs.load("items/text,items/style,items/styleBuiltIn");
|
||||
@ -2509,12 +2561,28 @@
|
||||
await context.sync();
|
||||
return bodyParagraphs.items;
|
||||
});
|
||||
console.info("[Word Companion Scan] Word extraction completed.", {
|
||||
scanID: scanCommandValue(command, "scanID", "ScanID", "scanId"),
|
||||
paragraphCount: paragraphs.length,
|
||||
durationMs: Date.now() - extractionStartedAt
|
||||
});
|
||||
|
||||
const previewStartedAt = Date.now();
|
||||
const preview = buildOnboardingScanPreview(paragraphs, command);
|
||||
console.info("[Word Companion Scan] Preview built.", {
|
||||
scanID: scanCommandValue(command, "scanID", "ScanID", "scanId"),
|
||||
chapterCount: preview.chapterCount,
|
||||
totalWordCount: preview.totalWordCount,
|
||||
durationMs: Date.now() - previewStartedAt
|
||||
});
|
||||
await reportOnboardingScanProgress(command, "Preparing chapter review", 85, preview);
|
||||
await reportOnboardingScanProgress(command, "Preparing preview", 95, preview);
|
||||
lastOnboardingScanPreview = preview;
|
||||
await companionPresenceConnection.invoke("CompleteOnboardingScan", preview);
|
||||
await completeOnboardingScanInChunks(command, preview);
|
||||
console.info("[Word Companion Scan] Onboarding document scan uploaded.", {
|
||||
scanID: scanCommandValue(command, "scanID", "ScanID", "scanId"),
|
||||
durationMs: Date.now() - scanStartedAt
|
||||
});
|
||||
} catch (error) {
|
||||
console.error("Unable to complete onboarding scan.", error);
|
||||
await failOnboardingScan(command, friendlyScanError(error));
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user