Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion docs/migration/ravendb-to-sql-migration-instructions.md
Original file line number Diff line number Diff line change
Expand Up @@ -190,7 +190,7 @@ Keep RavenDB running until every category is Done or Abandoned. If RavenDB goes
## Step 8: Finish the migration

1. Stop ServiceControl.
2. Run `--migration-verify`. It counts the rows in RavenDB again, and compares them with the rows that the copy read. It also shows the skipped and merged rows. It exits with 0 only when every category is Done or Abandoned.
2. Run `--migration-verify`. It counts the rows in RavenDB again, and compares them with the rows that the copy read. It also shows the skipped rows. Merged and changed rows are not shown here. Each one is logged as a warning by the copy run that merged or changed it, and that run's summary counts them. It exits with 0 only when every category is Done or Abandoned.
3. Start ServiceControl. Finished checkpoints remain in SQL, but no background copy runs and RavenDB is not opened.

Verify reads all the data in RavenDB again, but not the message bodies. On a large instance this takes time, and ServiceControl is stopped during that time.
Expand Down
60 changes: 36 additions & 24 deletions docs/migration/ravendb-to-sql-migration-overview.md

Large diffs are not rendered by default.

34 changes: 18 additions & 16 deletions docs/migration/ravendb-to-sql-migration-system-design.md

Large diffs are not rendered by default.

Original file line number Diff line number Diff line change
@@ -0,0 +1,97 @@
namespace ServiceControl.Migration.AcceptanceTests;

using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using NUnit.Framework;
using ServiceControl.Contracts.CustomChecks;
using ServiceControl.Operations;
using ServiceControl.Persistence;
using ServiceControl.Persistence.DataMigration;
using ServiceControl.Persistence.Infrastructure;

[TestFixture]
[NonParallelizable]
class CustomCheckMigrationTests : MigrationAcceptanceTest
{
static readonly EndpointDetails Endpoint = new() { Name = "Sales", Host = "MACHINE", HostId = Guid.NewGuid() };

static Guid IdOf(EndpointDetails endpoint, string customCheckId) =>
new CustomCheckDetail { OriginatingEndpoint = endpoint, CustomCheckId = customCheckId, Category = "Ping" }.GetDeterministicId();

async Task<IList<CustomCheckView>> CopiedChecks() => (await CustomChecks.GetStats(new PagingInfo())).Results!;

// One of each status, because Status.Fail is the enum's zero and a status that never deserializes would read as Fail.
[Test]
public async Task Copies_a_failing_and_a_passing_custom_check_with_their_deterministic_ids_and_reported_status()
{
await SeedSourceCustomCheck(Endpoint, "Sales.HealthCheck", "Ping", failed: true, reason: "timed out");
await SeedSourceCustomCheck(Endpoint, "Sales.DiskCheck", "Disk", failed: false);

var result = await CopyCategory("CustomChecks");

var copied = await CopiedChecks();
var failing = copied.Single(check => check.CustomCheckId == "Sales.HealthCheck");
var passing = copied.Single(check => check.CustomCheckId == "Sales.DiskCheck");

using (Assert.EnterMultipleScope())
{
Assert.That(result.Copied, Is.EqualTo(2));
Assert.That(await Target.Count(MigrationCategoryRegistry.Find(MigrationCategoryIds.CustomChecks)!), Is.EqualTo(2), "Count is what --migration-verify reads");
Assert.That(Guid.Parse(failing.Id!), Is.EqualTo(IdOf(Endpoint, "Sales.HealthCheck")), "a live report of the same check upserts on this id, so it replaces the copied row rather than adding a second");
Assert.That(failing.Status, Is.EqualTo(Status.Fail));
Assert.That(failing.Category, Is.EqualTo("Ping"));
Assert.That(failing.FailureReason, Is.EqualTo("timed out"));
Assert.That(failing.ReportedAt, Is.EqualTo(SeededReportedAt));
Assert.That((failing.OriginatingEndpoint!.Name, failing.OriginatingEndpoint.Host, failing.OriginatingEndpoint.HostId), Is.EqualTo((Endpoint.Name, Endpoint.Host, Endpoint.HostId)));
Assert.That(Guid.Parse(passing.Id!), Is.EqualTo(IdOf(Endpoint, "Sales.DiskCheck")));
Assert.That(passing.Status, Is.EqualTo(Status.Pass), "a status read back as the enum's zero would show every copied check red");
Assert.That(passing.Category, Is.EqualTo("Disk"));
Assert.That(passing.FailureReason, Is.Null);
}
}

[Test]
public async Task A_check_with_no_originating_endpoint_is_skipped_as_a_required_value_missing()
{
await SeedSource(($"CustomChecks/{Guid.NewGuid()}", new { CustomCheckId = "Sales.HealthCheck", Category = "Ping", Status = (int)Status.Pass, ReportedAt = DateTime.UtcNow }));

var result = await CopyCategory("CustomChecks");
var checkpoint = await ReadCheckpoint("CustomChecks");

using (Assert.EnterMultipleScope())
{
Assert.That(result.Saved.State, Is.EqualTo(MigrationCategoryState.CompleteWithErrors), "one fault skip ends a required category Failed");
Assert.That(checkpoint.CopiedCount, Is.Zero);
Assert.That(checkpoint.SkipReasons?.GetValueOrDefault(MigrationSkipReason.RequiredValueMissing), Is.EqualTo(1), "CustomChecks.OriginatingEndpointName is NOT NULL, and a throw would stop the batch rather than count the row");
}
}

[Test]
public async Task Only_a_check_whose_id_and_category_match_a_RavenDB_persister_check_is_skipped_as_harmless_and_the_category_ends_done()
{
var serviceControl = new EndpointDetails { Name = "Particular.ServiceControl", Host = "MACHINE", HostId = Guid.NewGuid() };
var audit = new EndpointDetails { Name = "Particular.ServiceControl.Audit", Host = "MACHINE", HostId = Guid.NewGuid() };
await SeedSourceCustomCheck(serviceControl, "Error Database Index Lag", "ServiceControl Health", failed: true, reason: "index lag");
await SeedSourceCustomCheck(audit, "Audit Database Index Lag", "ServiceControl.Audit Health", failed: true, reason: "index lag");
await SeedSourceCustomCheck(audit, "RavenDB dirty memory", "ServiceControl.Audit Health", failed: true, reason: "dirty memory");
await SeedSourceCustomCheck(Endpoint, "Sales.HealthCheck", "Ping", failed: true, reason: "timed out");
await SeedSourceCustomCheck(Endpoint, "Message Ingestion Process", "Sales", failed: true, reason: "stuck");

var result = await CopyCategory("CustomChecks");

using (Assert.EnterMultipleScope())
{
Assert.That(result.Saved.State, Is.EqualTo(MigrationCategoryState.Complete), "a harmless skip leaves a required category Done, so ServiceControl can open");
Assert.That(result.Saved.SkipReasons, Is.EqualTo(new Dictionary<MigrationSkipReason, long> { [MigrationSkipReason.SourceStorageCheck] = 1 }), "only the primary's index lag check is the RavenDB persister's own");
Assert.That((await CopiedChecks()).Select(check => (check.CustomCheckId, check.Category)), Is.EquivalentTo(new[]
{
("Audit Database Index Lag", "ServiceControl.Audit Health"),
("RavenDB dirty memory", "ServiceControl.Audit Health"),
("Sales.HealthCheck", "Ping"),
("Message Ingestion Process", "Sales")
}), "the skip matches the id and the category, so the audit instance's checks and a customer check with a colliding id keep their failing results");
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,162 @@
namespace ServiceControl.Migration.AcceptanceTests;

using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using Microsoft.EntityFrameworkCore;
using NUnit.Framework;
using ServiceControl.Operations;
using ServiceControl.Persistence.DataMigration;
using ServiceControl.Persistence.EFCore.Entities;

[TestFixture]
[NonParallelizable]
class FailedErrorImportMigrationTests : MigrationAcceptanceTest
{
[Test]
public async Task Copies_a_small_body_inline_with_the_time_RavenDB_stored_the_import_as_its_failure_time()
{
await SeedSourceFailedErrorImport("native-1", [1, 2, 3], "Deserialization failed");

var result = await CopyCategory(MigrationCategoryIds.FailedErrorImports);
var import = await GetImport("native-1");

using (Assert.EnterMultipleScope())
{
Assert.That(result.Saved.State, Is.EqualTo(MigrationCategoryState.Complete));
Assert.That(result.Copied, Is.EqualTo(1));
Assert.That(await Target.Count(Category), Is.EqualTo(1), "Count is what --migration-verify reads");
Assert.That(import.Body, Is.EqualTo(new byte[] { 1, 2, 3 }));
Assert.That(import.BodyStoredExternally, Is.False);
Assert.That(import.ExceptionInfo, Is.EqualTo("Deserialization failed"));
Assert.That(import.FailedAt, Is.EqualTo(await SourceLastModified("native-1")).Within(TimeSpan.FromMilliseconds(1)), "replay runs oldest first by this time, so the time of the copy would reorder a customer's stuck imports");
Assert.That(RecordedBodies.Written, Is.Empty);
}
}

[Test]
public async Task Migrated_bodies_replay_through_the_products_own_import_path()
{
byte[] small = [1, 2, 3];
var large = LargeBody();
await SeedSourceFailedErrorImport("native-3", small, "Deserialization failed");
await SeedSourceFailedErrorImport("native-4", large, "Deserialization failed");

await CopyCategory(MigrationCategoryIds.FailedErrorImports);

var external = await GetImport("native-4");
Assert.That(external.BodyStoredExternally, Is.True, "otherwise this proves nothing about the external placement");
Assert.That(external.Body, Is.Empty, "the product keeps no inline copy of a body it stores in body storage");

var replayed = new Dictionary<string, FailedTransportMessage>();
await FailedImportStore.ProcessFailedErrorImports((message, _) =>
{
replayed[message.Id] = message;
return Task.CompletedTask;
}, TestContext.CurrentContext.CancellationToken);

using (Assert.EnterMultipleScope())
{
Assert.That(replayed.GetValueOrDefault("native-3")?.Body, Is.EqualTo(small), "an inline body is read from the row");
Assert.That(replayed.GetValueOrDefault("native-4")?.Body, Is.EqualTo(large), "an external body is read from body storage under the body id the SQL import store uses for that key");
Assert.That(replayed.GetValueOrDefault("native-3")?.Headers, Is.EqualTo(HeadersFor("native-3")), "the replay ingests the message under the headers the copy stored");
Assert.That(replayed.GetValueOrDefault("native-4")?.Headers, Is.EqualTo(HeadersFor("native-4")));
}
}

[Test]
public async Task A_retry_finds_every_import_already_present_and_writes_no_body_a_second_time()
{
await SeedSourceFailedErrorImport("native-2", [9], "Deserialization failed");
await SeedSourceFailedErrorImport("native-7", LargeBody(), "Deserialization failed");
await CopyCategory(MigrationCategoryIds.FailedErrorImports);
var bodiesWritten = RecordedBodies.Written.Count;

await Target.ResetForRetry(RetryResetMigrationTests.ResetForRetry(await ReadCheckpoint(MigrationCategoryIds.FailedErrorImports)));
var second = await CopyCategory(MigrationCategoryIds.FailedErrorImports);

using (Assert.EnterMultipleScope())
{
Assert.That(bodiesWritten, Is.EqualTo(1), "the large body is the only one the first copy writes to body storage");
Assert.That(second.AlreadyPresent, Is.EqualTo(2), "--migration-retry re-reads from the start, and the rows the first copy wrote are left alone");
Assert.That(await Target.Count(Category), Is.EqualTo(2));
Assert.That(RecordedBodies.Written, Has.Count.EqualTo(bodiesWritten), "a body is written only for an import the target does not hold yet");
}
}

[Test]
public async Task Only_the_first_import_of_a_key_in_a_batch_has_its_body_written()
{
var first = LargeBody();
var second = LargeBody();
await SeedSource((DocumentId("native-8-a"), ImportOf("native-8", first)), (DocumentId("native-8-b"), ImportOf("native-8", second)));

var result = await CopyCategory(MigrationCategoryIds.FailedErrorImports);

using (Assert.EnterMultipleScope())
{
Assert.That(result.Copied, Is.EqualTo(1));
Assert.That(result.AlreadyPresent, Is.EqualTo(1), "both documents derive one key, and the insert keeps the first row of a key");
Assert.That(RecordedBodies.Written.Select(written => written.Body), Is.EqualTo(new[] { first }), "a second body under the same id would replace the body of the row the insert kept");
}
}

[Test]
public async Task An_import_with_no_message_is_skipped_as_a_required_value_missing()
{
await SeedSource((DocumentId("native-5"), new FailedErrorImport { Id = "native-5", Message = null, ExceptionInfo = "Deserialization failed" }));
await SeedSourceFailedErrorImport("native-6", [6], "Deserialization failed");

var result = await CopyCategory(MigrationCategoryIds.FailedErrorImports);
Assert.That(result.Saved.State, Is.EqualTo(MigrationCategoryState.CompleteWithErrors), "one fault skip ends a required category Failed");

var checkpoint = await ReadCheckpoint(MigrationCategoryIds.FailedErrorImports);

using (Assert.EnterMultipleScope())
{
Assert.That(checkpoint.CopiedCount, Is.EqualTo(1), "a skip copies the rest of the batch, where a throw would have stopped it");
Assert.That(checkpoint.SkippedCount, Is.EqualTo(1));
Assert.That(checkpoint.SkipReasons, Is.EqualTo(new Dictionary<MigrationSkipReason, long> { [MigrationSkipReason.RequiredValueMissing] = 1 }),
"FailedErrorImports.MessageId is NOT NULL; the source attaches an empty body, so nothing is skipped as BodyMissing or BodyUnreadable");
}
}

static readonly MigrationCategory Category = MigrationCategoryRegistry.Find(MigrationCategoryIds.FailedErrorImports)!;

static string DocumentId(string nativeMessageId) => $"FailedErrorImports/{nativeMessageId}";

static Dictionary<string, string> HeadersFor(string nativeMessageId) => new() { ["NServiceBus.MessageId"] = nativeMessageId };

// One byte over the limit, because the store moves a body out only when it is larger than the limit.
byte[] LargeBody()
{
var body = new byte[EFSettings.BodyStorage.MaxBodySizeToStore + 1];
Random.Shared.NextBytes(body);
return body;
}

Task SeedSourceFailedErrorImport(string nativeMessageId, byte[] body, string exceptionInfo) =>
SeedSource((DocumentId(nativeMessageId), ImportOf(nativeMessageId, body, exceptionInfo)));

static FailedErrorImport ImportOf(string nativeMessageId, byte[] body, string exceptionInfo = "Deserialization failed") => new()
{
Id = nativeMessageId,
Message = new FailedTransportMessage { Id = nativeMessageId, Headers = HeadersFor(nativeMessageId), Body = body },
ExceptionInfo = exceptionInfo
};

async Task<DateTime> SourceLastModified(string nativeMessageId)
{
using var session = SourceStore.OpenAsyncSession(Source.PrimaryDatabase);

return session.Advanced.GetLastModifiedFor(await session.LoadAsync<FailedErrorImport>(DocumentId(nativeMessageId)))!.Value.ToUniversalTime();
}

Task<FailedErrorImportEntity> GetImport(string nativeMessageId)
{
var key = FailedErrorImport.DeriveKey(HeadersFor(nativeMessageId), nativeMessageId);

return QueryTarget(dbContext => dbContext.FailedErrorImports.AsNoTracking().SingleAsync(row => row.UniqueMessageId == key));
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
namespace ServiceControl.Migration.AcceptanceTests;

using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using NUnit.Framework;
using ServiceControl.MessageFailures;
using ServiceControl.Persistence.DataMigration;
using ServiceControl.Persistence.Infrastructure;
using ServiceControl.Persistence.Tests;

[TestFixture]
[NonParallelizable]
class GroupCommentsMigrationTests : MigrationAcceptanceTest
{
static readonly MigrationCategory GroupCommentsCategory = MigrationCategoryRegistry.Find(MigrationCategoryIds.GroupComments)!;

[Test]
public async Task Copies_each_comment_onto_its_group_and_an_orphaned_one_too_and_skips_a_null_or_blank_one_as_harmless()
{
var kept = new IngestedFailure();
var cleared = new IngestedFailure();
var neverWritten = new IngestedFailure();
var spaces = new IngestedFailure();
await SeedSourceWithBodies([.. new[] { kept, cleared, neverWritten, spaces }.Select(failure => (failure.ToFailedMessage(), failure.Body, failure.ContentType))]);
await SeedSourceGroupComments(
(kept.Groups.Single().Id, "Waiting on the payments team"),
(cleared.Groups.Single().Id, ""),
(neverWritten.Groups.Single().Id, null),
(spaces.Groups.Single().Id, " "),
("group-with-no-copied-message", "Raised with the shipping team"));
await CopyCategory(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages);

var result = await CopyCategory(MigrationCategoryIds.GroupComments);
var comments = (await GroupsStore.GetUnresolvedGroupsByClassifier(kept.Groups.Single().Type, null, new PagingInfo())).ToDictionary(view => view.Id, view => view.Comment);

using (Assert.EnterMultipleScope())
{
Assert.That(result.Saved.State, Is.EqualTo(MigrationCategoryState.Complete), "SQL never stores a blank comment, so leaving one behind loses nothing and must not keep ServiceControl closed");
Assert.That(result.Copied, Is.EqualTo(2), "a comment whose group has no copied message is copied too, and left to the retention sweeper");
Assert.That(await Target.Count(GroupCommentsCategory), Is.EqualTo(2), "--migration-verify reads this count");
Assert.That(result.Saved.SkipReasons, Is.EqualTo(new Dictionary<MigrationSkipReason, long> { [MigrationSkipReason.BlankGroupComment] = 3 }), "a harmless skip is still counted, so status and verify can show it");
Assert.That(comments[kept.Groups.Single().Id], Is.EqualTo("Waiting on the payments team"), "ServicePulse's group list shows the comment on the group it was written for");
Assert.That(new[] { cleared, neverWritten, spaces }.Select(failure => comments[failure.Groups.Single().Id]), Is.All.Null, "the product shows no comment for a group whose comment was blank");
}
}

// The prefix is retyped because this project cannot reference the RavenDB persister that declares it.
Task SeedSourceGroupComments(params (string GroupId, string Comment)[] comments) =>
SeedSource([.. comments.Select(comment => ($"GroupComment/{comment.GroupId}", (object)new GroupComment { Id = $"GroupComment/{comment.GroupId}", Comment = comment.Comment }))]);
}
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,7 @@ public async Task Two_names_differing_only_in_a_trailing_space_never_fail_the_ba
using (Assert.EnterMultipleScope())
{
Assert.That(result.Copied, Is.EqualTo(keyIgnoresTrailingSpace ? 1 : 2), "SQL Server's key ignores the trailing space and keeps the first row; PostgreSQL's does not and keeps both");
Assert.That(result.AlreadyPresent, Is.EqualTo(keyIgnoresTrailingSpace ? 1 : 0), "the row SQL Server's key treats as the first's is already present, not lost");
Assert.That(result.AlreadyPresent, Is.EqualTo(keyIgnoresTrailingSpace ? 1 : 0), "the row SQL Server's key treats as the first's is counted as already present");
Assert.That(result.Skipped, Is.Zero);
Assert.That(await Target.Count(category), Is.EqualTo(result.Copied), "every row reported as copied is a row in the table");

Expand Down
Loading
Loading