Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions src/Particular.LicensingComponent.Contracts/Endpoint.cs
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ public Endpoint(string name, ThroughputSource source)
{
}

public DateTime? CreatedDate { get; set; }
public string SanitizedName { get; set; }
public string[] EndpointIndicators { get; set; }
public string UserIndicator { get; set; }
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,251 @@
namespace Particular.LicensingComponent.UnitTests;

using System;
using System.Collections.Generic;
using System.Linq;
using System.Runtime.CompilerServices;
using System.Threading;
using System.Threading.Tasks;
using Contracts;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Time.Testing;
using NUnit.Framework;
using Persistence;
using Persistence.InMemory;
using Report;

[TestFixture]
class DecommissionedEndpointRemovalHostedServiceTests
{
[TestCase(-25, true)]
[TestCase(-24, true)]
[TestCase(-23, false)]
[TestCase(0, false)]
[TestCase(24, false)]
public async Task EnsuringCreatedDateAllowsOneDayBeforeRemovingEndpoints(int hoursSinceNow, bool shouldBeRemoved)
{
var timeProvider = new FakeTimeProvider();
var endpoint = new Endpoint("sales", ThroughputSource.Broker)
{
CreatedDate = timeProvider.GetUtcNow().UtcDateTime.AddHours(hoursSinceNow)
};
var dataStore = new InMemoryLicensingDataStore();
await dataStore.SaveEndpoint(endpoint);
var throughputCollector = new MockedThroughputCollector(_ => [CreateThroughput([endpoint])]);

await RunService(dataStore, throughputCollector, timeProvider);

var remainingEndpoints = (await dataStore.GetAllEndpoints(true)).ToArray();
Assert.That(remainingEndpoints, shouldBeRemoved ? Is.Empty : Is.EquivalentTo(new[] { endpoint }));
}

[TestCase(false)]
[TestCase(true)]
public async Task EnsuringEndpointsWithoutCreatedDateOrThroughputAreRemoved(bool hasEmptyThroughputData)
{
var endpoint = new Endpoint("sales", ThroughputSource.Broker);
var dataStore = new InMemoryLicensingDataStore();
await dataStore.SaveEndpoint(endpoint);
var throughput = hasEmptyThroughputData
? CreateThroughput([endpoint], [], [])
: CreateThroughput([endpoint]);
var throughputCollector = new MockedThroughputCollector(_ => [throughput]);

await RunService(dataStore, throughputCollector, new FakeTimeProvider());

Assert.That(await dataStore.GetAllEndpoints(true), Is.Empty);
}

[TestCase(0)]
[TestCase(10)]
public async Task EnsuringEndpointsWithThroughputAreNotRemoved(long messageCount)
{
var timeProvider = new FakeTimeProvider();
var endpoint = new Endpoint("sales", ThroughputSource.Broker);
var dataStore = new InMemoryLicensingDataStore();
await dataStore.SaveEndpoint(endpoint);
var throughputData = new ThroughputData
{
[DateOnly.FromDateTime(timeProvider.GetUtcNow().UtcDateTime)] = messageCount
};
var throughputCollector = new MockedThroughputCollector(_ => [CreateThroughput([endpoint], [], throughputData)]);

await RunService(dataStore, throughputCollector, timeProvider);

Assert.That(await dataStore.GetAllEndpoints(true), Is.EquivalentTo(new[] { endpoint }));
}

[Test]
public async Task EnsuringOnlyOlderEndpointsInTheSameGroupAreRemoved()
{
var timeProvider = new FakeTimeProvider();
var olderEndpoint = new Endpoint("sales", ThroughputSource.Broker)
{
CreatedDate = timeProvider.GetUtcNow().UtcDateTime.AddDays(-2)
};
var legacyEndpoint = new Endpoint("sales", ThroughputSource.Audit);
var newEndpoint = new Endpoint("sales", ThroughputSource.Monitoring)
{
CreatedDate = timeProvider.GetUtcNow().UtcDateTime
};
var dataStore = new InMemoryLicensingDataStore();
await dataStore.SaveEndpoint(olderEndpoint);
await dataStore.SaveEndpoint(legacyEndpoint);
await dataStore.SaveEndpoint(newEndpoint);
var throughputCollector = new MockedThroughputCollector(_ => [CreateThroughput([olderEndpoint, legacyEndpoint, newEndpoint])]);

await RunService(dataStore, throughputCollector, timeProvider);

Assert.That(await dataStore.GetAllEndpoints(true), Is.EquivalentTo(new[] { newEndpoint }));
}

[TestCase(false)]
[TestCase(true)]
public async Task EnsuringCleanupRunsAgainTheNextDayAfterCollection(bool firstCollectionThrows)
{
using var tokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(3));
var token = tokenSource.Token;
var timeProvider = new FakeTimeProvider();
var endpoint = new Endpoint("sales", ThroughputSource.Broker)
{
CreatedDate = timeProvider.GetUtcNow().UtcDateTime
};
var dataStore = new InMemoryLicensingDataStore();
await dataStore.SaveEndpoint(endpoint);
var throughputCollector = new MockedThroughputCollector(call =>
{
if (firstCollectionThrows && call == 1)
{
throw new InvalidOperationException("Collection failed");
}

return [CreateThroughput([endpoint])];
});
using var service = new DecommissionedEndpointRemovalHostedService(
NullLogger<DecommissionedEndpointRemovalHostedService>.Instance,
dataStore, throughputCollector, timeProvider);

await service.StartAsync(token);
await throughputCollector.FirstCollectionCompleted.Task.WaitAsync(token);
Assert.That(await dataStore.GetAllEndpoints(true, token), Is.EquivalentTo(new[] { endpoint }));

timeProvider.Advance(TimeSpan.FromHours(23));
Assert.That(throughputCollector.GetDistinctEndpointThroughputCalls, Is.EqualTo(1));

timeProvider.Advance(TimeSpan.FromHours(1));
await throughputCollector.SecondCollectionCompleted.Task.WaitAsync(token);
await service.StopAsync(token);

using (Assert.EnterMultipleScope())
{
Assert.That(throughputCollector.GetDistinctEndpointThroughputCalls, Is.EqualTo(2));
Assert.That(await dataStore.GetAllEndpoints(true, token), Is.Empty);
}
}

[TestCase(false)]
[TestCase(true)]
public async Task EnsuringRemovalExceptionsAreHandledAndCleanupKeepsGoing(bool throwsCancellationException)
{
var failedEndpoint = new Endpoint("sales", ThroughputSource.Broker);
var otherEndpoint = new Endpoint("marketing", ThroughputSource.Broker);
var dataStore = new MockedLicensingDataStoreThatThrowsExceptions(failedEndpoint.Id, throwsCancellationException);
await dataStore.SaveEndpoint(failedEndpoint);
await dataStore.SaveEndpoint(otherEndpoint);
var throughputCollector = new MockedThroughputCollector(_ =>
[CreateThroughput([failedEndpoint]), CreateThroughput([otherEndpoint])]);

await RunService(dataStore, throughputCollector, new FakeTimeProvider());

Assert.That(await dataStore.GetAllEndpoints(true), Is.EquivalentTo([failedEndpoint]));
}

[Test]
public async Task EnsuringHostedServiceStopsCleanly()
{
using var tokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(3));
var token = tokenSource.Token;
var throughputCollector = new MockedThroughputCollector(_ => []);
using var service = new DecommissionedEndpointRemovalHostedService(
NullLogger<DecommissionedEndpointRemovalHostedService>.Instance,
new InMemoryLicensingDataStore(), throughputCollector, new FakeTimeProvider());

await service.StartAsync(token);
await throughputCollector.FirstCollectionCompleted.Task.WaitAsync(token);
await service.StopAsync(token);

Assert.That(service.ExecuteTask?.IsCompletedSuccessfully, Is.True);
}

static EndpointThroughput CreateThroughput(Endpoint[] endpoints, params ThroughputData[] throughputData) =>
new(endpoints[0].Id.Name, endpoints.GroupBy(endpoint => endpoint.Id.Name).Single(), [.. throughputData]);

static async Task RunService(ILicensingDataStore dataStore, MockedThroughputCollector throughputCollector, FakeTimeProvider timeProvider)
{
using var tokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(3));
var token = tokenSource.Token;
using var service = new DecommissionedEndpointRemovalHostedService(
NullLogger<DecommissionedEndpointRemovalHostedService>.Instance,
dataStore, throughputCollector, timeProvider);

await service.StartAsync(token);
await throughputCollector.FirstCollectionCompleted.Task.WaitAsync(token);
await service.StopAsync(token);
}

class MockedLicensingDataStoreThatThrowsExceptions(EndpointIdentifier failedEndpointId, bool throwsCancellationException)
: InMemoryLicensingDataStore, ILicensingDataStore
{
public new Task RemoveEndpoints(EndpointIdentifier[] endpointIds, CancellationToken cancellationToken = default)
{
if (endpointIds.Contains(failedEndpointId))
{
throw throwsCancellationException ? new OperationCanceledException() : new InvalidOperationException("Removal failed");
}

return base.RemoveEndpoints(endpointIds, cancellationToken);
}
}

class MockedThroughputCollector(Func<int, IEnumerable<EndpointThroughput>> getThroughput) : IThroughputCollector
{
public int GetDistinctEndpointThroughputCalls { get; private set; }
public TaskCompletionSource FirstCollectionCompleted { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
public TaskCompletionSource SecondCollectionCompleted { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);

public async IAsyncEnumerable<EndpointThroughput> GetDistinctEndpointThroughput(DateOnly? throughputMaxDate = null,
[EnumeratorCancellation] CancellationToken cancellationToken = default)
{
await Task.CompletedTask;
var call = ++GetDistinctEndpointThroughputCalls;
try
{
foreach (var throughput in getThroughput(call))
{
cancellationToken.ThrowIfCancellationRequested();
yield return throughput;
}
}
finally
{
if (call == 1)
{
FirstCollectionCompleted.TrySetResult();
}
else
{
SecondCollectionCompleted.TrySetResult();
}
}
}

public Task<List<EndpointThroughputSummary>> GetThroughputSummary(CancellationToken cancellationToken = default) => throw new NotImplementedException();
public Task UpdateUserIndicatorsOnEndpoints(List<UpdateUserIndicator> userIndicatorUpdates, CancellationToken cancellationToken = default) => throw new NotImplementedException();
public Task<ThroughputConnectionSettings> GetThroughputConnectionSettingsInformation(CancellationToken cancellationToken = default) => throw new NotImplementedException();
public Task<ConnectionTestResults> TestConnectionSettings(CancellationToken cancellationToken = default) => throw new NotImplementedException();
public Task<SignedReport> GenerateThroughputReport(string spVersion, DateTime? reportEndDate, CancellationToken cancellationToken = default) => throw new NotImplementedException();
public Task<ReportGenerationState> GetReportGenerationState(CancellationToken cancellationToken = default) => throw new NotImplementedException();
public Task<List<string>> GetReportMasks(CancellationToken cancellationToken = default) => throw new NotImplementedException();
public Task UpdateReportMasks(List<string> reportMaskUpdates, CancellationToken cancellationToken = default) => throw new NotImplementedException();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
using AuditThroughput;
using Contracts;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Time.Testing;
using MonitoringThroughput;
using Persistence;
using Persistence.InMemory;
Expand All @@ -30,6 +31,7 @@ public Task Configure(

var serviceCollection = new ServiceCollection();
serviceCollection.AddSingleton(throughputSettings);
serviceCollection.AddSingleton<TimeProvider>(new FakeTimeProvider());

serviceCollection.AddLogging();

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ namespace Particular.LicensingComponent.UnitTests;
using Approvals;
using Contracts;
using Infrastructure;
using Microsoft.Extensions.Time.Testing;
using MonitoringThroughput;
using NUnit.Framework;
using Persistence;
Expand Down Expand Up @@ -84,7 +85,7 @@ public async Task Should_sanitize_endpoint_name()
EndpointThroughputData = new EndpointThroughputData[] { new() { Name = endpointName, Throughput = 15 } }
};

var monitoringService = new MonitoringService(DataStore, new BrokerThroughputQuery_WithSanitization());
var monitoringService = new MonitoringService(DataStore, new FakeTimeProvider(), new BrokerThroughputQuery_WithSanitization());
byte[] messageBytes = JsonSerializer.SerializeToUtf8Bytes(message);
await monitoringService.RecordMonitoringThroughput(messageBytes);
string endpointNameSanitized = "e-ndpoint-1";
Expand Down Expand Up @@ -166,7 +167,7 @@ public async Task Should_record_throughput_for_every_endpoint_in_the_message()
{
// Arrange
var dataStore = new YieldingLicensingDataStore();
var monitoringService = new MonitoringService(dataStore);
var monitoringService = new MonitoringService(dataStore, new FakeTimeProvider());

string[] endpointNames = [.. Enumerable.Range(1, 20).Select(i => $"Endpoint{i}")];
var message = new RecordEndpointThroughputData
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,8 @@ Endpoint ConvertToEndpoint(ServiceControlEndpoint scEndpoint)
var endpoint = new Endpoint(scEndpoint.Name, ThroughputSource.Audit)
{
SanitizedName = brokerThroughputQuery != null ? brokerThroughputQuery.SanitizeEndpointName(scEndpoint.Name) : scEndpoint.Name,
EndpointIndicators = [EndpointIndicator.KnownEndpoint.ToString()]
EndpointIndicators = [EndpointIndicator.KnownEndpoint.ToString()],
CreatedDate = timeProvider.GetUtcNow().UtcDateTime,
};

if (PlatformEndpointHelper.IsPlatformEndpoint(scEndpoint.Name, throughputSettings))
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,8 @@ async Task Exec(IBrokerQueue queueName, string postfix)
{
SanitizedName = queueName.SanitizedName + postfix,
Scope = queueName.Scope,
EndpointIndicators = [.. queueName.EndpointIndicators]
EndpointIndicators = [.. queueName.EndpointIndicators],
CreatedDate = timeProvider.GetUtcNow().UtcDateTime,
};

await dataStore.SaveEndpoint(endpoint, cancellationToken);
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
namespace Particular.LicensingComponent
{
using System;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using Particular.LicensingComponent.Persistence;

/// <summary>
/// Removes EndpointDocument/LicensingEndpoint entities that haven't seen throughput for ThroughputReporting.ReportedMonths,
/// since these can be considered to be decommissioned
/// </summary>
public class DecommissionedEndpointRemovalHostedService(
ILogger<DecommissionedEndpointRemovalHostedService> logger,
ILicensingDataStore dataStore,
IThroughputCollector throughputCollector,
TimeProvider timeProvider)
: BackgroundService
{
// This doesn't need to be run too often, since the difference between ThroughputReporting.ReportedMonths and
// ThroughputReporting.ReportedMonths - 1day is insignificant when determining whether an endpoint should be
// considered decommissioned or not
const int timerPeriodDays = 1;

protected override async Task ExecuteAsync(CancellationToken cancellationToken = default)
{
logger.LogInformation("Starting {ServiceName}", nameof(DecommissionedEndpointRemovalHostedService));

try
{
using PeriodicTimer timer = new(TimeSpan.FromDays(timerPeriodDays), timeProvider);

do
{
try
{
await foreach (var data in throughputCollector.GetDistinctEndpointThroughput(null, cancellationToken).Where(data => !data.ThroughputData.Any(x => x.Any())))
{
try
{
//it's possible that endpoints have been created but haven't yet had their throughput written. Give leeway on CreatedDate to allow for this scenario.
//endpoints that were created before the CreatedDate field was introduced are guaranteed to be older and hence have valid throughput
var endpointsWithoutThroughput = data.EndpointGroupPerQueue.Where(endpoint => !endpoint.CreatedDate.HasValue || endpoint.CreatedDate.Value <= timeProvider.GetUtcNow().UtcDateTime.AddDays(-1)).ToArray();
if (endpointsWithoutThroughput.Any())
{
logger.LogInformation("Removing Endpoint {EndpointName} from throughput due to no recent activity", data.EndpointName);
await dataStore.RemoveEndpoints([.. endpointsWithoutThroughput.Select(endpoint => endpoint.Id)], cancellationToken);
}
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
throw;
}
catch (Exception ex)
{
logger.LogError(ex, $"Error encountered during decommissioned endpoint cleanup for endpoint {data.EndpointName}. Will try again in {timerPeriodDays} days.");
}
}
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
throw;
}
catch (Exception ex)
{
logger.LogError(ex, $"Error encountered during decommissioned endpoint cleanup. Will try again in {timerPeriodDays} days.");
}
} while (await timer.WaitForNextTickAsync(cancellationToken));
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
logger.LogInformation("Stopping {ServiceName}", nameof(DecommissionedEndpointRemovalHostedService));
}
}
}
}
Loading
Loading