From 6de9642fb463341f9d8433df0acd0a973dc48477 Mon Sep 17 00:00:00 2001 From: David Runemalm Date: Mon, 17 Feb 2025 17:17:24 +0100 Subject: [PATCH 001/109] Fix target in makefile. --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 437654a..7409780 100644 --- a/Makefile +++ b/Makefile @@ -203,7 +203,7 @@ templates-pack: ##@Template Pack the OpenDDD.NET project template into a NuGet dotnet pack $(TEMPLATES_CSPROJ) -o $(TEMPLATES_OUT) .PHONY: templates-publish -templates-publish: template-pack ##@Template Publish the template to NuGet +templates-publish: ##@Template Publish the template to NuGet dotnet nuget push $(TEMPLATES_NUPKG) --api-key $(NUGET_API_KEY) --source https://api.nuget.org/v3/index.json .PHONY: templates-rebuild From 89c8d95b3b5f8226e426c447cc43c4873e55079e Mon Sep 17 00:00:00 2001 From: David Runemalm Date: Mon, 17 Feb 2025 17:34:10 +0100 Subject: [PATCH 002/109] Update README. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 435eb94..7115fe9 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ [![License](https://img.shields.io/badge/License-GPLv3-blue.svg)](https://www.gnu.org/licenses/gpl-3.0.html) [![NuGet](https://img.shields.io/nuget/v/OpenDDD.NET.svg)](https://www.nuget.org/packages/OpenDDD.NET/) -OpenDDD.NET is an open-source framework for domain-driven design (DDD) development using C# and .NET. It provides a set of powerful tools and abstractions to help developers build scalable, maintainable, and testable applications following the principles of DDD. +OpenDDD.NET is an open-source framework for domain-driven design (DDD) development using C# and ASP.NET Core. It provides a set of powerful tools and abstractions to help developers build scalable, maintainable, and testable applications following the principles of DDD. > **Note:** OpenDDD.NET is currently in a beta state as part of new major version 3. Use with caution in production environments. From f93eb240e8e2f02fc81bde88df40251ebb1c1325 Mon Sep 17 00:00:00 2001 From: David Runemalm Date: Mon, 17 Feb 2025 17:42:21 +0100 Subject: [PATCH 003/109] Update docs. --- docs/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.rst b/docs/index.rst index 8c6d64a..775c802 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -5,7 +5,7 @@ OpenDDD.NET =========== -OpenDDD.NET is an open-source framework for domain-driven design (DDD) development using C# and .NET. It provides a set of powerful tools and abstractions to help developers build scalable, maintainable, and testable applications following the principles of DDD. +OpenDDD.NET is an open-source framework for domain-driven design (DDD) development using C# and ASP.NET Core. It provides a set of powerful tools and abstractions to help developers build scalable, maintainable, and testable applications following the principles of DDD. Purpose ------- From 1c889dfc3ca03539c5fa998a56cc26312a1563c5 Mon Sep 17 00:00:00 2001 From: David Runemalm Date: Mon, 17 Feb 2025 18:25:50 +0100 Subject: [PATCH 004/109] Fix docs. --- docs/userguide.rst | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/docs/userguide.rst b/docs/userguide.rst index 5c90370..3203537 100644 --- a/docs/userguide.rst +++ b/docs/userguide.rst @@ -79,9 +79,6 @@ Add OpenDDD.NET services and middleware to your application in the `Program.cs` .. code-block:: csharp using OpenDDD.API.Extensions; - using YourProjectName.Domain.Model.Ports; - using YourProjectName.Infrastructure.Adapters.Console; - using YourProjectName.Infrastructure.Persistence.EfCore; var builder = WebApplication.CreateBuilder(args); @@ -466,8 +463,8 @@ Add the following configuration to your `appsettings.json` file to customize Ope "DatabaseProvider": "InMemory", "MessagingProvider": "InMemory", "Events": { - "DomainEventTopicTemplate": "YourProjectName.Domain.{EventName}", - "IntegrationEventTopicTemplate": "YourProjectName.Interchange.{EventName}", + "DomainEventTopic": "YourProjectName.Domain.{EventName}", + "IntegrationEventTopic": "YourProjectName.Interchange.{EventName}", "ListenerGroup": "Default" }, "SQLite": { @@ -476,11 +473,6 @@ Add the following configuration to your `appsettings.json` file to customize Ope "Postgres": { "ConnectionString": "Host=localhost;Port=5432;Database=yourprojectname;Username=your_username;Password=your_password" }, - "Events": { - "DomainEventTopicTemplate": "YourProjectName.Domain.{EventName}", - "IntegrationEventTopicTemplate": "YourProjectName.Interchange.{EventName}", - "ListenerGroup": "Default" - }, "AzureServiceBus": { "ConnectionString": "", "AutoCreateTopics": true From 092b0c9ab1b1818ac7edfa89b2cea6ef9e75ab9f Mon Sep 17 00:00:00 2001 From: David Runemalm Date: Mon, 17 Feb 2025 18:26:43 +0100 Subject: [PATCH 005/109] Fix docs. --- docs/configuration.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/configuration.rst b/docs/configuration.rst index 8144cde..826e8c4 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -30,8 +30,8 @@ An example configuration in `appsettings.json`: "DatabaseProvider": "InMemory", "MessagingProvider": "InMemory", "Events": { - "DomainEventTopicTemplate": "Bookstore.Domain.{EventName}", - "IntegrationEventTopicTemplate": "Bookstore.Interchange.{EventName}", + "DomainEventTopic": "Bookstore.Domain.{EventName}", + "IntegrationEventTopic": "Bookstore.Interchange.{EventName}", "ListenerGroup": "Default" }, "SQLite": { From 0cb543763b2abd2bc49cdc58c2c130856152f431 Mon Sep 17 00:00:00 2001 From: David Runemalm Date: Mon, 17 Feb 2025 18:30:54 +0100 Subject: [PATCH 006/109] Fix docs. --- docs/configuration.rst | 2 +- src/OpenDDD/API/Options/OpenDddEventsOptions.cs | 4 ++-- src/OpenDDD/API/Options/OpenDddOptions.cs | 4 ++-- src/OpenDDD/Domain/Model/Helpers/EventTopicHelper.cs | 8 ++++---- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/configuration.rst b/docs/configuration.rst index 826e8c4..8f96e99 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -79,7 +79,7 @@ Instead of using `appsettings.json`, OpenDDD.NET can be configured **dynamically { options.UseInMemoryDatabase() .UseInMemoryMessaging() - .SetEventTopicTemplates( + .SetEventTopics( "Bookstore.Domain.{EventName}", "Bookstore.Interchange.{EventName}" ) diff --git a/src/OpenDDD/API/Options/OpenDddEventsOptions.cs b/src/OpenDDD/API/Options/OpenDddEventsOptions.cs index 2db84cd..9627462 100644 --- a/src/OpenDDD/API/Options/OpenDddEventsOptions.cs +++ b/src/OpenDDD/API/Options/OpenDddEventsOptions.cs @@ -2,8 +2,8 @@ { public class OpenDddEventsOptions { - public string DomainEventTopicTemplate { get; set; } = "YourProjectName.Domain.{EventName}"; - public string IntegrationEventTopicTemplate { get; set; } = "YourProjectName.Interchange.{EventName}"; + public string DomainEventTopic { get; set; } = "YourProjectName.Domain.{EventName}"; + public string IntegrationEventTopic { get; set; } = "YourProjectName.Interchange.{EventName}"; public string ListenerGroup { get; set; } = "Default"; } } diff --git a/src/OpenDDD/API/Options/OpenDddOptions.cs b/src/OpenDDD/API/Options/OpenDddOptions.cs index 4a0cf96..d7bf397 100644 --- a/src/OpenDDD/API/Options/OpenDddOptions.cs +++ b/src/OpenDDD/API/Options/OpenDddOptions.cs @@ -103,8 +103,8 @@ public OpenDddOptions SetEventListenerGroup(string group) public OpenDddOptions SetEventTopics(string domainEventTemplate, string integrationEventTemplate) { - Events.DomainEventTopicTemplate = domainEventTemplate; - Events.IntegrationEventTopicTemplate = integrationEventTemplate; + Events.DomainEventTopic = domainEventTemplate; + Events.IntegrationEventTopic = integrationEventTemplate; return this; } diff --git a/src/OpenDDD/Domain/Model/Helpers/EventTopicHelper.cs b/src/OpenDDD/Domain/Model/Helpers/EventTopicHelper.cs index b2b3cca..c9c3b32 100644 --- a/src/OpenDDD/Domain/Model/Helpers/EventTopicHelper.cs +++ b/src/OpenDDD/Domain/Model/Helpers/EventTopicHelper.cs @@ -18,8 +18,8 @@ public static string DetermineTopic(Type eventClassType, OpenDddEventsOptions ev // Select the correct format from configuration string topicTemplate = isIntegrationEvent - ? eventOptions.IntegrationEventTopicTemplate - : eventOptions.DomainEventTopicTemplate; + ? eventOptions.IntegrationEventTopic + : eventOptions.DomainEventTopic; // Ensure the topic format contains "{EventName}" if (!topicTemplate.Contains("{EventName}")) @@ -53,8 +53,8 @@ public static string DetermineTopic(string eventType, string eventName, OpenDddE // Select the correct format from configuration string topicTemplate = eventType == "Integration" - ? eventOptions.IntegrationEventTopicTemplate - : eventOptions.DomainEventTopicTemplate; + ? eventOptions.IntegrationEventTopic + : eventOptions.DomainEventTopic; // Ensure the topic format contains "{EventName}" if (!topicTemplate.Contains("{EventName}")) From 4bb7407f9a500695033e00c32089ebb662b6e869 Mon Sep 17 00:00:00 2001 From: David Runemalm Date: Mon, 17 Feb 2025 18:31:55 +0100 Subject: [PATCH 007/109] Fix docs. --- docs/configuration.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuration.rst b/docs/configuration.rst index 8f96e99..ae94923 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -168,7 +168,7 @@ Event settings define how domain and integration events are published: .. code-block:: csharp - options.SetEventTopicTemplates( + options.SetEventTopics( "Bookstore.Domain.{EventName}", "Bookstore.Interchange.{EventName}" ) From 18a49b2f4199ab83744dc4a71ac07c179a5d06bd Mon Sep 17 00:00:00 2001 From: David Runemalm Date: Mon, 17 Feb 2025 18:50:11 +0100 Subject: [PATCH 008/109] Update README. --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 7115fe9..86d5de4 100644 --- a/README.md +++ b/README.md @@ -59,11 +59,11 @@ To get started with OpenDDD.NET, follow these simple steps: { options.UseInMemoryDatabase() .UseInMemoryMessaging() - .SetEventListenerGroup("Default") - .SetEventTopicTemplates( + .SetEventTopics( "Bookstore.Domain.{EventName}", "Bookstore.Interchange.{EventName}" ) + .SetEventListenerGroup("Default") .EnableAutoRegistration(); }); From fcd7005583405d7b32fb43c924e84533cf34f144 Mon Sep 17 00:00:00 2001 From: David Runemalm Date: Tue, 18 Feb 2025 10:42:02 +0100 Subject: [PATCH 009/109] Add missing 'Add Web API Adapter' step to user-guide. --- docs/building-blocks.rst | 2 - docs/userguide.rst | 121 ++++++++++++++++++++++++++++++++++++++- 2 files changed, 119 insertions(+), 4 deletions(-) diff --git a/docs/building-blocks.rst b/docs/building-blocks.rst index c2f0c5e..67564cb 100644 --- a/docs/building-blocks.rst +++ b/docs/building-blocks.rst @@ -184,12 +184,10 @@ Repositories are **auto-registered** with `IRepository`. If **Example: Default Auto-Registered Repositories** - `IRepository` → `PostgresOpenDddRepository` -- `IRepository` → `EfCoreRepository` **Example: Custom Auto-Registered Repositories** - `ICustomerRepository` → `PostgresOpenDddCustomerRepository` -- `ICustomerRepository` → `EfCoreCustomerRepository` **NOTE:** If you have more than one implementation of a repository the framework won't know which of them to auto-register. In this case you need to delete one of the implementations or disable auto-registration and register the implementation you want manually. diff --git a/docs/userguide.rst b/docs/userguide.rst index 3203537..12c56ec 100644 --- a/docs/userguide.rst +++ b/docs/userguide.rst @@ -450,8 +450,125 @@ Then register the port with the adapter class in `Program.cs` like this: // ... +---------------------- +7: Add Web API Adapter +---------------------- + +Create an http adapter for your application layer actions. We need to: + +- Create a **controller** to open endpoints and invoke actions. +- Add **Controller-**, **Swagger-** and **API Explorer** services in `Program.cs`. +- Add **HTTPS Redirection-**, **CORS-** and **Swagger** middleware in `Program.cs`. +- Map controllers to endpoints in `Program.cs`. + +Example definitions: + +.. code-block:: csharp + + using Microsoft.AspNetCore.Mvc; + using YourProjectName.Application.Actions.GetCustomer; + using YourProjectName.Application.Actions.GetCustomers; + using YourProjectName.Application.Actions.RegisterCustomer; + using YourProjectName.Domain.Model; + + namespace YourProjectName.Infrastructure.Adapters.WebAPI.Controllers + { + [ApiController] + [Route("api/customers")] + public class CustomerController : ControllerBase + { + private readonly RegisterCustomerAction _registerCustomerAction; + private readonly GetCustomerAction _getCustomerAction; + private readonly GetCustomersAction _getCustomersAction; + + public CustomerController( + RegisterCustomerAction registerCustomerAction, + GetCustomerAction getCustomerAction, + GetCustomersAction getCustomersAction) + { + _registerCustomerAction = registerCustomerAction; + _getCustomerAction = getCustomerAction; + _getCustomersAction = getCustomersAction; + } + + [HttpPost("register-customer")] + public async Task> RegisterCustomer([FromBody] RegisterCustomerCommand command, CancellationToken ct) + { + try + { + var customer = await _registerCustomerAction.ExecuteAsync(command, ct); + return CreatedAtAction(nameof(GetCustomer), new { id = customer.Id }, customer); + } + catch (Exception ex) + { + return BadRequest(new { Message = ex.Message }); + } + } + } + } + +.. code-block:: csharp + + using OpenDDD.API.Extensions; + using YourProjectName.Domain.Model.Ports; + using YourProjectName.Infrastructure.Adapters.Console; + + var builder = WebApplication.CreateBuilder(args); + + // Add Swagger Services + builder.Services.AddEndpointsApiExplorer(); + builder.Services.AddSwaggerGen(); + + // Add OpenDDD services + builder.Services.AddOpenDDD(builder.Configuration, + options => + { + options.UseInMemoryDatabase() + .UseInMemoryMessaging() + .SetEventListenerGroup("YourProjectName") + .SetEventTopics( + "YourProjectName.Domain.{EventName}", + "YourProjectName.Interchange.{EventName}" + ) + .EnableAutoRegistration(); + }, + services => + { + services.AddTransient(); + } + ); + + // Add Controller Services + builder.Services.AddControllers(); + + // Build the application + var app = builder.Build(); + + // Use OpenDDD Middleware + app.UseOpenDDD(); + + // Use Swagger Middleware + if (app.Environment.IsDevelopment()) + { + app.UseSwagger(); + app.UseSwaggerUI(); + app.UseDeveloperExceptionPage(); + } + + // Use HTTP->HTTPS Redirection Middleware + app.UseHttpsRedirection(); + + // Use CORS Middleware + app.UseCors("AllowAll"); + + // Map Controller Actions to Endpoints + app.MapControllers(); + + // Run the application + app.Run(); + -------------------------- -7: Edit `appsettings.json` +8: Edit `appsettings.json` -------------------------- Add the following configuration to your `appsettings.json` file to customize OpenDDD.NET behavior: @@ -501,7 +618,7 @@ Add the following configuration to your `appsettings.json` file to customize Ope For all information about configuration, see :ref:`Configuration `. ---------------------- -8: Run the Application +9: Run the Application ---------------------- Now you are ready to run the application: From 3559ab8b0578fca4d195fe3228dd969449f1be13 Mon Sep 17 00:00:00 2001 From: David Runemalm Date: Wed, 19 Feb 2025 10:10:08 +0100 Subject: [PATCH 010/109] Add tests for domain publisher. --- .../EfCore/EfCoreConfigurationTests.cs | 4 +- .../Events/DomainPublisherTests.cs | 72 +++++++++++++++++++ 2 files changed, 74 insertions(+), 2 deletions(-) create mode 100644 src/OpenDDD/Tests/Infrastructure/Events/DomainPublisherTests.cs diff --git a/samples/Bookstore/src/Bookstore/Tests/Infrastructure/Persistence/EfCore/EfCoreConfigurationTests.cs b/samples/Bookstore/src/Bookstore/Tests/Infrastructure/Persistence/EfCore/EfCoreConfigurationTests.cs index e3f2db6..fddcb62 100644 --- a/samples/Bookstore/src/Bookstore/Tests/Infrastructure/Persistence/EfCore/EfCoreConfigurationTests.cs +++ b/samples/Bookstore/src/Bookstore/Tests/Infrastructure/Persistence/EfCore/EfCoreConfigurationTests.cs @@ -9,11 +9,11 @@ using OpenDDD.Infrastructure.Repository.EfCore; using OpenDDD.Infrastructure.Persistence.DatabaseSession; using OpenDDD.Infrastructure.Persistence.EfCore.DatabaseSession; -using Bookstore.Domain.Model; -using Bookstore.Infrastructure.Persistence.EfCore; using OpenDDD.Infrastructure.Events; using OpenDDD.Infrastructure.TransactionalOutbox; using OpenDDD.Infrastructure.TransactionalOutbox.EfCore; +using Bookstore.Domain.Model; +using Bookstore.Infrastructure.Persistence.EfCore; namespace Bookstore.Tests.Infrastructure.Persistence.EfCore { diff --git a/src/OpenDDD/Tests/Infrastructure/Events/DomainPublisherTests.cs b/src/OpenDDD/Tests/Infrastructure/Events/DomainPublisherTests.cs new file mode 100644 index 0000000..ff9fbd5 --- /dev/null +++ b/src/OpenDDD/Tests/Infrastructure/Events/DomainPublisherTests.cs @@ -0,0 +1,72 @@ +using FluentAssertions; +using OpenDDD.Domain.Model; +using OpenDDD.Infrastructure.Events; +using Xunit; + +namespace OpenDDD.Tests.Infrastructure.Events +{ + public class DomainPublisherTests + { + private class TestEvent : IDomainEvent { } + + [Fact] + public async Task PublishAsync_ShouldStoreEvent_WhenValidEventIsPublished() + { + // Arrange + var publisher = new DomainPublisher(); + var domainEvent = new TestEvent(); + + // Act + await publisher.PublishAsync(domainEvent, CancellationToken.None); + + // Assert + publisher.GetPublishedEvents().Should().ContainSingle() + .Which.Should().Be(domainEvent); + } + + [Fact] + public async Task PublishAsync_ShouldThrowArgumentNullException_WhenEventIsNull() + { + // Arrange + var publisher = new DomainPublisher(); + + // Act + Func act = async () => await publisher.PublishAsync(null!, CancellationToken.None); + + // Assert + await act.Should().ThrowAsync() + .WithParameterName("domainEvent"); + } + + [Fact] + public async Task GetPublishedEvents_ShouldReturnEmptyList_WhenNoEventsArePublished() + { + // Arrange + var publisher = new DomainPublisher(); + + // Act + var events = publisher.GetPublishedEvents(); + + // Assert + events.Should().BeEmpty(); + } + + [Fact] + public async Task GetPublishedEvents_ShouldReturnAllPublishedEvents() + { + // Arrange + var publisher = new DomainPublisher(); + var event1 = new TestEvent(); + var event2 = new TestEvent(); + + // Act + await publisher.PublishAsync(event1, CancellationToken.None); + await publisher.PublishAsync(event2, CancellationToken.None); + var publishedEvents = publisher.GetPublishedEvents(); + + // Assert + publishedEvents.Should().HaveCount(2); + publishedEvents.Should().ContainInOrder(event1, event2); + } + } +} From 80795298caad0343ae46c0a6b5f70ad6f3afbfca Mon Sep 17 00:00:00 2001 From: David Runemalm Date: Wed, 19 Feb 2025 10:26:35 +0100 Subject: [PATCH 011/109] Add unit tests for integration publisher. --- src/OpenDDD/OpenDDD.csproj | 1 + .../Events/IntegrationPublisherTests.cs | 72 +++++++++++++++++++ 2 files changed, 73 insertions(+) create mode 100644 src/OpenDDD/Tests/Infrastructure/Events/IntegrationPublisherTests.cs diff --git a/src/OpenDDD/OpenDDD.csproj b/src/OpenDDD/OpenDDD.csproj index cf25118..b17d9ae 100644 --- a/src/OpenDDD/OpenDDD.csproj +++ b/src/OpenDDD/OpenDDD.csproj @@ -25,6 +25,7 @@ + diff --git a/src/OpenDDD/Tests/Infrastructure/Events/IntegrationPublisherTests.cs b/src/OpenDDD/Tests/Infrastructure/Events/IntegrationPublisherTests.cs new file mode 100644 index 0000000..3d750d4 --- /dev/null +++ b/src/OpenDDD/Tests/Infrastructure/Events/IntegrationPublisherTests.cs @@ -0,0 +1,72 @@ +using FluentAssertions; +using OpenDDD.Domain.Model; +using OpenDDD.Infrastructure.Events; +using Xunit; + +namespace OpenDDD.Tests.Infrastructure.Events +{ + public class IntegrationPublisherTests + { + private class TestIntegrationEvent : IIntegrationEvent { } + + [Fact] + public async Task PublishAsync_ShouldStoreEvent_WhenValidEventIsPublished() + { + // Arrange + var publisher = new IntegrationPublisher(); + var integrationEvent = new TestIntegrationEvent(); + + // Act + await publisher.PublishAsync(integrationEvent, CancellationToken.None); + + // Assert + publisher.GetPublishedEvents().Should().ContainSingle() + .Which.Should().Be(integrationEvent); + } + + [Fact] + public async Task PublishAsync_ShouldThrowArgumentNullException_WhenEventIsNull() + { + // Arrange + var publisher = new IntegrationPublisher(); + + // Act + Func act = async () => await publisher.PublishAsync(null!, CancellationToken.None); + + // Assert + await act.Should().ThrowAsync() + .WithParameterName("integrationEvent"); + } + + [Fact] + public async Task GetPublishedEvents_ShouldReturnEmptyList_WhenNoEventsArePublished() + { + // Arrange + var publisher = new IntegrationPublisher(); + + // Act + var events = publisher.GetPublishedEvents(); + + // Assert + events.Should().BeEmpty(); + } + + [Fact] + public async Task GetPublishedEvents_ShouldReturnAllPublishedEvents() + { + // Arrange + var publisher = new IntegrationPublisher(); + var event1 = new TestIntegrationEvent(); + var event2 = new TestIntegrationEvent(); + + // Act + await publisher.PublishAsync(event1, CancellationToken.None); + await publisher.PublishAsync(event2, CancellationToken.None); + var publishedEvents = publisher.GetPublishedEvents(); + + // Assert + publishedEvents.Should().HaveCount(2); + publishedEvents.Should().ContainInOrder(event1, event2); + } + } +} From 182b3b6b054231bb7885f3a350a738f060476fa9 Mon Sep 17 00:00:00 2001 From: David Runemalm Date: Wed, 19 Feb 2025 10:30:05 +0100 Subject: [PATCH 012/109] Add unit tests for InMemoryMessagingProvider. --- .../Events/InMemoryMessagingProviderTests.cs | 190 ++++++++++++++++++ 1 file changed, 190 insertions(+) create mode 100644 src/OpenDDD/Tests/Infrastructure/Events/InMemoryMessagingProviderTests.cs diff --git a/src/OpenDDD/Tests/Infrastructure/Events/InMemoryMessagingProviderTests.cs b/src/OpenDDD/Tests/Infrastructure/Events/InMemoryMessagingProviderTests.cs new file mode 100644 index 0000000..19c1da1 --- /dev/null +++ b/src/OpenDDD/Tests/Infrastructure/Events/InMemoryMessagingProviderTests.cs @@ -0,0 +1,190 @@ +using Microsoft.Extensions.Logging; +using Xunit; +using Moq; +using FluentAssertions; +using OpenDDD.Infrastructure.Events.InMemory; + +namespace OpenDDD.Tests.Infrastructure.Events +{ + public class InMemoryMessagingProviderTests + { + private readonly Mock> _mockLogger; + private readonly InMemoryMessagingProvider _messagingProvider; + + public InMemoryMessagingProviderTests() + { + _mockLogger = new Mock>(); + _messagingProvider = new InMemoryMessagingProvider(_mockLogger.Object); + } + + [Fact] + public async Task SubscribeAsync_ShouldStoreMessageHandler_ForGivenTopicAndConsumerGroup() + { + // Arrange + var topic = "TestTopic"; + var consumerGroup = "TestGroup"; + Func handler = async (message, ct) => await Task.CompletedTask; + + // Act + await _messagingProvider.SubscribeAsync(topic, consumerGroup, handler, CancellationToken.None); + + // Assert + _mockLogger.Verify( + log => log.Log( + LogLevel.Information, + It.IsAny(), + It.Is((v, t) => v.ToString().Contains($"Subscribed to topic: {topic} in listener group: {consumerGroup}")), + It.IsAny(), + It.IsAny>() + ), + Times.Once + ); + } + + [Fact] + public async Task PublishAsync_ShouldInvokeSubscribedHandler() + { + // Arrange + var topic = "TestTopic"; + var consumerGroup = "TestGroup"; + var receivedMessages = new List(); + + Func handler = async (message, ct) => + { + receivedMessages.Add(message); + await Task.CompletedTask; + }; + + await _messagingProvider.SubscribeAsync(topic, consumerGroup, handler, CancellationToken.None); + + // Act + await _messagingProvider.PublishAsync(topic, "Hello, World!", CancellationToken.None); + + // Allow some time for async handlers to execute + await Task.Delay(100); + + // Assert + receivedMessages.Should().ContainSingle() + .Which.Should().Be("Hello, World!"); + } + + [Fact] + public async Task PublishAsync_ShouldNotThrow_WhenNoSubscribersExist() + { + // Arrange + var topic = "NonExistentTopic"; + + // Act + Func act = async () => await _messagingProvider.PublishAsync(topic, "Message", CancellationToken.None); + + // Assert + await act.Should().NotThrowAsync(); + } + + [Fact] + public async Task PublishAsync_ShouldInvokeAllHandlers_ForMultipleSubscriptions() + { + // Arrange + var topic = "TestTopic"; + var consumerGroup1 = "Group1"; + var consumerGroup2 = "Group2"; + var receivedMessages1 = new List(); + var receivedMessages2 = new List(); + + Func handler1 = async (message, ct) => + { + receivedMessages1.Add(message); + await Task.CompletedTask; + }; + + Func handler2 = async (message, ct) => + { + receivedMessages2.Add(message); + await Task.CompletedTask; + }; + + await _messagingProvider.SubscribeAsync(topic, consumerGroup1, handler1, CancellationToken.None); + await _messagingProvider.SubscribeAsync(topic, consumerGroup2, handler2, CancellationToken.None); + + // Act + await _messagingProvider.PublishAsync(topic, "Event 1", CancellationToken.None); + + // Allow time for async handlers + await Task.Delay(100); + + // Assert + receivedMessages1.Should().ContainSingle().Which.Should().Be("Event 1"); + receivedMessages2.Should().ContainSingle().Which.Should().Be("Event 1"); + } + + [Fact] + public async Task PublishAsync_ShouldDeliverMessageToOnlyOneConsumer_InCompetingConsumerGroup() + { + // Arrange + var topic = "TestTopic"; + var consumerGroup = "CompetingGroup"; + var receivedMessages1 = new List(); + var receivedMessages2 = new List(); + + Func handler1 = async (message, ct) => + { + receivedMessages1.Add(message); + await Task.CompletedTask; + }; + + Func handler2 = async (message, ct) => + { + receivedMessages2.Add(message); + await Task.CompletedTask; + }; + + // Two consumers in the same group + await _messagingProvider.SubscribeAsync(topic, consumerGroup, handler1, CancellationToken.None); + await _messagingProvider.SubscribeAsync(topic, consumerGroup, handler2, CancellationToken.None); + + // Act + await _messagingProvider.PublishAsync(topic, "Competing Message", CancellationToken.None); + + // Allow time for async handlers + await Task.Delay(100); + + // Assert: Only one of the consumers should receive the message + var totalMessagesReceived = receivedMessages1.Count + receivedMessages2.Count; + totalMessagesReceived.Should().Be(1); + } + + [Fact] + public async Task PublishAsync_ShouldHandleException_WhenHandlerThrows() + { + // Arrange + var topic = "TestTopic"; + var consumerGroup = "TestGroup"; + + Func failingHandler = async (message, ct) => + { + await Task.CompletedTask; + throw new InvalidOperationException("Handler error"); + }; + + await _messagingProvider.SubscribeAsync(topic, consumerGroup, failingHandler, CancellationToken.None); + + // Act + await _messagingProvider.PublishAsync(topic, "Test Message", CancellationToken.None); + + // Allow some time for async handlers to execute + await Task.Delay(100); + + // Assert + _mockLogger.Verify( + log => log.Log( + LogLevel.Error, + It.IsAny(), + It.Is((v, t) => v.ToString().Contains($"Error in handler for topic '{topic}': Handler error")), + It.IsAny(), + It.IsAny>() + ), + Times.Once + ); + } + } +} From a604474ec76388b266e98934d34ce86db3f4181e Mon Sep 17 00:00:00 2001 From: David Runemalm Date: Wed, 19 Feb 2025 10:30:39 +0100 Subject: [PATCH 013/109] Fix competing consumer behaviour in InMemoryMessagingProviderTests after added test failed. --- .../InMemory/InMemoryMessagingProvider.cs | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/src/OpenDDD/Infrastructure/Events/InMemory/InMemoryMessagingProvider.cs b/src/OpenDDD/Infrastructure/Events/InMemory/InMemoryMessagingProvider.cs index 21a51d3..9ec830c 100644 --- a/src/OpenDDD/Infrastructure/Events/InMemory/InMemoryMessagingProvider.cs +++ b/src/OpenDDD/Infrastructure/Events/InMemory/InMemoryMessagingProvider.cs @@ -31,22 +31,22 @@ public Task PublishAsync(string topic, string message, CancellationToken ct) foreach (var groupKey in matchingGroups) { - if (_subscribers.TryGetValue(groupKey, out var handlers)) + if (_subscribers.TryGetValue(groupKey, out var handlers) && handlers.Any()) { - foreach (var handler in handlers) + // Select one handler at random to simulate competing consumers + var handler = handlers.OrderBy(_ => Guid.NewGuid()).First(); + + _ = Task.Run(async () => { - _ = Task.Run(async () => + try + { + await handler(message, ct); + } + catch (Exception ex) { - try - { - await handler(message, ct); - } - catch (Exception ex) - { - _logger.LogError(ex, $"Error in handler for topic '{topic}': {ex.Message}"); - } - }, ct); - } + _logger.LogError(ex, $"Error in handler for topic '{topic}': {ex.Message}"); + } + }, ct); } } From 455eaaf59679afadb619c3decc9faec60bbc5051 Mon Sep 17 00:00:00 2001 From: David Runemalm Date: Wed, 19 Feb 2025 12:29:11 +0100 Subject: [PATCH 014/109] Move test class. --- .../Events/{ => InMemory}/InMemoryMessagingProviderTests.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) rename src/OpenDDD/Tests/Infrastructure/Events/{ => InMemory}/InMemoryMessagingProviderTests.cs (99%) diff --git a/src/OpenDDD/Tests/Infrastructure/Events/InMemoryMessagingProviderTests.cs b/src/OpenDDD/Tests/Infrastructure/Events/InMemory/InMemoryMessagingProviderTests.cs similarity index 99% rename from src/OpenDDD/Tests/Infrastructure/Events/InMemoryMessagingProviderTests.cs rename to src/OpenDDD/Tests/Infrastructure/Events/InMemory/InMemoryMessagingProviderTests.cs index 19c1da1..bad7ad2 100644 --- a/src/OpenDDD/Tests/Infrastructure/Events/InMemoryMessagingProviderTests.cs +++ b/src/OpenDDD/Tests/Infrastructure/Events/InMemory/InMemoryMessagingProviderTests.cs @@ -1,10 +1,10 @@ using Microsoft.Extensions.Logging; -using Xunit; -using Moq; using FluentAssertions; +using Moq; +using Xunit; using OpenDDD.Infrastructure.Events.InMemory; -namespace OpenDDD.Tests.Infrastructure.Events +namespace OpenDDD.Tests.Infrastructure.Events.InMemory { public class InMemoryMessagingProviderTests { From adb8634970597ff7c26414031db3a427f2ee98c1 Mon Sep 17 00:00:00 2001 From: David Runemalm Date: Wed, 19 Feb 2025 12:42:06 +0100 Subject: [PATCH 015/109] Add azure target to sample project makefile. --- samples/Bookstore/Makefile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/samples/Bookstore/Makefile b/samples/Bookstore/Makefile index 4948b06..72bcc8d 100644 --- a/samples/Bookstore/Makefile +++ b/samples/Bookstore/Makefile @@ -155,6 +155,10 @@ apply-migrations: ##@Migrations Apply all pending migrations to the database # AZURE ########################################################################## +.PHONY: azure-create-resource-group +azure-create-resource-group: ##@Azure Create the Azure Resource Group + az group create --name $(AZURE_RESOURCE_GROUP) --location $(AZURE_REGION) + .PHONY: azure-create-servicebus-namespace azure-create-servicebus-namespace: ##@Azure Create the Azure Service Bus namespace az servicebus namespace create --name $(AZURE_SERVICEBUS_NAMESPACE) --resource-group $(AZURE_RESOURCE_GROUP) --location $(AZURE_REGION) --sku Standard From 9bb633f17245e123e8d2e52e2bef571a19dfbbae Mon Sep 17 00:00:00 2001 From: David Runemalm Date: Wed, 19 Feb 2025 12:43:34 +0100 Subject: [PATCH 016/109] Refactor azure service bus messaging provider to improve performance and adhere to dependency injection for testability. --- .../OpenDddServiceCollectionExtensions.cs | 37 +++++++++++++++- .../Azure/AzureServiceBusMessagingProvider.cs | 44 +++++++------------ 2 files changed, 53 insertions(+), 28 deletions(-) diff --git a/src/OpenDDD/API/Extensions/OpenDddServiceCollectionExtensions.cs b/src/OpenDDD/API/Extensions/OpenDddServiceCollectionExtensions.cs index 61fcdf1..08492fb 100644 --- a/src/OpenDDD/API/Extensions/OpenDddServiceCollectionExtensions.cs +++ b/src/OpenDDD/API/Extensions/OpenDddServiceCollectionExtensions.cs @@ -1,8 +1,11 @@ using System.Reflection; +using Azure.Messaging.ServiceBus; +using Azure.Messaging.ServiceBus.Administration; using Microsoft.AspNetCore.Hosting; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using Npgsql; using OpenDDD.API.Attributes; @@ -256,7 +259,39 @@ private static void AddInMemoryOpenDddPersistence(this IServiceCollection servic private static void AddAzureServiceBus(this IServiceCollection services) { - services.AddSingleton(); + services.AddSingleton(provider => + { + var options = provider.GetRequiredService>().Value; + var azureOptions = options.AzureServiceBus ?? throw new InvalidOperationException("Azure Service Bus options are missing."); + + if (string.IsNullOrWhiteSpace(azureOptions.ConnectionString)) + { + throw new InvalidOperationException("Azure Service Bus connection string is missing."); + } + + return new ServiceBusClient(azureOptions.ConnectionString); + }); + + services.AddSingleton(provider => + { + var options = provider.GetRequiredService>().Value; + var azureOptions = options.AzureServiceBus ?? throw new InvalidOperationException("Azure Service Bus options are missing."); + + return new ServiceBusAdministrationClient(azureOptions.ConnectionString); + }); + + services.AddSingleton(provider => + { + var options = provider.GetRequiredService>().Value; + var azureOptions = options.AzureServiceBus ?? throw new InvalidOperationException("Azure Service Bus options are missing."); + + return new AzureServiceBusMessagingProvider( + provider.GetRequiredService(), + provider.GetRequiredService(), + azureOptions.AutoCreateTopics, + provider.GetRequiredService>() + ); + }); } private static void AddRabbitMq(this IServiceCollection services) diff --git a/src/OpenDDD/Infrastructure/Events/Azure/AzureServiceBusMessagingProvider.cs b/src/OpenDDD/Infrastructure/Events/Azure/AzureServiceBusMessagingProvider.cs index 273638b..f679a79 100644 --- a/src/OpenDDD/Infrastructure/Events/Azure/AzureServiceBusMessagingProvider.cs +++ b/src/OpenDDD/Infrastructure/Events/Azure/AzureServiceBusMessagingProvider.cs @@ -1,45 +1,38 @@ -using Azure.Messaging.ServiceBus; +using Microsoft.Extensions.Logging; +using Azure.Messaging.ServiceBus; using Azure.Messaging.ServiceBus.Administration; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; -using OpenDDD.API.Options; -using OpenDDD.Infrastructure.Events.Azure.Options; namespace OpenDDD.Infrastructure.Events.Azure { public class AzureServiceBusMessagingProvider : IMessagingProvider { private readonly ServiceBusClient _client; - private readonly OpenDddAzureServiceBusOptions _options; + private readonly ServiceBusAdministrationClient _adminClient; + private readonly bool _autoCreateTopics; private readonly ILogger _logger; public AzureServiceBusMessagingProvider( - IOptions options, + ServiceBusClient client, + ServiceBusAdministrationClient adminClient, + bool autoCreateTopics, ILogger logger) { - var openDddOptions = options.Value ?? throw new ArgumentNullException(nameof(options)); - _options = openDddOptions.AzureServiceBus ?? throw new InvalidOperationException("AzureServiceBus settings are missing in OpenDddOptions."); - - if (string.IsNullOrWhiteSpace(_options.ConnectionString)) - { - throw new InvalidOperationException("Azure Service Bus connection string is missing."); - } - - _client = new ServiceBusClient(_options.ConnectionString); + _client = client ?? throw new ArgumentNullException(nameof(client)); + _adminClient = adminClient ?? throw new ArgumentNullException(nameof(adminClient)); + _autoCreateTopics = autoCreateTopics; _logger = logger ?? throw new ArgumentNullException(nameof(logger)); } public async Task SubscribeAsync(string topic, string consumerGroup, Func messageHandler, CancellationToken cancellationToken = default) { topic = topic.ToLower(); - var subscriptionName = consumerGroup; - if (_options.AutoCreateTopics) + if (_autoCreateTopics) { await CreateTopicIfNotExistsAsync(topic, cancellationToken); } - + await CreateSubscriptionIfNotExistsAsync(topic, subscriptionName, cancellationToken); var processor = _client.CreateProcessor(topic, subscriptionName); @@ -64,7 +57,7 @@ public async Task PublishAsync(string topic, string message, CancellationToken c { topic = topic.ToLower(); - if (_options.AutoCreateTopics) + if (_autoCreateTopics) { await CreateTopicIfNotExistsAsync(topic, cancellationToken); } @@ -76,23 +69,20 @@ public async Task PublishAsync(string topic, string message, CancellationToken c private async Task CreateTopicIfNotExistsAsync(string topic, CancellationToken cancellationToken) { - var adminClient = new ServiceBusAdministrationClient(_options.ConnectionString); topic = topic.ToLower(); - if (!await adminClient.TopicExistsAsync(topic, cancellationToken)) + if (!await _adminClient.TopicExistsAsync(topic, cancellationToken)) { - await adminClient.CreateTopicAsync(topic, cancellationToken); + await _adminClient.CreateTopicAsync(topic, cancellationToken); _logger.LogInformation("Created topic: {Topic}", topic); } } private async Task CreateSubscriptionIfNotExistsAsync(string topic, string subscriptionName, CancellationToken cancellationToken) { - var adminClient = new ServiceBusAdministrationClient(_options.ConnectionString); - - if (!await adminClient.SubscriptionExistsAsync(topic, subscriptionName, cancellationToken)) + if (!await _adminClient.SubscriptionExistsAsync(topic, subscriptionName, cancellationToken)) { - await adminClient.CreateSubscriptionAsync(topic, subscriptionName, cancellationToken); + await _adminClient.CreateSubscriptionAsync(topic, subscriptionName, cancellationToken); _logger.LogInformation("Created subscription: {Subscription} for topic: {Topic}", subscriptionName, topic); } } From 53ab1cf10787df952ea737dae968d413928ece5c Mon Sep 17 00:00:00 2001 From: David Runemalm Date: Wed, 19 Feb 2025 12:43:49 +0100 Subject: [PATCH 017/109] Add tests for AzureServiceBusMessagingProvider. --- .../AzureServiceBusMessagingProviderTests.cs | 111 ++++++++++++++++++ 1 file changed, 111 insertions(+) create mode 100644 src/OpenDDD/Tests/Infrastructure/Events/Azure/AzureServiceBusMessagingProviderTests.cs diff --git a/src/OpenDDD/Tests/Infrastructure/Events/Azure/AzureServiceBusMessagingProviderTests.cs b/src/OpenDDD/Tests/Infrastructure/Events/Azure/AzureServiceBusMessagingProviderTests.cs new file mode 100644 index 0000000..6ce78af --- /dev/null +++ b/src/OpenDDD/Tests/Infrastructure/Events/Azure/AzureServiceBusMessagingProviderTests.cs @@ -0,0 +1,111 @@ +using System.Text; +using Microsoft.Extensions.Logging; +using Xunit; +using Moq; +using Azure; +using Azure.Messaging.ServiceBus; +using Azure.Messaging.ServiceBus.Administration; +using OpenDDD.Infrastructure.Events.Azure; + +namespace OpenDDD.Tests.Infrastructure.Events.Azure +{ + public class AzureServiceBusMessagingProviderTests + { + private readonly Mock _mockClient; + private readonly Mock _mockAdminClient; + private readonly Mock _mockSender; + private readonly Mock _mockProcessor; + private readonly Mock> _mockLogger; + private readonly AzureServiceBusMessagingProvider _provider; + private readonly string _testTopic = "test-topic"; + private readonly string _testSubscription = "test-subscription"; + + public AzureServiceBusMessagingProviderTests() + { + _mockClient = new Mock(); + _mockAdminClient = new Mock(); + _mockSender = new Mock(); + _mockProcessor = new Mock(); + _mockLogger = new Mock>(); + + _mockClient + .Setup(client => client.CreateSender(It.IsAny())) + .Returns(_mockSender.Object); + + _mockClient + .Setup(client => client.CreateProcessor(It.IsAny(), It.IsAny())) + .Returns(_mockProcessor.Object); + + _mockAdminClient + .Setup(admin => admin.TopicExistsAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(Response.FromValue(true, Mock.Of())); + + _mockAdminClient + .Setup(admin => admin.SubscriptionExistsAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync(Response.FromValue(true, Mock.Of())); + + _provider = new AzureServiceBusMessagingProvider( + client: _mockClient.Object, + adminClient: _mockAdminClient.Object, + autoCreateTopics: true, + logger: _mockLogger.Object + ); + } + + [Fact] + public async Task SubscribeAsync_ShouldCreateTopicIfNotExists_WhenAutoCreateEnabled() + { + // Arrange: topic don't exist + _mockAdminClient.Setup(admin => admin.TopicExistsAsync(_testTopic, It.IsAny())) + .ReturnsAsync(Response.FromValue(false, Mock.Of())); + + // Act + await _provider.SubscribeAsync(_testTopic, _testSubscription, (msg, token) => Task.CompletedTask, CancellationToken.None); + + // Assert: create-topic was called + _mockAdminClient.Verify(admin => admin.CreateTopicAsync(_testTopic, It.IsAny()), Times.Once); + } + + [Fact] + public async Task SubscribeAsync_ShouldCreateSubscriptionIfNotExists() + { + // Arrange: subscription don't exist + _mockAdminClient.Setup(admin => admin.SubscriptionExistsAsync(_testTopic, _testSubscription, It.IsAny())) + .ReturnsAsync(Response.FromValue(false, Mock.Of())); + + // Act + await _provider.SubscribeAsync(_testTopic, _testSubscription, (msg, token) => Task.CompletedTask, CancellationToken.None); + + // Assert: create-subscription was called + _mockAdminClient.Verify(admin => admin.CreateSubscriptionAsync(_testTopic, _testSubscription, It.IsAny()), Times.Once); + } + + [Fact] + public async Task SubscribeAsync_ShouldStartProcessingMessages() + { + // Arrange: no-op handler + Func handler = (msg, token) => Task.CompletedTask; + + // Act + await _provider.SubscribeAsync(_testTopic, _testSubscription, handler, CancellationToken.None); + + // Assert: start-processing was called + _mockProcessor.Verify(processor => processor.StartProcessingAsync(It.IsAny()), Times.Once); + } + + [Fact] + public async Task PublishAsync_ShouldSendMessageToTopic() + { + // Arrange + var testMessage = "Hello, Azure Service Bus!"; + + // Act + await _provider.PublishAsync(_testTopic, testMessage, CancellationToken.None); + + // Assert: sender was called with the message + _mockSender.Verify(sender => sender.SendMessageAsync(It.Is( + msg => Encoding.UTF8.GetString(msg.Body.ToArray()) == testMessage), + It.IsAny()), Times.Once); + } + } +} From 6987ecae76744c3cf591add98744eab3f7ff06b8 Mon Sep 17 00:00:00 2001 From: David Runemalm Date: Wed, 19 Feb 2025 12:57:21 +0100 Subject: [PATCH 018/109] Dispose azure messaging provider resources properly on shutdown. --- .../Azure/AzureServiceBusMessagingProvider.cs | 23 ++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/src/OpenDDD/Infrastructure/Events/Azure/AzureServiceBusMessagingProvider.cs b/src/OpenDDD/Infrastructure/Events/Azure/AzureServiceBusMessagingProvider.cs index f679a79..dc4c191 100644 --- a/src/OpenDDD/Infrastructure/Events/Azure/AzureServiceBusMessagingProvider.cs +++ b/src/OpenDDD/Infrastructure/Events/Azure/AzureServiceBusMessagingProvider.cs @@ -4,12 +4,14 @@ namespace OpenDDD.Infrastructure.Events.Azure { - public class AzureServiceBusMessagingProvider : IMessagingProvider + public class AzureServiceBusMessagingProvider : IMessagingProvider, IAsyncDisposable { private readonly ServiceBusClient _client; private readonly ServiceBusAdministrationClient _adminClient; private readonly bool _autoCreateTopics; private readonly ILogger _logger; + private readonly List _processors = new(); + private bool _disposed; public AzureServiceBusMessagingProvider( ServiceBusClient client, @@ -36,6 +38,7 @@ public async Task SubscribeAsync(string topic, string consumerGroup, Func { @@ -86,5 +89,23 @@ private async Task CreateSubscriptionIfNotExistsAsync(string topic, string subsc _logger.LogInformation("Created subscription: {Subscription} for topic: {Topic}", subscriptionName, topic); } } + + public async ValueTask DisposeAsync() + { + if (_disposed) return; + _disposed = true; + + _logger.LogDebug("Disposing AzureServiceBusMessagingProvider..."); + + foreach (var processor in _processors) + { + _logger.LogDebug("Stopping message processor..."); + await processor.StopProcessingAsync(); + await processor.DisposeAsync(); + } + + _logger.LogDebug("Disposing ServiceBusClient..."); + await _client.DisposeAsync(); + } } } From 27efee66a3dfccdd4f9ee93e167d468ed4d8cdbd Mon Sep 17 00:00:00 2001 From: David Runemalm Date: Wed, 19 Feb 2025 18:07:36 +0100 Subject: [PATCH 019/109] Refactor kafka messaging provider to improve performance, adherance to dependency injection and testability. --- .../OpenDddServiceCollectionExtensions.cs | 21 +- .../Azure/AzureServiceBusMessagingProvider.cs | 7 +- .../Kafka/Factories/KafkaConsumerFactory.cs | 33 +++ .../Events/Kafka/KafkaMessagingProvider.cs | 198 ++++++++++-------- .../Kafka/Options/OpenDddKafkaOptions.cs | 1 + 5 files changed, 168 insertions(+), 92 deletions(-) create mode 100644 src/OpenDDD/Infrastructure/Events/Kafka/Factories/KafkaConsumerFactory.cs diff --git a/src/OpenDDD/API/Extensions/OpenDddServiceCollectionExtensions.cs b/src/OpenDDD/API/Extensions/OpenDddServiceCollectionExtensions.cs index 08492fb..48c775c 100644 --- a/src/OpenDDD/API/Extensions/OpenDddServiceCollectionExtensions.cs +++ b/src/OpenDDD/API/Extensions/OpenDddServiceCollectionExtensions.cs @@ -8,6 +8,7 @@ using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using Npgsql; +using Confluent.Kafka; using OpenDDD.API.Attributes; using OpenDDD.API.HostedServices; using OpenDDD.API.Options; @@ -21,6 +22,7 @@ using OpenDDD.Infrastructure.Events.Base; using OpenDDD.Infrastructure.Events.InMemory; using OpenDDD.Infrastructure.Events.Kafka; +using OpenDDD.Infrastructure.Events.Kafka.Factories; using OpenDDD.Infrastructure.Events.RabbitMq; using OpenDDD.Infrastructure.Persistence.DatabaseSession; using OpenDDD.Infrastructure.Persistence.EfCore.Base; @@ -301,7 +303,24 @@ private static void AddRabbitMq(this IServiceCollection services) private static void AddKafka(this IServiceCollection services) { - services.AddSingleton(); + services.AddSingleton(provider => + { + var options = provider.GetRequiredService>().Value; + var kafkaOptions = options.Kafka ?? throw new InvalidOperationException("Kafka options are missing."); + + if (string.IsNullOrWhiteSpace(kafkaOptions.BootstrapServers)) + throw new InvalidOperationException("Kafka bootstrap servers must be configured."); + + var logger = provider.GetRequiredService>(); + return new KafkaMessagingProvider( + kafkaOptions.BootstrapServers, + new AdminClientBuilder(new AdminClientConfig { BootstrapServers = kafkaOptions.BootstrapServers, ClientId = "OpenDDD" }).Build(), + new ProducerBuilder(new ProducerConfig { BootstrapServers = kafkaOptions.BootstrapServers, ClientId = "OpenDDD" }).Build(), + new KafkaConsumerFactory(kafkaOptions.BootstrapServers), + kafkaOptions.AutoCreateTopics, + logger + ); + }); } private static void AddInMemoryMessaging(this IServiceCollection services) diff --git a/src/OpenDDD/Infrastructure/Events/Azure/AzureServiceBusMessagingProvider.cs b/src/OpenDDD/Infrastructure/Events/Azure/AzureServiceBusMessagingProvider.cs index dc4c191..71d3cef 100644 --- a/src/OpenDDD/Infrastructure/Events/Azure/AzureServiceBusMessagingProvider.cs +++ b/src/OpenDDD/Infrastructure/Events/Azure/AzureServiceBusMessagingProvider.cs @@ -27,7 +27,6 @@ public AzureServiceBusMessagingProvider( public async Task SubscribeAsync(string topic, string consumerGroup, Func messageHandler, CancellationToken cancellationToken = default) { - topic = topic.ToLower(); var subscriptionName = consumerGroup; if (_autoCreateTopics) @@ -38,7 +37,7 @@ public async Task SubscribeAsync(string topic, string consumerGroup, Func { @@ -58,8 +57,6 @@ public async Task SubscribeAsync(string topic, string consumerGroup, Func Create(string consumerGroup) + { + var consumerConfig = new ConsumerConfig + { + BootstrapServers = _bootstrapServers, + ClientId = "OpenDDD", + GroupId = consumerGroup, + EnableAutoCommit = false, + AutoOffsetReset = AutoOffsetReset.Earliest + }; + + return new ConsumerBuilder(consumerConfig) + .SetValueDeserializer(Deserializers.Utf8) + .Build(); + } + } +} diff --git a/src/OpenDDD/Infrastructure/Events/Kafka/KafkaMessagingProvider.cs b/src/OpenDDD/Infrastructure/Events/Kafka/KafkaMessagingProvider.cs index 4d75264..a7cca1d 100644 --- a/src/OpenDDD/Infrastructure/Events/Kafka/KafkaMessagingProvider.cs +++ b/src/OpenDDD/Infrastructure/Events/Kafka/KafkaMessagingProvider.cs @@ -1,7 +1,6 @@ -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; -using OpenDDD.API.Options; -using OpenDDD.Infrastructure.Events.Kafka.Options; +using System.Collections.Concurrent; +using Microsoft.Extensions.Logging; +using OpenDDD.Infrastructure.Events.Kafka.Factories; using Confluent.Kafka; using Confluent.Kafka.Admin; @@ -9,118 +8,147 @@ namespace OpenDDD.Infrastructure.Events.Kafka { public class KafkaMessagingProvider : IMessagingProvider, IAsyncDisposable { + private readonly string _bootstrapServers; private readonly IProducer _producer; private readonly IAdminClient _adminClient; - private readonly OpenDddKafkaOptions _options; + private readonly bool _autoCreateTopics; + private readonly KafkaConsumerFactory _consumerFactory; private readonly ILogger _logger; - - public KafkaMessagingProvider(IOptions options, ILogger logger) + private readonly ConcurrentBag> _consumers = new(); + private readonly List _consumerTasks = new(); + private readonly CancellationTokenSource _cts = new(); + private bool _disposed; + + public KafkaMessagingProvider( + string bootstrapServers, + IAdminClient adminClient, + IProducer producer, + KafkaConsumerFactory consumerFactory, + bool autoCreateTopics, + ILogger logger) { - var openDddOptions = options?.Value ?? throw new ArgumentNullException(nameof(options)); - _options = openDddOptions.Kafka ?? throw new InvalidOperationException("Kafka settings are missing in OpenDddOptions."); - - if (string.IsNullOrWhiteSpace(_options.BootstrapServers)) - throw new InvalidOperationException("Kafka bootstrap servers must be configured."); - - var producerConfig = new ProducerConfig - { - BootstrapServers = _options.BootstrapServers, - ClientId = "OpenDDD" - }; - - var adminConfig = new AdminClientConfig + if (string.IsNullOrWhiteSpace(bootstrapServers)) + throw new ArgumentException("Kafka bootstrap servers must be configured.", nameof(bootstrapServers)); + + _bootstrapServers = bootstrapServers; + _adminClient = adminClient ?? throw new ArgumentNullException(nameof(adminClient)); + _producer = producer ?? throw new ArgumentNullException(nameof(producer)); + _consumerFactory = consumerFactory ?? throw new ArgumentNullException(nameof(consumerFactory)); + _autoCreateTopics = autoCreateTopics; + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task SubscribeAsync( + string topic, + string consumerGroup, + Func messageHandler, + CancellationToken cancellationToken) + { + if (_autoCreateTopics) { - BootstrapServers = _options.BootstrapServers, - ClientId = "OpenDDD" - }; + await CreateTopicIfNotExistsAsync(topic, cancellationToken); + } - _producer = new ProducerBuilder(producerConfig).Build(); - _adminClient = new AdminClientBuilder(adminConfig).Build(); + var consumer = _consumerFactory.Create(consumerGroup); + _consumers.Add(consumer); + consumer.Subscribe(topic); - _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _logger.LogDebug("Subscribed to Kafka topic '{Topic}' with consumer group '{ConsumerGroup}'", topic, consumerGroup); + + var consumerTask = Task.Run(() => StartConsumerLoop(consumer, messageHandler, _cts.Token), _cts.Token); + _consumerTasks.Add(consumerTask); } - - private async Task EnsureTopicExistsAsync(string topic, CancellationToken cancellationToken) + + private async Task StartConsumerLoop( + IConsumer consumer, + Func messageHandler, + CancellationToken cancellationToken) { try { - var metadata = _adminClient.GetMetadata(topic, TimeSpan.FromSeconds(5)); - if (metadata.Topics.Any(t => t.Topic == topic)) return; // Topic already exists - - _logger.LogInformation("Creating Kafka topic: {Topic}", topic); - await _adminClient.CreateTopicsAsync(new[] + while (!cancellationToken.IsCancellationRequested) { - new TopicSpecification { Name = topic, NumPartitions = 1, ReplicationFactor = 1 } - }); + // Seems like consume don't respect cancellation token. + // See: https://github.com/confluentinc/confluent-kafka-dotnet/issues/1085 + var result = consumer.Consume(cancellationToken); + if (result?.Message != null) + { + _logger.LogDebug("Received message from Kafka: {Message}", result.Message.Value); + await messageHandler(result.Message.Value, cancellationToken); + consumer.Commit(result); + _logger.LogDebug("Message processed and offset committed: {Offset}", result.Offset); + } + } + } + catch (OperationCanceledException) + { + _logger.LogDebug("Kafka consumer loop cancelled."); } catch (Exception ex) { - _logger.LogWarning("Could not check or create Kafka topic {Topic}: {Message}", topic, ex.Message); + _logger.LogError(ex, "Error occurred in Kafka consumer loop."); + } + finally + { + _logger.LogDebug("Closing consumer."); + consumer.Close(); } } - public async Task PublishAsync(string topic, string message, CancellationToken cancellationToken = default) + public async Task PublishAsync(string topic, string message, CancellationToken cancellationToken) { - await EnsureTopicExistsAsync(topic, cancellationToken); + if (_autoCreateTopics) + { + await CreateTopicIfNotExistsAsync(topic, cancellationToken); + } await _producer.ProduceAsync(topic, new Message { Value = message }, cancellationToken); - _logger.LogInformation("Published message to Kafka topic '{Topic}'", topic); + _logger.LogDebug("Published message to Kafka topic '{Topic}'", topic); } - - public async Task SubscribeAsync(string topic, string consumerGroup, Func messageHandler, CancellationToken cancellationToken = default) + + private async Task CreateTopicIfNotExistsAsync(string topic, CancellationToken cancellationToken) { - await EnsureTopicExistsAsync(topic, cancellationToken); + try + { + var metadata = _adminClient.GetMetadata(topic, TimeSpan.FromSeconds(5)); + if (metadata.Topics.Any(t => t.Topic == topic)) return; // Topic exists - var consumerConfig = new ConsumerConfig + _logger.LogDebug("Creating Kafka topic: {Topic}", topic); + await _adminClient.CreateTopicsAsync(new[] + { + new TopicSpecification { Name = topic, NumPartitions = 1, ReplicationFactor = 1 } + }, null); + } + catch (Exception ex) { - BootstrapServers = _options.BootstrapServers, - ClientId = "OpenDDD", - GroupId = consumerGroup, - EnableAutoCommit = false, - AutoOffsetReset = AutoOffsetReset.Earliest - }; + _logger.LogError("Could not check or create Kafka topic {Topic}: {Message}", topic, ex.Message); + } + } - var consumer = new ConsumerBuilder(consumerConfig) - .SetValueDeserializer(Deserializers.Utf8) - .Build(); + public async ValueTask DisposeAsync() + { + if (_disposed) return; + _disposed = true; - consumer.Subscribe(topic); + _logger.LogDebug("Disposing KafkaMessagingProvider..."); - _logger.LogInformation("Subscribed to Kafka topic '{Topic}' with consumer group '{ConsumerGroup}'", topic, consumerGroup); + _logger.LogDebug("Cancelling consumer tasks..."); + _cts.Cancel(); + + _logger.LogDebug("Waiting for all consumer tasks to complete..."); + await Task.WhenAll(_consumerTasks); - _ = Task.Run(async () => + foreach (var consumer in _consumers) { - try - { - while (!cancellationToken.IsCancellationRequested) - { - // Seems like consume don't respect cancellation token. - // See: https://github.com/confluentinc/confluent-kafka-dotnet/issues/1085 - var result = consumer.Consume(cancellationToken); - if (result.Message != null) - { - _logger.LogInformation("Received message from Kafka: {Message}", result.Message.Value); - await messageHandler(result.Message.Value, cancellationToken); - consumer.Commit(result); - _logger.LogInformation("Message processed and offset committed: {Offset}", result.Offset); - } - } - } - catch (OperationCanceledException) - { - // Handle cancellation gracefully - } - finally - { - consumer.Close(); - } - }, cancellationToken); - } + _logger.LogDebug("Disposing consumer..."); + consumer.Dispose(); + } - public async ValueTask DisposeAsync() - { - _producer?.Dispose(); - _adminClient?.Dispose(); + _logger.LogDebug("Disposing producer..."); + _producer.Dispose(); + + _logger.LogDebug("Disposing admin client..."); + _adminClient.Dispose(); } } } diff --git a/src/OpenDDD/Infrastructure/Events/Kafka/Options/OpenDddKafkaOptions.cs b/src/OpenDDD/Infrastructure/Events/Kafka/Options/OpenDddKafkaOptions.cs index 44dd974..fa6ba03 100644 --- a/src/OpenDDD/Infrastructure/Events/Kafka/Options/OpenDddKafkaOptions.cs +++ b/src/OpenDDD/Infrastructure/Events/Kafka/Options/OpenDddKafkaOptions.cs @@ -3,5 +3,6 @@ public class OpenDddKafkaOptions { public string BootstrapServers { get; set; } = string.Empty; + public bool AutoCreateTopics { get; set; } = true; } } From 0927dfcff3661e4b70af8e98a96cf7b8626174ab Mon Sep 17 00:00:00 2001 From: David Runemalm Date: Wed, 19 Feb 2025 18:07:51 +0100 Subject: [PATCH 020/109] Add KafkaMessagingProvider tests. --- .../Kafka/KafkaMessagingProviderTests.cs | 176 ++++++++++++++++++ 1 file changed, 176 insertions(+) create mode 100644 src/OpenDDD/Tests/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs diff --git a/src/OpenDDD/Tests/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs b/src/OpenDDD/Tests/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs new file mode 100644 index 0000000..0ccd0f1 --- /dev/null +++ b/src/OpenDDD/Tests/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs @@ -0,0 +1,176 @@ +using Microsoft.Extensions.Logging; +using Moq; +using Xunit; +using Confluent.Kafka; +using Confluent.Kafka.Admin; +using OpenDDD.Infrastructure.Events.Kafka; +using OpenDDD.Infrastructure.Events.Kafka.Factories; + +namespace OpenDDD.Tests.Infrastructure.Events.Kafka +{ + public class KafkaMessagingProviderTests + { + private readonly Mock> _mockProducer; + private readonly Mock _mockAdminClient; + private readonly Mock _mockConsumerFactory; + private readonly Mock> _mockConsumer; + private readonly Mock> _mockLogger; + private readonly KafkaMessagingProvider _provider; + private const string BootstrapServers = "localhost:9092"; + private const string Topic = "test-topic"; + private const string Message = "Hello, Kafka!"; + private const string ConsumerGroup = "test-group"; + + public KafkaMessagingProviderTests() + { + _mockProducer = new Mock>(); + _mockAdminClient = new Mock(); + _mockConsumerFactory = new Mock(BootstrapServers); + _mockConsumer = new Mock>(); + _mockLogger = new Mock>(); + + _provider = new KafkaMessagingProvider( + BootstrapServers, + _mockAdminClient.Object, + _mockProducer.Object, + _mockConsumerFactory.Object, + autoCreateTopics: true, + _mockLogger.Object); + + // Mock factory to always return same consumer + _mockConsumerFactory + .Setup(f => f.Create(It.IsAny())) + .Returns(_mockConsumer.Object); + + // Mock metadata retrieval for topics + var metadata = new Metadata( + new List(), + new List(), + -1, + "" + ); + + _mockAdminClient + .Setup(a => a.GetMetadata(Topic, It.IsAny())) + .Returns(metadata); + } + + [Fact] + public void Constructor_ShouldThrowException_WhenBootstrapServersIsNullOrEmpty() + { + Assert.Throws(() => new KafkaMessagingProvider( + "", + _mockAdminClient.Object, + _mockProducer.Object, + _mockConsumerFactory.Object, + true, + _mockLogger.Object)); + } + + [Fact] + public async Task PublishAsync_ShouldCall_ProduceAsync() + { + // Arrange + var mockDeliveryResult = new DeliveryResult(); + _mockProducer + .Setup(p => p.ProduceAsync(It.IsAny(), It.IsAny>(), It.IsAny())) + .ReturnsAsync(mockDeliveryResult); + + // Act + await _provider.PublishAsync(Topic, Message, CancellationToken.None); + + // Assert + _mockProducer.Verify(p => p.ProduceAsync( + Topic, + It.Is>(m => m.Value == Message), + It.IsAny()), + Times.Once); + } + + [Fact] + public async Task PublishAsync_ShouldCall_CreateTopicIfNotExists_WhenAutoCreateTopicsIsEnabled() + { + // Act + await _provider.PublishAsync(Topic, Message, CancellationToken.None); + + // Assert + _mockAdminClient.Verify(a => a.CreateTopicsAsync(It.IsAny>(), null), Times.Once); + } + + [Fact] + public async Task SubscribeAsync_ShouldCreateConsumer_AndSubscribeToTopic() + { + // Act + await _provider.SubscribeAsync(Topic, ConsumerGroup, async (_, _) => await Task.CompletedTask, CancellationToken.None); + + // Assert + _mockConsumer.Verify(c => c.Subscribe(Topic), Times.Once); + } + + [Fact] + public async Task SubscribeAsync_ShouldProcessReceivedMessages() + { + // Arrange + _mockConsumer + .SetupSequence(c => c.Consume(It.IsAny())) + .Returns(new ConsumeResult + { + Message = new Message { Value = Message } + }) + .Throws(new OperationCanceledException()); // Stop after the first message + + var messageReceived = new TaskCompletionSource(); + + // Act: Start the subscription + await _provider.SubscribeAsync(Topic, ConsumerGroup, async (msg, _) => + { + if (msg == Message) messageReceived.SetResult(true); + await Task.CompletedTask; + }, CancellationToken.None); + + // Ensure the background task has enough time to consume the message + await Task.Delay(100); + + // Assert: Check if the message was received + Assert.True(await messageReceived.Task.WaitAsync(TimeSpan.FromSeconds(2)), "Message handler was not called."); + + // Ensure commit is called after processing + _mockConsumer.Verify(c => c.Commit(It.IsAny>()), Times.Once); + } + + [Fact] + public async Task SubscribeAsync_ShouldHandleConsumerExceptionsGracefully() + { + // Arrange + _mockConsumer.Setup(c => c.Consume(It.IsAny())) + .Throws(new KafkaException(ErrorCode.Local_Transport)); + + // Act + await _provider.SubscribeAsync(Topic, ConsumerGroup, async (_, _) => await Task.CompletedTask, CancellationToken.None); + + // Assert + _mockLogger.Verify( + l => l.Log(LogLevel.Error, It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>()), + Times.Once); + } + + [Fact] + public async Task DisposeAsync_ShouldDisposeAllConsumers_AndKafkaClients() + { + // Arrange + await _provider.SubscribeAsync(Topic, ConsumerGroup, async (_, _) => await Task.CompletedTask, CancellationToken.None); + + // Ensure the background task has enough time to start + await Task.Delay(100); + + // Act + await _provider.DisposeAsync(); + + // Assert + _mockConsumer.Verify(c => c.Close(), Times.Once); + _mockConsumer.Verify(c => c.Dispose(), Times.Once); + _mockProducer.Verify(p => p.Dispose(), Times.Once); + _mockAdminClient.Verify(a => a.Dispose(), Times.Once); + } + } +} From aaf20d0c763eee59f915a3e97326f5420738860e Mon Sep 17 00:00:00 2001 From: David Runemalm Date: Wed, 19 Feb 2025 18:08:19 +0100 Subject: [PATCH 021/109] Update kafka targets in makefile of sample project. --- samples/Bookstore/Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/samples/Bookstore/Makefile b/samples/Bookstore/Makefile index 72bcc8d..9809c5c 100644 --- a/samples/Bookstore/Makefile +++ b/samples/Bookstore/Makefile @@ -245,7 +245,7 @@ endif .PHONY: kafka-list-brokers kafka-list-brokers: ##@Kafka List Kafka broker configurations - docker exec -it $(KAFKA_CONTAINER) /opt/kafka/bin/kafka-configs.sh --bootstrap-server localhost:9092 --describe --entity-type brokers + @docker exec -it $(KAFKA_CONTAINER) /opt/kafka/bin/kafka-configs.sh --bootstrap-server localhost:9092 --describe --entity-type brokers .PHONY: kafka-list-topics kafka-list-topics: ##@Kafka List all Kafka topics @@ -253,11 +253,11 @@ kafka-list-topics: ##@Kafka List all Kafka topics .PHONY: kafka-broker-status kafka-broker-status: ##@Kafka Show Kafka broker status - docker exec -it bookstore-kafka /opt/kafka/bin/kafka-broker-api-versions.sh --bootstrap-server localhost:9092 + @docker exec -it bookstore-kafka /opt/kafka/bin/kafka-broker-api-versions.sh --bootstrap-server localhost:9092 .PHONY: kafka-list-consumers kafka-list-consumers: ##@Kafka List active Kafka consumer groups - docker exec -it bookstore-kafka /opt/kafka/bin/kafka-consumer-groups.sh --bootstrap-server localhost:9092 --list + @docker exec -it bookstore-kafka /opt/kafka/bin/kafka-consumer-groups.sh --bootstrap-server localhost:9092 --list .PHONY: kafka-consume kafka-consume: ##@Kafka Consume messages from a Kafka topic (uses NAME) From f0fc895c41699d5d6dca1318c7de92a9e42aa72b Mon Sep 17 00:00:00 2001 From: David Runemalm Date: Wed, 19 Feb 2025 18:08:59 +0100 Subject: [PATCH 022/109] Add new kafka setting to appsettings.json of sample project. --- samples/Bookstore/src/Bookstore/appsettings.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/samples/Bookstore/src/Bookstore/appsettings.json b/samples/Bookstore/src/Bookstore/appsettings.json index cf4c806..4a43318 100644 --- a/samples/Bookstore/src/Bookstore/appsettings.json +++ b/samples/Bookstore/src/Bookstore/appsettings.json @@ -35,7 +35,8 @@ "VirtualHost": "/" }, "Kafka": { - "BootstrapServers": "localhost:9092" + "BootstrapServers": "localhost:9092", + "AutoCreateTopics": true }, "AutoRegister": { "Actions": true, From f66e5ea0a54394e02082b2c23d0d052a442407bb Mon Sep 17 00:00:00 2001 From: David Runemalm Date: Wed, 19 Feb 2025 18:12:50 +0100 Subject: [PATCH 023/109] Add new AutoCreateTopics to config in docs. --- docs/configuration.rst | 3 ++- docs/userguide.rst | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/configuration.rst b/docs/configuration.rst index ae94923..dd4072b 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -52,7 +52,8 @@ An example configuration in `appsettings.json`: "VirtualHost": "/" }, "Kafka": { - "BootstrapServers": "localhost:9092" + "BootstrapServers": "localhost:9092", + "AutoCreateTopics": true }, "AutoRegister": { "Actions": true, diff --git a/docs/userguide.rst b/docs/userguide.rst index 12c56ec..9c23a1d 100644 --- a/docs/userguide.rst +++ b/docs/userguide.rst @@ -602,7 +602,8 @@ Add the following configuration to your `appsettings.json` file to customize Ope "VirtualHost": "/" }, "Kafka": { - "BootstrapServers": "localhost:9092" + "BootstrapServers": "localhost:9092", + "AutoCreateTopics": true }, "AutoRegister": { "Actions": true, From 89a1125958cdba156ca595cb75dc8d80f5749bfe Mon Sep 17 00:00:00 2001 From: David Runemalm Date: Thu, 20 Feb 2025 11:04:50 +0100 Subject: [PATCH 024/109] Refactor rabbitmq provider a bit. --- .../OpenDddServiceCollectionExtensions.cs | 28 +++++- .../Factories/IRabbitMqConsumerFactory.cs | 9 ++ .../Factories/RabbitMqConsumerFactory.cs | 23 +++++ .../RabbitMq/RabbitMqCustomAsyncConsumer.cs | 6 +- .../RabbitMq/RabbitMqMessagingProvider.cs | 96 ++++++++++--------- 5 files changed, 115 insertions(+), 47 deletions(-) create mode 100644 src/OpenDDD/Infrastructure/Events/RabbitMq/Factories/IRabbitMqConsumerFactory.cs create mode 100644 src/OpenDDD/Infrastructure/Events/RabbitMq/Factories/RabbitMqConsumerFactory.cs diff --git a/src/OpenDDD/API/Extensions/OpenDddServiceCollectionExtensions.cs b/src/OpenDDD/API/Extensions/OpenDddServiceCollectionExtensions.cs index 48c775c..f4dd682 100644 --- a/src/OpenDDD/API/Extensions/OpenDddServiceCollectionExtensions.cs +++ b/src/OpenDDD/API/Extensions/OpenDddServiceCollectionExtensions.cs @@ -9,6 +9,7 @@ using Microsoft.Extensions.Options; using Npgsql; using Confluent.Kafka; +using RabbitMQ.Client; using OpenDDD.API.Attributes; using OpenDDD.API.HostedServices; using OpenDDD.API.Options; @@ -24,6 +25,7 @@ using OpenDDD.Infrastructure.Events.Kafka; using OpenDDD.Infrastructure.Events.Kafka.Factories; using OpenDDD.Infrastructure.Events.RabbitMq; +using OpenDDD.Infrastructure.Events.RabbitMq.Factories; using OpenDDD.Infrastructure.Persistence.DatabaseSession; using OpenDDD.Infrastructure.Persistence.EfCore.Base; using OpenDDD.Infrastructure.Persistence.EfCore.DatabaseSession; @@ -298,7 +300,31 @@ private static void AddAzureServiceBus(this IServiceCollection services) private static void AddRabbitMq(this IServiceCollection services) { - services.AddSingleton(); + services.AddSingleton(provider => + { + var options = provider.GetRequiredService>().Value; + var rabbitMqOptions = options.RabbitMq ?? throw new InvalidOperationException("RabbitMQ options are missing."); + + if (string.IsNullOrWhiteSpace(rabbitMqOptions.HostName)) + { + throw new InvalidOperationException("RabbitMQ host is missing."); + } + + var connectionFactory = new ConnectionFactory + { + HostName = rabbitMqOptions.HostName, + Port = rabbitMqOptions.Port, + UserName = rabbitMqOptions.Username, + Password = rabbitMqOptions.Password, + VirtualHost = rabbitMqOptions.VirtualHost + }; + + var logger = provider.GetRequiredService>(); + + var consumerFactory = new RabbitMqConsumerFactory(logger); + + return new RabbitMqMessagingProvider(connectionFactory, consumerFactory, logger); + }); } private static void AddKafka(this IServiceCollection services) diff --git a/src/OpenDDD/Infrastructure/Events/RabbitMq/Factories/IRabbitMqConsumerFactory.cs b/src/OpenDDD/Infrastructure/Events/RabbitMq/Factories/IRabbitMqConsumerFactory.cs new file mode 100644 index 0000000..2cb754d --- /dev/null +++ b/src/OpenDDD/Infrastructure/Events/RabbitMq/Factories/IRabbitMqConsumerFactory.cs @@ -0,0 +1,9 @@ +using RabbitMQ.Client; + +namespace OpenDDD.Infrastructure.Events.RabbitMq.Factories +{ + public interface IRabbitMqConsumerFactory + { + RabbitMqCustomAsyncConsumer CreateConsumer(IChannel channel, Func messageHandler); + } +} diff --git a/src/OpenDDD/Infrastructure/Events/RabbitMq/Factories/RabbitMqConsumerFactory.cs b/src/OpenDDD/Infrastructure/Events/RabbitMq/Factories/RabbitMqConsumerFactory.cs new file mode 100644 index 0000000..92b9d2f --- /dev/null +++ b/src/OpenDDD/Infrastructure/Events/RabbitMq/Factories/RabbitMqConsumerFactory.cs @@ -0,0 +1,23 @@ +using Microsoft.Extensions.Logging; +using RabbitMQ.Client; + +namespace OpenDDD.Infrastructure.Events.RabbitMq.Factories +{ + public class RabbitMqConsumerFactory : IRabbitMqConsumerFactory + { + private readonly ILogger _logger; + + public RabbitMqConsumerFactory(ILogger logger) + { + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public RabbitMqCustomAsyncConsumer CreateConsumer(IChannel channel, Func messageHandler) + { + if (channel == null) throw new ArgumentNullException(nameof(channel)); + if (messageHandler == null) throw new ArgumentNullException(nameof(messageHandler)); + + return new RabbitMqCustomAsyncConsumer(channel, messageHandler, _logger); + } + } +} diff --git a/src/OpenDDD/Infrastructure/Events/RabbitMq/RabbitMqCustomAsyncConsumer.cs b/src/OpenDDD/Infrastructure/Events/RabbitMq/RabbitMqCustomAsyncConsumer.cs index 5a766f5..13e33d4 100644 --- a/src/OpenDDD/Infrastructure/Events/RabbitMq/RabbitMqCustomAsyncConsumer.cs +++ b/src/OpenDDD/Infrastructure/Events/RabbitMq/RabbitMqCustomAsyncConsumer.cs @@ -9,8 +9,12 @@ public class RabbitMqCustomAsyncConsumer : IAsyncBasicConsumer { private readonly Func _messageHandler; private readonly ILogger _logger; + private bool _disposed; - public RabbitMqCustomAsyncConsumer(IChannel channel, Func messageHandler, ILogger logger) + public RabbitMqCustomAsyncConsumer( + IChannel channel, + Func messageHandler, + ILogger logger) { Channel = channel ?? throw new ArgumentNullException(nameof(channel)); _messageHandler = messageHandler ?? throw new ArgumentNullException(nameof(messageHandler)); diff --git a/src/OpenDDD/Infrastructure/Events/RabbitMq/RabbitMqMessagingProvider.cs b/src/OpenDDD/Infrastructure/Events/RabbitMq/RabbitMqMessagingProvider.cs index a3b8876..83d99a8 100644 --- a/src/OpenDDD/Infrastructure/Events/RabbitMq/RabbitMqMessagingProvider.cs +++ b/src/OpenDDD/Infrastructure/Events/RabbitMq/RabbitMqMessagingProvider.cs @@ -1,95 +1,101 @@ -using System.Text; +using System.Collections.Concurrent; +using System.Text; using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; -using OpenDDD.API.Options; -using OpenDDD.Infrastructure.Events.RabbitMq.Options; +using OpenDDD.Infrastructure.Events.RabbitMq.Factories; using RabbitMQ.Client; namespace OpenDDD.Infrastructure.Events.RabbitMq { public class RabbitMqMessagingProvider : IMessagingProvider, IAsyncDisposable { - private readonly ConnectionFactory _factory; + private readonly IConnectionFactory _connectionFactory; + private readonly IRabbitMqConsumerFactory _consumerFactory; + private readonly ILogger _logger; private IConnection? _connection; private IChannel? _channel; - private readonly OpenDddRabbitMqOptions _options; - private readonly ILogger _logger; - + private readonly ConcurrentBag _consumers = new(); + public RabbitMqMessagingProvider( - IOptions options, + IConnectionFactory factory, + IRabbitMqConsumerFactory consumerFactory, ILogger logger) { - var openDddOptions = options?.Value ?? throw new ArgumentNullException(nameof(options)); - _options = openDddOptions.RabbitMq ?? throw new InvalidOperationException("RabbitMQ settings are missing in OpenDddOptions."); - - if (string.IsNullOrWhiteSpace(_options.HostName)) - { - throw new InvalidOperationException("RabbitMQ host is missing."); - } - - _factory = new ConnectionFactory - { - HostName = _options.HostName, - Port = _options.Port, - UserName = _options.Username, - Password = _options.Password, - VirtualHost = _options.VirtualHost - }; - + _connectionFactory = factory ?? throw new ArgumentNullException(nameof(factory)); + _consumerFactory = consumerFactory ?? throw new ArgumentNullException(nameof(consumerFactory)); _logger = logger ?? throw new ArgumentNullException(nameof(logger)); } - private async Task EnsureConnectedAsync(CancellationToken cancellationToken) + public async Task SubscribeAsync(string topic, string consumerGroup, Func messageHandler, CancellationToken cancellationToken = default) { - if (_connection is { IsOpen: true } && _channel is { IsOpen: true }) return; + if (string.IsNullOrWhiteSpace(topic)) + throw new ArgumentException("Topic cannot be null or empty.", nameof(topic)); - _connection = await _factory.CreateConnectionAsync(cancellationToken); - _channel = await _connection.CreateChannelAsync(null, cancellationToken); - } + if (string.IsNullOrWhiteSpace(consumerGroup)) + throw new ArgumentException("Consumer group cannot be null or empty.", nameof(consumerGroup)); + + if (messageHandler == null) + throw new ArgumentNullException(nameof(messageHandler), "Message handler cannot be null."); - public async Task PublishAsync(string topic, string message, CancellationToken cancellationToken = default) - { await EnsureConnectedAsync(cancellationToken); if (_channel is null) throw new InvalidOperationException("RabbitMQ channel is not available."); await _channel.ExchangeDeclareAsync(topic, ExchangeType.Fanout, durable: true, autoDelete: false, cancellationToken: cancellationToken); + var queueName = $"{consumerGroup}.{topic}"; + await _channel.QueueDeclareAsync(queueName, durable: true, exclusive: false, autoDelete: false, cancellationToken: cancellationToken); + await _channel.QueueBindAsync(queueName, topic, "", cancellationToken: cancellationToken); - var body = Encoding.UTF8.GetBytes(message); - await _channel.BasicPublishAsync(topic, "", body, cancellationToken: cancellationToken); + var consumer = _consumerFactory.CreateConsumer(_channel, messageHandler); + _consumers.Add(consumer); + await _channel.BasicConsumeAsync(queueName, autoAck: false, consumer, cancellationToken); - _logger.LogInformation("Published message to topic '{Topic}'", topic); + _logger.LogDebug("Subscribed to RabbitMQ topic '{Topic}' with consumer group '{ConsumerGroup}'", topic, consumerGroup); } - public async Task SubscribeAsync(string topic, string consumerGroup, Func messageHandler, CancellationToken cancellationToken = default) + public async Task PublishAsync(string topic, string message, CancellationToken cancellationToken = default) { + if (string.IsNullOrWhiteSpace(topic)) + throw new ArgumentException("Topic cannot be null or empty.", nameof(topic)); + + if (string.IsNullOrWhiteSpace(message)) + throw new ArgumentException("Message cannot be null or empty.", nameof(message)); + await EnsureConnectedAsync(cancellationToken); if (_channel is null) throw new InvalidOperationException("RabbitMQ channel is not available."); await _channel.ExchangeDeclareAsync(topic, ExchangeType.Fanout, durable: true, autoDelete: false, cancellationToken: cancellationToken); - var queueName = $"{consumerGroup}.{topic}"; - await _channel.QueueDeclareAsync(queueName, durable: true, exclusive: false, autoDelete: false, cancellationToken: cancellationToken); - await _channel.QueueBindAsync(queueName, topic, "", cancellationToken: cancellationToken); - var consumer = new RabbitMqCustomAsyncConsumer(_channel, messageHandler, _logger); - await _channel.BasicConsumeAsync(queueName, autoAck: false, consumer, cancellationToken); + var body = Encoding.UTF8.GetBytes(message); + await _channel.BasicPublishAsync(topic, "", body, cancellationToken: cancellationToken); + + _logger.LogDebug("Published message to topic '{Topic}'", topic); + } + + private async Task EnsureConnectedAsync(CancellationToken cancellationToken) + { + if (_connection is { IsOpen: true } && _channel is { IsOpen: true }) return; - _logger.LogInformation("Subscribed to RabbitMQ topic '{Topic}' with consumer group '{ConsumerGroup}'", topic, consumerGroup); + _connection = await _connectionFactory.CreateConnectionAsync(cancellationToken); + _channel = await _connection.CreateChannelAsync(null, cancellationToken); } public async ValueTask DisposeAsync() { + _logger.LogDebug("Disposing RabbitMqMessagingProvider..."); + if (_channel is not null) { + _logger.LogDebug("Disposing RabbitMQ channel..."); await _channel.CloseAsync(); - _channel.Dispose(); + await _channel.DisposeAsync(); } if (_connection is not null) { + _logger.LogDebug("Disposing RabbitMQ connection..."); await _connection.CloseAsync(); - _connection.Dispose(); + await _connection.DisposeAsync(); } } } From 72b47ff4eba4023460fa3a58191f5b67d5255e32 Mon Sep 17 00:00:00 2001 From: David Runemalm Date: Thu, 20 Feb 2025 11:05:02 +0100 Subject: [PATCH 025/109] Add rabbitmq provider tests. --- .../RabbitMqMessagingProviderTests.cs | 96 +++++++++++++++++++ 1 file changed, 96 insertions(+) create mode 100644 src/OpenDDD/Tests/Infrastructure/Events/RabbitMq/RabbitMqMessagingProviderTests.cs diff --git a/src/OpenDDD/Tests/Infrastructure/Events/RabbitMq/RabbitMqMessagingProviderTests.cs b/src/OpenDDD/Tests/Infrastructure/Events/RabbitMq/RabbitMqMessagingProviderTests.cs new file mode 100644 index 0000000..31f23b8 --- /dev/null +++ b/src/OpenDDD/Tests/Infrastructure/Events/RabbitMq/RabbitMqMessagingProviderTests.cs @@ -0,0 +1,96 @@ +using Microsoft.Extensions.Logging; +using Xunit; +using Moq; +using OpenDDD.Infrastructure.Events.RabbitMq; +using OpenDDD.Infrastructure.Events.RabbitMq.Factories; +using RabbitMQ.Client; + +namespace OpenDDD.Tests.Infrastructure.Events.RabbitMq +{ + public class RabbitMqMessagingProviderTests + { + private readonly Mock _mockConnectionFactory; + private readonly Mock _mockConsumerFactory; + private readonly Mock> _mockLogger; + private readonly RabbitMqMessagingProvider _provider; + + private const string TestTopic = "test-topic"; + private const string TestConsumerGroup = "test-group"; + + public RabbitMqMessagingProviderTests() + { + _mockConnectionFactory = new Mock(); + _mockConsumerFactory = new Mock(); + _mockLogger = new Mock>(); + + _provider = new RabbitMqMessagingProvider( + _mockConnectionFactory.Object, + _mockConsumerFactory.Object, + _mockLogger.Object + ); + } + + [Theory] + [InlineData(null, "consumerFactory", "logger")] + [InlineData("connectionFactory", null, "logger")] + [InlineData("connectionFactory", "consumerFactory", null)] + public void Constructor_ShouldThrowException_WhenDependenciesAreNull( + string? connectionFactory, string? consumerFactory, string? logger) + { + var mockConnectionFactory = connectionFactory is null ? null! : _mockConnectionFactory.Object; + var mockConsumerFactory = consumerFactory is null ? null! : _mockConsumerFactory.Object; + var mockLogger = logger is null ? null! : _mockLogger.Object; + + Assert.Throws(() => + new RabbitMqMessagingProvider(mockConnectionFactory, mockConsumerFactory, mockLogger)); + } + + [Theory] + [InlineData(null)] + [InlineData("")] + [InlineData(" ")] + public async Task SubscribeAsync_ShouldThrowException_WhenTopicIsInvalid(string invalidTopic) + { + await Assert.ThrowsAsync(() => + _provider.SubscribeAsync(invalidTopic, TestConsumerGroup, (msg, token) => Task.CompletedTask, CancellationToken.None)); + } + + [Theory] + [InlineData(null)] + [InlineData("")] + [InlineData(" ")] + public async Task SubscribeAsync_ShouldThrowException_WhenConsumerGroupIsInvalid(string invalidConsumerGroup) + { + await Assert.ThrowsAsync(() => + _provider.SubscribeAsync(TestTopic, invalidConsumerGroup, (msg, token) => Task.CompletedTask, CancellationToken.None)); + } + + [Fact] + public async Task SubscribeAsync_ShouldThrowException_WhenHandlerIsNull() + { + // Act & Assert + await Assert.ThrowsAsync(() => + _provider.SubscribeAsync(TestTopic, TestConsumerGroup, null!, CancellationToken.None)); + } + + [Theory] + [InlineData(null)] + [InlineData("")] + [InlineData(" ")] + public async Task PublishAsync_ShouldThrowException_WhenTopicIsInvalid(string invalidTopic) + { + await Assert.ThrowsAsync(() => + _provider.PublishAsync(invalidTopic, "message", CancellationToken.None)); + } + + [Theory] + [InlineData(null)] + [InlineData("")] + [InlineData(" ")] + public async Task PublishAsync_ShouldThrowException_WhenMessageIsInvalid(string invalidMessage) + { + await Assert.ThrowsAsync(() => + _provider.PublishAsync(TestTopic, invalidMessage, CancellationToken.None)); + } + } +} From 7a5c9b686b6a4d62c8893df56b8939faaf767e05 Mon Sep 17 00:00:00 2001 From: David Runemalm Date: Thu, 20 Feb 2025 11:05:28 +0100 Subject: [PATCH 026/109] Make minor changes. --- samples/Bookstore/Makefile | 7 +------ .../Events/Kafka/KafkaMessagingProviderTests.cs | 3 +++ 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/samples/Bookstore/Makefile b/samples/Bookstore/Makefile index 9809c5c..ef72cbb 100644 --- a/samples/Bookstore/Makefile +++ b/samples/Bookstore/Makefile @@ -178,7 +178,7 @@ azure-get-servicebus-connection: ##@Azure Get the Service Bus connection string .PHONY: rabbitmq-start rabbitmq-start: ##@@RabbitMQ Start a RabbitMQ container - docker run -d --name rabbitmq --hostname rabbitmq \ + docker run --rm -d --name rabbitmq --hostname rabbitmq \ -e RABBITMQ_DEFAULT_USER=$(RABBITMQ_DEFAULT_USER) \ -e RABBITMQ_DEFAULT_PASS=$(RABBITMQ_DEFAULT_PASS) \ -p 5672:$(RABBITMQ_PORT) -p 15672:15672 rabbitmq:management @@ -197,11 +197,6 @@ rabbitmq-status: ##@RabbitMQ Check RabbitMQ container status rabbitmq-get-connection: ##@RabbitMQ Get the RabbitMQ connection string @echo "amqp://$(RABBITMQ_DEFAULT_USER):$(RABBITMQ_DEFAULT_PASS)@localhost:$(RABBITMQ_PORT)/" -.PHONY: rabbitmq-clean -rabbitmq-clean: ##@RabbitMQ Remove RabbitMQ container and its volumes - docker rm -f rabbitmq || true - @echo "RabbitMQ container removed." - .PHONY: rabbitmq-logs rabbitmq-logs: ##@RabbitMQ Show RabbitMQ logs docker logs -f rabbitmq diff --git a/src/OpenDDD/Tests/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs b/src/OpenDDD/Tests/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs index 0ccd0f1..4ced299 100644 --- a/src/OpenDDD/Tests/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs +++ b/src/OpenDDD/Tests/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs @@ -147,6 +147,9 @@ public async Task SubscribeAsync_ShouldHandleConsumerExceptionsGracefully() // Act await _provider.SubscribeAsync(Topic, ConsumerGroup, async (_, _) => await Task.CompletedTask, CancellationToken.None); + + // Ensure the background task has enough time to consume the message + await Task.Delay(100); // Assert _mockLogger.Verify( From d601bf9876f8a58ebaeccadadf29d89941fdd5c7 Mon Sep 17 00:00:00 2001 From: David Runemalm Date: Thu, 20 Feb 2025 11:19:58 +0100 Subject: [PATCH 027/109] Add validation to azure messaging provider. --- .../Azure/AzureServiceBusMessagingProvider.cs | 25 ++++++++++ .../AzureServiceBusMessagingProviderTests.cs | 47 +++++++++++++++++++ 2 files changed, 72 insertions(+) diff --git a/src/OpenDDD/Infrastructure/Events/Azure/AzureServiceBusMessagingProvider.cs b/src/OpenDDD/Infrastructure/Events/Azure/AzureServiceBusMessagingProvider.cs index 71d3cef..ef52942 100644 --- a/src/OpenDDD/Infrastructure/Events/Azure/AzureServiceBusMessagingProvider.cs +++ b/src/OpenDDD/Infrastructure/Events/Azure/AzureServiceBusMessagingProvider.cs @@ -27,6 +27,21 @@ public AzureServiceBusMessagingProvider( public async Task SubscribeAsync(string topic, string consumerGroup, Func messageHandler, CancellationToken cancellationToken = default) { + if (string.IsNullOrWhiteSpace(topic)) + { + throw new ArgumentException("Topic cannot be null or empty.", nameof(topic)); + } + + if (string.IsNullOrWhiteSpace(consumerGroup)) + { + throw new ArgumentException("Consumer group cannot be null or empty.", nameof(consumerGroup)); + } + + if (messageHandler is null) + { + throw new ArgumentNullException(nameof(messageHandler)); + } + var subscriptionName = consumerGroup; if (_autoCreateTopics) @@ -57,6 +72,16 @@ public async Task SubscribeAsync(string topic, string consumerGroup, Func(() => + _provider.SubscribeAsync(invalidTopic, _testSubscription, (msg, token) => Task.CompletedTask, CancellationToken.None)); + } + + [Theory] + [InlineData(null)] + [InlineData("")] + [InlineData(" ")] + public async Task SubscribeAsync_ShouldThrowException_WhenConsumerGroupIsInvalid(string invalidConsumerGroup) + { + await Assert.ThrowsAsync(() => + _provider.SubscribeAsync(_testTopic, invalidConsumerGroup, (msg, token) => Task.CompletedTask, CancellationToken.None)); + } + + [Fact] + public async Task SubscribeAsync_ShouldThrowException_WhenHandlerIsNull() + { + await Assert.ThrowsAsync(() => + _provider.SubscribeAsync(_testTopic, _testSubscription, null!, CancellationToken.None)); + } + + [Theory] + [InlineData(null)] + [InlineData("")] + [InlineData(" ")] + public async Task PublishAsync_ShouldThrowException_WhenTopicIsInvalid(string invalidTopic) + { + await Assert.ThrowsAsync(() => + _provider.PublishAsync(invalidTopic, "message", CancellationToken.None)); + } + + [Theory] + [InlineData(null)] + [InlineData("")] + [InlineData(" ")] + public async Task PublishAsync_ShouldThrowException_WhenMessageIsInvalid(string invalidMessage) + { + await Assert.ThrowsAsync(() => + _provider.PublishAsync(_testTopic, invalidMessage, CancellationToken.None)); + } + [Fact] public async Task SubscribeAsync_ShouldCreateTopicIfNotExists_WhenAutoCreateEnabled() { From 2fdd8987bb413eedd531dd0d2f6b1c875c470ea0 Mon Sep 17 00:00:00 2001 From: David Runemalm Date: Thu, 20 Feb 2025 11:23:20 +0100 Subject: [PATCH 028/109] Add test for azure provider constructor. --- .../AzureServiceBusMessagingProviderTests.cs | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/OpenDDD/Tests/Infrastructure/Events/Azure/AzureServiceBusMessagingProviderTests.cs b/src/OpenDDD/Tests/Infrastructure/Events/Azure/AzureServiceBusMessagingProviderTests.cs index bd3aa12..bc091b5 100644 --- a/src/OpenDDD/Tests/Infrastructure/Events/Azure/AzureServiceBusMessagingProviderTests.cs +++ b/src/OpenDDD/Tests/Infrastructure/Events/Azure/AzureServiceBusMessagingProviderTests.cs @@ -52,6 +52,21 @@ public AzureServiceBusMessagingProviderTests() ); } + [Theory] + [InlineData(null, "adminClient", "logger")] + [InlineData("client", null, "logger")] + [InlineData("client", "adminClient", null)] + public void Constructor_ShouldThrowException_WhenDependenciesAreNull( + string? client, string? adminClient, string? logger) + { + var mockClient = client is null ? null! : _mockClient.Object; + var mockAdminClient = adminClient is null ? null! : _mockAdminClient.Object; + var mockLogger = logger is null ? null! : _mockLogger.Object; + + Assert.Throws(() => + new AzureServiceBusMessagingProvider(mockClient, mockAdminClient, true, mockLogger)); + } + [Theory] [InlineData(null)] [InlineData("")] From e9740a58b0fca03c017a6f575cf0441c10bc62f8 Mon Sep 17 00:00:00 2001 From: David Runemalm Date: Thu, 20 Feb 2025 11:33:42 +0100 Subject: [PATCH 029/109] Add validation to kafka messaging provider. --- .../Events/Kafka/KafkaMessagingProvider.cs | 17 ++++- .../Kafka/KafkaMessagingProviderTests.cs | 70 ++++++++++++++++--- 2 files changed, 78 insertions(+), 9 deletions(-) diff --git a/src/OpenDDD/Infrastructure/Events/Kafka/KafkaMessagingProvider.cs b/src/OpenDDD/Infrastructure/Events/Kafka/KafkaMessagingProvider.cs index a7cca1d..1ac9083 100644 --- a/src/OpenDDD/Infrastructure/Events/Kafka/KafkaMessagingProvider.cs +++ b/src/OpenDDD/Infrastructure/Events/Kafka/KafkaMessagingProvider.cs @@ -28,7 +28,7 @@ public KafkaMessagingProvider( ILogger logger) { if (string.IsNullOrWhiteSpace(bootstrapServers)) - throw new ArgumentException("Kafka bootstrap servers must be configured.", nameof(bootstrapServers)); + throw new ArgumentNullException(nameof(bootstrapServers)); _bootstrapServers = bootstrapServers; _adminClient = adminClient ?? throw new ArgumentNullException(nameof(adminClient)); @@ -44,6 +44,15 @@ public async Task SubscribeAsync( Func messageHandler, CancellationToken cancellationToken) { + if (string.IsNullOrWhiteSpace(topic)) + throw new ArgumentException("Topic cannot be null or empty.", nameof(topic)); + + if (string.IsNullOrWhiteSpace(consumerGroup)) + throw new ArgumentException("Consumer group cannot be null or empty.", nameof(consumerGroup)); + + if (messageHandler is null) + throw new ArgumentNullException(nameof(messageHandler)); + if (_autoCreateTopics) { await CreateTopicIfNotExistsAsync(topic, cancellationToken); @@ -97,6 +106,12 @@ private async Task StartConsumerLoop( public async Task PublishAsync(string topic, string message, CancellationToken cancellationToken) { + if (string.IsNullOrWhiteSpace(topic)) + throw new ArgumentException("Topic cannot be null or empty.", nameof(topic)); + + if (string.IsNullOrWhiteSpace(message)) + throw new ArgumentException("Message cannot be null or empty.", nameof(message)); + if (_autoCreateTopics) { await CreateTopicIfNotExistsAsync(topic, cancellationToken); diff --git a/src/OpenDDD/Tests/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs b/src/OpenDDD/Tests/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs index 4ced299..dd89a06 100644 --- a/src/OpenDDD/Tests/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs +++ b/src/OpenDDD/Tests/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs @@ -54,17 +54,71 @@ public KafkaMessagingProviderTests() .Setup(a => a.GetMetadata(Topic, It.IsAny())) .Returns(metadata); } + + [Theory] + [InlineData(null, "adminClient", "producer", "consumerFactory", "logger")] + [InlineData("bootstrapServers", null, "producer", "consumerFactory", "logger")] + [InlineData("bootstrapServers", "adminClient", null, "consumerFactory", "logger")] + [InlineData("bootstrapServers", "adminClient", "producer", null, "logger")] + [InlineData("bootstrapServers", "adminClient", "producer", "consumerFactory", null)] + public void Constructor_ShouldThrowException_WhenDependenciesAreNull( + string? bootstrapServers, string? adminClient, string? producer, string? consumerFactory, string? logger) + { + var bs = bootstrapServers is null ? null! : BootstrapServers; + var mockAdmin = adminClient is null ? null! : _mockAdminClient.Object; + var mockProducer = producer is null ? null! : _mockProducer.Object; + var mockConsumerFactory = consumerFactory is null ? null! : _mockConsumerFactory.Object; + var mockLogger = logger is null ? null! : _mockLogger.Object; + + Assert.Throws(() => + new KafkaMessagingProvider(bs, mockAdmin, mockProducer, mockConsumerFactory, true, mockLogger)); + } + + [Theory] + [InlineData(null)] + [InlineData("")] + [InlineData(" ")] + public async Task SubscribeAsync_ShouldThrowException_WhenTopicIsInvalid(string invalidTopic) + { + await Assert.ThrowsAsync(() => + _provider.SubscribeAsync(invalidTopic, ConsumerGroup, (msg, token) => Task.CompletedTask, CancellationToken.None)); + } + + [Theory] + [InlineData(null)] + [InlineData("")] + [InlineData(" ")] + public async Task SubscribeAsync_ShouldThrowException_WhenConsumerGroupIsInvalid(string invalidConsumerGroup) + { + await Assert.ThrowsAsync(() => + _provider.SubscribeAsync(Topic, invalidConsumerGroup, (msg, token) => Task.CompletedTask, CancellationToken.None)); + } [Fact] - public void Constructor_ShouldThrowException_WhenBootstrapServersIsNullOrEmpty() + public async Task SubscribeAsync_ShouldThrowException_WhenHandlerIsNull() { - Assert.Throws(() => new KafkaMessagingProvider( - "", - _mockAdminClient.Object, - _mockProducer.Object, - _mockConsumerFactory.Object, - true, - _mockLogger.Object)); + await Assert.ThrowsAsync(() => + _provider.SubscribeAsync(Topic, ConsumerGroup, null!, CancellationToken.None)); + } + + [Theory] + [InlineData(null)] + [InlineData("")] + [InlineData(" ")] + public async Task PublishAsync_ShouldThrowException_WhenTopicIsInvalid(string invalidTopic) + { + await Assert.ThrowsAsync(() => + _provider.PublishAsync(invalidTopic, Message, CancellationToken.None)); + } + + [Theory] + [InlineData(null)] + [InlineData("")] + [InlineData(" ")] + public async Task PublishAsync_ShouldThrowException_WhenMessageIsInvalid(string invalidMessage) + { + await Assert.ThrowsAsync(() => + _provider.PublishAsync(Topic, invalidMessage, CancellationToken.None)); } [Fact] From 634cc1ba538382d4e9f810f18a4f00d4fe2e80de Mon Sep 17 00:00:00 2001 From: David Runemalm Date: Thu, 20 Feb 2025 13:58:21 +0100 Subject: [PATCH 030/109] Add unit- and integration test base classes. --- .../OpenDdd/Postgres/PostgresOpenDddRepository.cs | 2 +- src/OpenDDD/Tests/Base/IntegrationTests.cs | 7 +++++++ src/OpenDDD/Tests/Base/UnitTests.cs | 7 +++++++ .../Events/Azure/AzureServiceBusMessagingProviderTests.cs | 3 ++- .../Tests/Infrastructure/Events/DomainPublisherTests.cs | 3 ++- .../Tests/Infrastructure/Events/EventSerializerTests.cs | 3 ++- .../Events/InMemory/InMemoryMessagingProviderTests.cs | 3 ++- .../Infrastructure/Events/IntegrationPublisherTests.cs | 3 ++- .../Events/Kafka/KafkaMessagingProviderTests.cs | 3 ++- .../Events/RabbitMq/RabbitMqMessagingProviderTests.cs | 3 ++- .../OpenDdd/Expressions/JsonbExpressionParserTests.cs | 3 ++- .../OpenDdd/Serializers/OpenDddAggregateSerializerTests.cs | 3 ++- .../OpenDdd/Serializers/OpenDddSerializerTests.cs | 3 ++- 13 files changed, 35 insertions(+), 11 deletions(-) create mode 100644 src/OpenDDD/Tests/Base/IntegrationTests.cs create mode 100644 src/OpenDDD/Tests/Base/UnitTests.cs diff --git a/src/OpenDDD/Infrastructure/Repository/OpenDdd/Postgres/PostgresOpenDddRepository.cs b/src/OpenDDD/Infrastructure/Repository/OpenDdd/Postgres/PostgresOpenDddRepository.cs index 705a334..d152c85 100644 --- a/src/OpenDDD/Infrastructure/Repository/OpenDdd/Postgres/PostgresOpenDddRepository.cs +++ b/src/OpenDDD/Infrastructure/Repository/OpenDdd/Postgres/PostgresOpenDddRepository.cs @@ -4,8 +4,8 @@ using OpenDDD.Infrastructure.Repository.OpenDdd.Base; using OpenDDD.API.Extensions; using OpenDDD.Infrastructure.Persistence.OpenDdd.DatabaseSession.Postgres; -using Npgsql; using OpenDDD.Infrastructure.Persistence.OpenDdd.Expressions; +using Npgsql; namespace OpenDDD.Infrastructure.Repository.OpenDdd.Postgres { diff --git a/src/OpenDDD/Tests/Base/IntegrationTests.cs b/src/OpenDDD/Tests/Base/IntegrationTests.cs new file mode 100644 index 0000000..8ae87aa --- /dev/null +++ b/src/OpenDDD/Tests/Base/IntegrationTests.cs @@ -0,0 +1,7 @@ +namespace OpenDDD.Tests.Base +{ + public class IntegrationTests + { + + } +} diff --git a/src/OpenDDD/Tests/Base/UnitTests.cs b/src/OpenDDD/Tests/Base/UnitTests.cs new file mode 100644 index 0000000..ee4e2b8 --- /dev/null +++ b/src/OpenDDD/Tests/Base/UnitTests.cs @@ -0,0 +1,7 @@ +namespace OpenDDD.Tests.Base +{ + public class UnitTests + { + + } +} diff --git a/src/OpenDDD/Tests/Infrastructure/Events/Azure/AzureServiceBusMessagingProviderTests.cs b/src/OpenDDD/Tests/Infrastructure/Events/Azure/AzureServiceBusMessagingProviderTests.cs index bc091b5..cd39801 100644 --- a/src/OpenDDD/Tests/Infrastructure/Events/Azure/AzureServiceBusMessagingProviderTests.cs +++ b/src/OpenDDD/Tests/Infrastructure/Events/Azure/AzureServiceBusMessagingProviderTests.cs @@ -6,10 +6,11 @@ using Azure.Messaging.ServiceBus; using Azure.Messaging.ServiceBus.Administration; using OpenDDD.Infrastructure.Events.Azure; +using OpenDDD.Tests.Base; namespace OpenDDD.Tests.Infrastructure.Events.Azure { - public class AzureServiceBusMessagingProviderTests + public class AzureServiceBusMessagingProviderTests : UnitTests { private readonly Mock _mockClient; private readonly Mock _mockAdminClient; diff --git a/src/OpenDDD/Tests/Infrastructure/Events/DomainPublisherTests.cs b/src/OpenDDD/Tests/Infrastructure/Events/DomainPublisherTests.cs index ff9fbd5..7c7a64d 100644 --- a/src/OpenDDD/Tests/Infrastructure/Events/DomainPublisherTests.cs +++ b/src/OpenDDD/Tests/Infrastructure/Events/DomainPublisherTests.cs @@ -1,11 +1,12 @@ using FluentAssertions; using OpenDDD.Domain.Model; using OpenDDD.Infrastructure.Events; +using OpenDDD.Tests.Base; using Xunit; namespace OpenDDD.Tests.Infrastructure.Events { - public class DomainPublisherTests + public class DomainPublisherTests : UnitTests { private class TestEvent : IDomainEvent { } diff --git a/src/OpenDDD/Tests/Infrastructure/Events/EventSerializerTests.cs b/src/OpenDDD/Tests/Infrastructure/Events/EventSerializerTests.cs index fc5e030..6a06e8e 100644 --- a/src/OpenDDD/Tests/Infrastructure/Events/EventSerializerTests.cs +++ b/src/OpenDDD/Tests/Infrastructure/Events/EventSerializerTests.cs @@ -1,9 +1,10 @@ using Xunit; using OpenDDD.Infrastructure.Events; +using OpenDDD.Tests.Base; namespace OpenDDD.Tests.Infrastructure.Events { - public class EventSerializerTests + public class EventSerializerTests : UnitTests { [Fact] public void Serialize_ShouldReturnJsonString_WhenEventIsValid() diff --git a/src/OpenDDD/Tests/Infrastructure/Events/InMemory/InMemoryMessagingProviderTests.cs b/src/OpenDDD/Tests/Infrastructure/Events/InMemory/InMemoryMessagingProviderTests.cs index bad7ad2..4619954 100644 --- a/src/OpenDDD/Tests/Infrastructure/Events/InMemory/InMemoryMessagingProviderTests.cs +++ b/src/OpenDDD/Tests/Infrastructure/Events/InMemory/InMemoryMessagingProviderTests.cs @@ -3,10 +3,11 @@ using Moq; using Xunit; using OpenDDD.Infrastructure.Events.InMemory; +using OpenDDD.Tests.Base; namespace OpenDDD.Tests.Infrastructure.Events.InMemory { - public class InMemoryMessagingProviderTests + public class InMemoryMessagingProviderTests : UnitTests { private readonly Mock> _mockLogger; private readonly InMemoryMessagingProvider _messagingProvider; diff --git a/src/OpenDDD/Tests/Infrastructure/Events/IntegrationPublisherTests.cs b/src/OpenDDD/Tests/Infrastructure/Events/IntegrationPublisherTests.cs index 3d750d4..7946faf 100644 --- a/src/OpenDDD/Tests/Infrastructure/Events/IntegrationPublisherTests.cs +++ b/src/OpenDDD/Tests/Infrastructure/Events/IntegrationPublisherTests.cs @@ -1,11 +1,12 @@ using FluentAssertions; using OpenDDD.Domain.Model; using OpenDDD.Infrastructure.Events; +using OpenDDD.Tests.Base; using Xunit; namespace OpenDDD.Tests.Infrastructure.Events { - public class IntegrationPublisherTests + public class IntegrationPublisherTests : UnitTests { private class TestIntegrationEvent : IIntegrationEvent { } diff --git a/src/OpenDDD/Tests/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs b/src/OpenDDD/Tests/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs index dd89a06..272e4c1 100644 --- a/src/OpenDDD/Tests/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs +++ b/src/OpenDDD/Tests/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs @@ -5,10 +5,11 @@ using Confluent.Kafka.Admin; using OpenDDD.Infrastructure.Events.Kafka; using OpenDDD.Infrastructure.Events.Kafka.Factories; +using OpenDDD.Tests.Base; namespace OpenDDD.Tests.Infrastructure.Events.Kafka { - public class KafkaMessagingProviderTests + public class KafkaMessagingProviderTests : UnitTests { private readonly Mock> _mockProducer; private readonly Mock _mockAdminClient; diff --git a/src/OpenDDD/Tests/Infrastructure/Events/RabbitMq/RabbitMqMessagingProviderTests.cs b/src/OpenDDD/Tests/Infrastructure/Events/RabbitMq/RabbitMqMessagingProviderTests.cs index 31f23b8..b66e46a 100644 --- a/src/OpenDDD/Tests/Infrastructure/Events/RabbitMq/RabbitMqMessagingProviderTests.cs +++ b/src/OpenDDD/Tests/Infrastructure/Events/RabbitMq/RabbitMqMessagingProviderTests.cs @@ -3,11 +3,12 @@ using Moq; using OpenDDD.Infrastructure.Events.RabbitMq; using OpenDDD.Infrastructure.Events.RabbitMq.Factories; +using OpenDDD.Tests.Base; using RabbitMQ.Client; namespace OpenDDD.Tests.Infrastructure.Events.RabbitMq { - public class RabbitMqMessagingProviderTests + public class RabbitMqMessagingProviderTests : UnitTests { private readonly Mock _mockConnectionFactory; private readonly Mock _mockConsumerFactory; diff --git a/src/OpenDDD/Tests/Infrastructure/Persistence/OpenDdd/Expressions/JsonbExpressionParserTests.cs b/src/OpenDDD/Tests/Infrastructure/Persistence/OpenDdd/Expressions/JsonbExpressionParserTests.cs index 1bc3ba4..b43102e 100644 --- a/src/OpenDDD/Tests/Infrastructure/Persistence/OpenDdd/Expressions/JsonbExpressionParserTests.cs +++ b/src/OpenDDD/Tests/Infrastructure/Persistence/OpenDdd/Expressions/JsonbExpressionParserTests.cs @@ -2,10 +2,11 @@ using Xunit; using FluentAssertions; using OpenDDD.Infrastructure.Persistence.OpenDdd.Expressions; +using OpenDDD.Tests.Base; namespace OpenDDD.Tests.Infrastructure.Persistence.OpenDdd.Expressions { - public class JsonbExpressionParserTests + public class JsonbExpressionParserTests : UnitTests { private class Customer { diff --git a/src/OpenDDD/Tests/Infrastructure/Persistence/OpenDdd/Serializers/OpenDddAggregateSerializerTests.cs b/src/OpenDDD/Tests/Infrastructure/Persistence/OpenDdd/Serializers/OpenDddAggregateSerializerTests.cs index 436200a..09b6769 100644 --- a/src/OpenDDD/Tests/Infrastructure/Persistence/OpenDdd/Serializers/OpenDddAggregateSerializerTests.cs +++ b/src/OpenDDD/Tests/Infrastructure/Persistence/OpenDdd/Serializers/OpenDddAggregateSerializerTests.cs @@ -1,10 +1,11 @@ using Xunit; using OpenDDD.Infrastructure.Persistence.OpenDdd.Serializers; +using OpenDDD.Tests.Base; using OpenDDD.Tests.Domain.Model; namespace OpenDDD.Tests.Infrastructure.Persistence.OpenDdd.Serializers { - public class OpenDddAggregateSerializerTests + public class OpenDddAggregateSerializerTests : UnitTests { private readonly OpenDddAggregateSerializer _serializer; diff --git a/src/OpenDDD/Tests/Infrastructure/Persistence/OpenDdd/Serializers/OpenDddSerializerTests.cs b/src/OpenDDD/Tests/Infrastructure/Persistence/OpenDdd/Serializers/OpenDddSerializerTests.cs index abb6c86..1320c83 100644 --- a/src/OpenDDD/Tests/Infrastructure/Persistence/OpenDdd/Serializers/OpenDddSerializerTests.cs +++ b/src/OpenDDD/Tests/Infrastructure/Persistence/OpenDdd/Serializers/OpenDddSerializerTests.cs @@ -1,9 +1,10 @@ using OpenDDD.Infrastructure.Persistence.OpenDdd.Serializers; +using OpenDDD.Tests.Base; using Xunit; namespace OpenDDD.Tests.Infrastructure.Persistence.OpenDdd.Serializers { - public class OpenDddSerializerTests + public class OpenDddSerializerTests : UnitTests { private readonly OpenDddSerializer _serializer; From b965ec441d1b396adb3b87cd3f0c6ceac6e9f20e Mon Sep 17 00:00:00 2001 From: David Runemalm Date: Thu, 20 Feb 2025 13:58:31 +0100 Subject: [PATCH 031/109] Add tests workflow. --- .github/workflows/tests.yml | 84 +++++++++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100644 .github/workflows/tests.yml diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000..61825f9 --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,84 @@ +name: Run Tests + +on: + push: + branches: + - develop + +jobs: + unit-tests: + name: Unit Tests (.NET ${{ matrix.dotnet-version }}) + runs-on: ubuntu-latest + strategy: + matrix: + dotnet-version: [8.0.x, 9.0.x] + + steps: + - name: Checkout Repository + uses: actions/checkout@v4 + + - name: Setup .NET + uses: actions/setup-dotnet@v3 + with: + dotnet-version: ${{ matrix.dotnet-version }} + + - name: Restore Dependencies + run: dotnet restore + + - name: Build Project + run: dotnet build --no-restore --configuration Release + + - name: Run Unit Tests + run: dotnet test --no-build --configuration Release --filter FullyQualifiedName~UnitTests + + # integration-tests: + # name: Integration Tests (.NET ${{ matrix.dotnet-version }}) + # runs-on: ubuntu-latest + # strategy: + # matrix: + # dotnet-version: [8.0.x, 9.0.x] + + # services: + # kafka: + # image: confluentinc/cp-kafka:latest + # env: + # KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:9092 + # KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + # ports: + # - 9092:9092 + # options: --network-alias kafka + + # rabbitmq: + # image: rabbitmq:3-management + # ports: + # - 5672:5672 + # - 15672:15672 + + # azurite: + # image: mcr.microsoft.com/azure-storage/azurite + # ports: + # - 10000:10000 + # - 10001:10001 + # - 10002:10002 + + # steps: + # - name: Checkout Repository + # uses: actions/checkout@v4 + + # - name: Setup .NET + # uses: actions/setup-dotnet@v3 + # with: + # dotnet-version: ${{ matrix.dotnet-version }} + + # - name: Restore Dependencies + # run: dotnet restore + + # - name: Build Project + # run: dotnet build --no-restore --configuration Release + + # - name: Run Integration Tests + # env: + # KAFKA_BROKER: localhost:9092 + # RABBITMQ_HOST: localhost + # AZURE_STORAGE_CONNECTION_STRING: "UseDevelopmentStorage=true;" + # run: dotnet test --no-build --configuration Release --filter FullyQualifiedName~IntegrationTests From 5f7a5b50597fdf0de92a678ce985ed9b7b574ad8 Mon Sep 17 00:00:00 2001 From: David Runemalm Date: Thu, 20 Feb 2025 14:20:29 +0100 Subject: [PATCH 032/109] Add act targets to makefile. --- .github/workflows/tests.yml | 3 ++ Makefile | 61 ++++++++++++++++++++++++++++++------- 2 files changed, 53 insertions(+), 11 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 61825f9..1fbe106 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -23,12 +23,15 @@ jobs: dotnet-version: ${{ matrix.dotnet-version }} - name: Restore Dependencies + working-directory: src/OpenDDD run: dotnet restore - name: Build Project + working-directory: src/OpenDDD run: dotnet build --no-restore --configuration Release - name: Run Unit Tests + working-directory: src/OpenDDD/Tests run: dotnet test --no-build --configuration Release --filter FullyQualifiedName~UnitTests # integration-tests: diff --git a/Makefile b/Makefile index 7409780..72deeec 100644 --- a/Makefile +++ b/Makefile @@ -32,17 +32,6 @@ FEED_DIR := $(HOME)/Projects/LocalFeed USER_NUGET_CONFIG_DIR=$(HOME)/.config/NuGet/NuGet.Config SPHINXDOC_IMG := openddd.net/sphinxdoc -DOCSAUTOBUILD_HOST_NAME := docsautobuild-openddd.net -DOCSAUTOBUILD_CONTAINER_NAME := docsautobuild-openddd.net -DOCSAUTOBUILD_PORT := 10001 - -TEMPLATES_DIR := $(PWD)/templates -TEMPLATES_CSPROJ := $(TEMPLATES_DIR)/templatepack.csproj -TEMPLATES_OUT := $(TEMPLATES_DIR)/bin/templates -TEMPLATES_NAME := OpenDDD.NET-Templates -TEMPLATES_VERSION := 3.0.0-alpha.1 -TEMPLATES_NUPKG := $(TEMPLATES_OUT)/$(TEMPLATES_NAME).$(TEMPLATES_VERSION).nupkg - BLUE := $(shell tput -Txterm setaf 4) GREEN := $(shell tput -Txterm setaf 2) TURQUOISE := $(shell tput -Txterm setaf 6) @@ -142,6 +131,10 @@ push: ##@Build Push the nuget to the global feed # DOCS ########################################################################## +DOCSAUTOBUILD_HOST_NAME := docsautobuild-openddd.net +DOCSAUTOBUILD_CONTAINER_NAME := docsautobuild-openddd.net +DOCSAUTOBUILD_PORT := 10001 + .PHONY: sphinx-buildimage sphinx-buildimage: ##@Docs Build the custom sphinxdoc image docker build -t $(SPHINXDOC_IMG) $(DOCS_DIR) @@ -190,6 +183,13 @@ clear-nuget-caches: ##@Build clean all nuget caches # TEMPLATES ########################################################################## +TEMPLATES_DIR := $(PWD)/templates +TEMPLATES_CSPROJ := $(TEMPLATES_DIR)/templatepack.csproj +TEMPLATES_OUT := $(TEMPLATES_DIR)/bin/templates +TEMPLATES_NAME := OpenDDD.NET-Templates +TEMPLATES_VERSION := 3.0.0-alpha.1 +TEMPLATES_NUPKG := $(TEMPLATES_OUT)/$(TEMPLATES_NAME).$(TEMPLATES_VERSION).nupkg + .PHONY: templates-install templates-install: ##@Template Install the OpenDDD.NET project template locally dotnet new install $(TEMPLATES_NUPKG) @@ -208,3 +208,42 @@ templates-publish: ##@Template Publish the template to NuGet .PHONY: templates-rebuild templates-rebuild: templates-uninstall templates-pack templates-install ##@Template Rebuild and reinstall the template + +########################################################################## +# Act +########################################################################## + +# ACT_IMAGE := ghcr.io/catthehacker/ubuntu:full-latest +ACT_IMAGE := ghcr.io/catthehacker/ubuntu:runner-latest + +.PHONY: act-install +act-install: ##@Act Install act CLI + brew install act + +.PHONY: act-list +act-list: ##@Act List available workflows + act -l + +.PHONY: act-test +act-test: ##@Act Run all tests locally using act + act -P ubuntu-latest=$(ACT_IMAGE) + +.PHONY: act-test-dotnet +act-test-dotnet: ##@Act Run tests for a specific .NET version (usage: make act-test-dotnet DOTNET_VERSION=8.0.x) + @if [ -z "$(DOTNET_VERSION)" ]; then \ + echo "Error: Specify .NET version using DOTNET_VERSION="; \ + exit 1; \ + fi + act -P ubuntu-latest=$(ACT_IMAGE) -s matrix.dotnet-version=$(DOTNET_VERSION) + +.PHONY: act-unit-tests +act-unit-tests: ##@Act Run only unit tests + act -P ubuntu-latest=$(ACT_IMAGE) -j unit-tests + +.PHONY: act-integration-tests +act-integration-tests: ##@Act Run only integration tests + act -P ubuntu-latest=$(ACT_IMAGE) -j integration-tests + +.PHONY: act-debug +act-debug: ##@Act Run act with verbose logging + act -P ubuntu-latest=$(ACT_IMAGE) --verbose From 32aee043a3c47689ff01ae65cdbddedae284d56a Mon Sep 17 00:00:00 2001 From: David Runemalm Date: Thu, 20 Feb 2025 14:28:35 +0100 Subject: [PATCH 033/109] Update workflow. --- .github/workflows/tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 1fbe106..458300f 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - dotnet-version: [8.0.x, 9.0.x] + dotnet-version: [8.0.x] steps: - name: Checkout Repository @@ -28,7 +28,7 @@ jobs: - name: Build Project working-directory: src/OpenDDD - run: dotnet build --no-restore --configuration Release + run: dotnet build --no-restore --configuration Release /p:TreatWarningsAsErrors=false - name: Run Unit Tests working-directory: src/OpenDDD/Tests From 343d6af17ced0c1cefddba2f9097f6e3f933e6f1 Mon Sep 17 00:00:00 2001 From: David Runemalm Date: Thu, 20 Feb 2025 14:30:04 +0100 Subject: [PATCH 034/109] Fix tests step. --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 458300f..8566b58 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -31,7 +31,7 @@ jobs: run: dotnet build --no-restore --configuration Release /p:TreatWarningsAsErrors=false - name: Run Unit Tests - working-directory: src/OpenDDD/Tests + working-directory: src/OpenDDD run: dotnet test --no-build --configuration Release --filter FullyQualifiedName~UnitTests # integration-tests: From ae9aa18f19a3941d20fad3a31122e6b90ac38f47 Mon Sep 17 00:00:00 2001 From: David Runemalm Date: Thu, 20 Feb 2025 14:52:47 +0100 Subject: [PATCH 035/109] Add test targets to makefile. --- Makefile | 39 +++++++++++-------- .../Events/IntegrationPublisherTests.cs | 2 +- 2 files changed, 24 insertions(+), 17 deletions(-) diff --git a/Makefile b/Makefile index 72deeec..3323731 100644 --- a/Makefile +++ b/Makefile @@ -93,17 +93,34 @@ help:: ##@Other Show this help. # TEST ########################################################################## .PHONY: test -test: ##@Test run all unit tests - ENV_FILE=env.test dotnet test $(TESTS_DIR) +test: ##@Test Run all tests (unit & integration) + cd $(SRC_DIR)/OpenDDD && dotnet test --no-build --configuration Release + +.PHONY: test-unit +test-unit: ##@Test Run only unit tests + cd $(SRC_DIR)/OpenDDD && dotnet test --no-build --configuration Release --filter FullyQualifiedName~UnitTests + +.PHONY: test-integration +test-integration: ##@Test Run only integration tests + cd $(SRC_DIR)/OpenDDD && dotnet test --no-build --configuration Release --filter FullyQualifiedName~IntegrationTests ########################################################################## # BUILD ########################################################################## + .PHONY: clean clean: ##@Build clean the solution find . $(SRC_DIR) -iname "bin" | xargs rm -rf find . $(SRC_DIR) -iname "obj" | xargs rm -rf +.PHONY: clear-nuget-caches +clear-nuget-caches: ##@Build clean all nuget caches + nuget locals all -clear + +.PHONY: restore +restore: ##@Build restore the solution + cd src && dotnet restore + .PHONY: build build: ##@Build build the solution cd $(SRC_DIR) && \ @@ -168,17 +185,6 @@ sphinx-autobuild: ##@Docs Activate autobuild of docs sphinx-opendocs: ##@Docs Open the docs in browser open $(DOCS_DIR)/_build/html/index.html -########################################################################## -# .NET -########################################################################## -.PHONY: restore -restore: ##@Build restore the solution - cd src && dotnet restore - -.PHONY: clear-nuget-caches -clear-nuget-caches: ##@Build clean all nuget caches - nuget locals all -clear - ########################################################################## # TEMPLATES ########################################################################## @@ -210,11 +216,12 @@ templates-publish: ##@Template Publish the template to NuGet templates-rebuild: templates-uninstall templates-pack templates-install ##@Template Rebuild and reinstall the template ########################################################################## -# Act +# ACT ########################################################################## -# ACT_IMAGE := ghcr.io/catthehacker/ubuntu:full-latest -ACT_IMAGE := ghcr.io/catthehacker/ubuntu:runner-latest +ACT_IMAGE := ghcr.io/catthehacker/ubuntu:full-latest +# ACT_IMAGE := ghcr.io/catthehacker/ubuntu:runner-latest +# ACT_IMAGE := ghcr.io/catthehacker/ubuntu:act-latest .PHONY: act-install act-install: ##@Act Install act CLI diff --git a/src/OpenDDD/Tests/Infrastructure/Events/IntegrationPublisherTests.cs b/src/OpenDDD/Tests/Infrastructure/Events/IntegrationPublisherTests.cs index 7946faf..46cc2a8 100644 --- a/src/OpenDDD/Tests/Infrastructure/Events/IntegrationPublisherTests.cs +++ b/src/OpenDDD/Tests/Infrastructure/Events/IntegrationPublisherTests.cs @@ -1,8 +1,8 @@ using FluentAssertions; +using Xunit; using OpenDDD.Domain.Model; using OpenDDD.Infrastructure.Events; using OpenDDD.Tests.Base; -using Xunit; namespace OpenDDD.Tests.Infrastructure.Events { From 502d8b5d01c332b3f7d3de11ec09a68d6cc1285f Mon Sep 17 00:00:00 2001 From: David Runemalm Date: Thu, 20 Feb 2025 17:58:06 +0100 Subject: [PATCH 036/109] Create dedicated tests project. --- .github/workflows/tests.yml | 6 ++-- Makefile | 28 +++++++++++------- samples/Bookstore/src/Bookstore.sln | 6 ++++ .../Bookstore/src/Bookstore/Bookstore.csproj | 1 + .../Base/IntegrationTests.cs | 0 .../Tests => OpenDDD.Tests}/Base/UnitTests.cs | 0 .../Domain/Model/TestAggregateRoot.cs | 0 .../Domain/Model/TestEntity.cs | 0 .../Domain/Model/TestValueObject.cs | 0 .../AzureServiceBusMessagingProviderTests.cs | 0 .../Events/DomainPublisherTests.cs | 1 - .../Events/EventSerializerTests.cs | 0 .../InMemoryMessagingProviderTests.cs | 0 .../Events/IntegrationPublisherTests.cs | 0 .../Kafka/KafkaMessagingProviderTests.cs | 0 .../RabbitMqMessagingProviderTests.cs | 0 .../Infrastructure/Events/TestEvent.cs | 0 .../Expressions/JsonbExpressionParserTests.cs | 0 .../OpenDddAggregateSerializerTests.cs | 0 .../Serializers/OpenDddSerializerTests.cs | 0 src/OpenDDD.Tests/OpenDDD.Tests.csproj | 29 +++++++++++++++++++ src/OpenDDD.sln | 7 +++++ src/OpenDDD/OpenDDD.csproj | 4 --- 23 files changed, 64 insertions(+), 18 deletions(-) rename src/{OpenDDD/Tests => OpenDDD.Tests}/Base/IntegrationTests.cs (100%) rename src/{OpenDDD/Tests => OpenDDD.Tests}/Base/UnitTests.cs (100%) rename src/{OpenDDD/Tests => OpenDDD.Tests}/Domain/Model/TestAggregateRoot.cs (100%) rename src/{OpenDDD/Tests => OpenDDD.Tests}/Domain/Model/TestEntity.cs (100%) rename src/{OpenDDD/Tests => OpenDDD.Tests}/Domain/Model/TestValueObject.cs (100%) rename src/{OpenDDD/Tests => OpenDDD.Tests}/Infrastructure/Events/Azure/AzureServiceBusMessagingProviderTests.cs (100%) rename src/{OpenDDD/Tests => OpenDDD.Tests}/Infrastructure/Events/DomainPublisherTests.cs (99%) rename src/{OpenDDD/Tests => OpenDDD.Tests}/Infrastructure/Events/EventSerializerTests.cs (100%) rename src/{OpenDDD/Tests => OpenDDD.Tests}/Infrastructure/Events/InMemory/InMemoryMessagingProviderTests.cs (100%) rename src/{OpenDDD/Tests => OpenDDD.Tests}/Infrastructure/Events/IntegrationPublisherTests.cs (100%) rename src/{OpenDDD/Tests => OpenDDD.Tests}/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs (100%) rename src/{OpenDDD/Tests => OpenDDD.Tests}/Infrastructure/Events/RabbitMq/RabbitMqMessagingProviderTests.cs (100%) rename src/{OpenDDD/Tests => OpenDDD.Tests}/Infrastructure/Events/TestEvent.cs (100%) rename src/{OpenDDD/Tests => OpenDDD.Tests}/Infrastructure/Persistence/OpenDdd/Expressions/JsonbExpressionParserTests.cs (100%) rename src/{OpenDDD/Tests => OpenDDD.Tests}/Infrastructure/Persistence/OpenDdd/Serializers/OpenDddAggregateSerializerTests.cs (100%) rename src/{OpenDDD/Tests => OpenDDD.Tests}/Infrastructure/Persistence/OpenDdd/Serializers/OpenDddSerializerTests.cs (100%) create mode 100644 src/OpenDDD.Tests/OpenDDD.Tests.csproj diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 8566b58..987b345 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -23,15 +23,15 @@ jobs: dotnet-version: ${{ matrix.dotnet-version }} - name: Restore Dependencies - working-directory: src/OpenDDD + working-directory: src/OpenDDD.Tests run: dotnet restore - name: Build Project - working-directory: src/OpenDDD + working-directory: src/OpenDDD.Tests run: dotnet build --no-restore --configuration Release /p:TreatWarningsAsErrors=false - name: Run Unit Tests - working-directory: src/OpenDDD + working-directory: src/OpenDDD.Tests run: dotnet test --no-build --configuration Release --filter FullyQualifiedName~UnitTests # integration-tests: diff --git a/Makefile b/Makefile index 3323731..910e52a 100644 --- a/Makefile +++ b/Makefile @@ -24,6 +24,7 @@ NUGET_NAME := OpenDDD.NET ROOT_NAMESPACE := OpenDDD SRC_DIR := $(PWD)/src +TESTS_DIR := $(SRC_DIR)/OpenDDD.Tests DOCS_DIR := $(PWD)/docs SAMPLES_DIR := $(PWD)/samples NAMESPACE_DIR := $(SRC_DIR)/$(ROOT_NAMESPACE) @@ -92,17 +93,18 @@ help:: ##@Other Show this help. ########################################################################## # TEST ########################################################################## + .PHONY: test test: ##@Test Run all tests (unit & integration) - cd $(SRC_DIR)/OpenDDD && dotnet test --no-build --configuration Release + cd $(TESTS_DIR) && dotnet test --configuration Release .PHONY: test-unit test-unit: ##@Test Run only unit tests - cd $(SRC_DIR)/OpenDDD && dotnet test --no-build --configuration Release --filter FullyQualifiedName~UnitTests + cd $(TESTS_DIR) && dotnet test --configuration Release --filter FullyQualifiedName~UnitTests .PHONY: test-integration test-integration: ##@Test Run only integration tests - cd $(SRC_DIR)/OpenDDD && dotnet test --no-build --configuration Release --filter FullyQualifiedName~IntegrationTests + cd $(TESTS_DIR) && dotnet test --configuration Release --filter FullyQualifiedName~IntegrationTests ########################################################################## # BUILD @@ -219,21 +221,27 @@ templates-rebuild: templates-uninstall templates-pack templates-install ##@Templ # ACT ########################################################################## -ACT_IMAGE := ghcr.io/catthehacker/ubuntu:full-latest +# ACT_IMAGE := ghcr.io/catthehacker/ubuntu:full-latest # ACT_IMAGE := ghcr.io/catthehacker/ubuntu:runner-latest -# ACT_IMAGE := ghcr.io/catthehacker/ubuntu:act-latest +ACT_IMAGE := ghcr.io/catthehacker/ubuntu:act-latest .PHONY: act-install act-install: ##@Act Install act CLI brew install act +.PHONY: act-clean +act-clean: ##@Act Stop and remove all act containers + @docker stop $$(docker ps -q --filter ancestor=$(ACT_IMAGE)) 2>/dev/null || true + @docker rm $$(docker ps -aq --filter ancestor=$(ACT_IMAGE)) 2>/dev/null || true + @echo "✅ All act containers stopped and removed." + .PHONY: act-list act-list: ##@Act List available workflows act -l .PHONY: act-test act-test: ##@Act Run all tests locally using act - act -P ubuntu-latest=$(ACT_IMAGE) + act -P ubuntu-latest=$(ACT_IMAGE) --reuse .PHONY: act-test-dotnet act-test-dotnet: ##@Act Run tests for a specific .NET version (usage: make act-test-dotnet DOTNET_VERSION=8.0.x) @@ -241,16 +249,16 @@ act-test-dotnet: ##@Act Run tests for a specific .NET version (usage: make act- echo "Error: Specify .NET version using DOTNET_VERSION="; \ exit 1; \ fi - act -P ubuntu-latest=$(ACT_IMAGE) -s matrix.dotnet-version=$(DOTNET_VERSION) + act -P ubuntu-latest=$(ACT_IMAGE) -s matrix.dotnet-version=$(DOTNET_VERSION) --reuse .PHONY: act-unit-tests act-unit-tests: ##@Act Run only unit tests - act -P ubuntu-latest=$(ACT_IMAGE) -j unit-tests + act -P ubuntu-latest=$(ACT_IMAGE) -j unit-tests --reuse .PHONY: act-integration-tests act-integration-tests: ##@Act Run only integration tests - act -P ubuntu-latest=$(ACT_IMAGE) -j integration-tests + act -P ubuntu-latest=$(ACT_IMAGE) -j integration-tests --reuse .PHONY: act-debug act-debug: ##@Act Run act with verbose logging - act -P ubuntu-latest=$(ACT_IMAGE) --verbose + act -P ubuntu-latest=$(ACT_IMAGE) --verbose --reuse diff --git a/samples/Bookstore/src/Bookstore.sln b/samples/Bookstore/src/Bookstore.sln index 7a95bb0..e6090ba 100644 --- a/samples/Bookstore/src/Bookstore.sln +++ b/samples/Bookstore/src/Bookstore.sln @@ -4,6 +4,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bookstore", "Bookstore\Book EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "OpenDDD", "..\..\..\src\OpenDDD\OpenDDD.csproj", "{05861AFA-45DB-409A-AF0D-E81936198EB0}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "OpenDDD.Tests", "..\..\..\src\OpenDDD.Tests\OpenDDD.Tests.csproj", "{DB152EF6-1F44-40D6-8701-6FB0434724F1}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -18,5 +20,9 @@ Global {05861AFA-45DB-409A-AF0D-E81936198EB0}.Debug|Any CPU.Build.0 = Debug|Any CPU {05861AFA-45DB-409A-AF0D-E81936198EB0}.Release|Any CPU.ActiveCfg = Release|Any CPU {05861AFA-45DB-409A-AF0D-E81936198EB0}.Release|Any CPU.Build.0 = Release|Any CPU + {DB152EF6-1F44-40D6-8701-6FB0434724F1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {DB152EF6-1F44-40D6-8701-6FB0434724F1}.Debug|Any CPU.Build.0 = Debug|Any CPU + {DB152EF6-1F44-40D6-8701-6FB0434724F1}.Release|Any CPU.ActiveCfg = Release|Any CPU + {DB152EF6-1F44-40D6-8701-6FB0434724F1}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection EndGlobal diff --git a/samples/Bookstore/src/Bookstore/Bookstore.csproj b/samples/Bookstore/src/Bookstore/Bookstore.csproj index 35b9a95..5a508ef 100644 --- a/samples/Bookstore/src/Bookstore/Bookstore.csproj +++ b/samples/Bookstore/src/Bookstore/Bookstore.csproj @@ -14,6 +14,7 @@ runtime; build; native; contentfiles; analyzers; buildtransitive + diff --git a/src/OpenDDD/Tests/Base/IntegrationTests.cs b/src/OpenDDD.Tests/Base/IntegrationTests.cs similarity index 100% rename from src/OpenDDD/Tests/Base/IntegrationTests.cs rename to src/OpenDDD.Tests/Base/IntegrationTests.cs diff --git a/src/OpenDDD/Tests/Base/UnitTests.cs b/src/OpenDDD.Tests/Base/UnitTests.cs similarity index 100% rename from src/OpenDDD/Tests/Base/UnitTests.cs rename to src/OpenDDD.Tests/Base/UnitTests.cs diff --git a/src/OpenDDD/Tests/Domain/Model/TestAggregateRoot.cs b/src/OpenDDD.Tests/Domain/Model/TestAggregateRoot.cs similarity index 100% rename from src/OpenDDD/Tests/Domain/Model/TestAggregateRoot.cs rename to src/OpenDDD.Tests/Domain/Model/TestAggregateRoot.cs diff --git a/src/OpenDDD/Tests/Domain/Model/TestEntity.cs b/src/OpenDDD.Tests/Domain/Model/TestEntity.cs similarity index 100% rename from src/OpenDDD/Tests/Domain/Model/TestEntity.cs rename to src/OpenDDD.Tests/Domain/Model/TestEntity.cs diff --git a/src/OpenDDD/Tests/Domain/Model/TestValueObject.cs b/src/OpenDDD.Tests/Domain/Model/TestValueObject.cs similarity index 100% rename from src/OpenDDD/Tests/Domain/Model/TestValueObject.cs rename to src/OpenDDD.Tests/Domain/Model/TestValueObject.cs diff --git a/src/OpenDDD/Tests/Infrastructure/Events/Azure/AzureServiceBusMessagingProviderTests.cs b/src/OpenDDD.Tests/Infrastructure/Events/Azure/AzureServiceBusMessagingProviderTests.cs similarity index 100% rename from src/OpenDDD/Tests/Infrastructure/Events/Azure/AzureServiceBusMessagingProviderTests.cs rename to src/OpenDDD.Tests/Infrastructure/Events/Azure/AzureServiceBusMessagingProviderTests.cs diff --git a/src/OpenDDD/Tests/Infrastructure/Events/DomainPublisherTests.cs b/src/OpenDDD.Tests/Infrastructure/Events/DomainPublisherTests.cs similarity index 99% rename from src/OpenDDD/Tests/Infrastructure/Events/DomainPublisherTests.cs rename to src/OpenDDD.Tests/Infrastructure/Events/DomainPublisherTests.cs index 7c7a64d..951037b 100644 --- a/src/OpenDDD/Tests/Infrastructure/Events/DomainPublisherTests.cs +++ b/src/OpenDDD.Tests/Infrastructure/Events/DomainPublisherTests.cs @@ -2,7 +2,6 @@ using OpenDDD.Domain.Model; using OpenDDD.Infrastructure.Events; using OpenDDD.Tests.Base; -using Xunit; namespace OpenDDD.Tests.Infrastructure.Events { diff --git a/src/OpenDDD/Tests/Infrastructure/Events/EventSerializerTests.cs b/src/OpenDDD.Tests/Infrastructure/Events/EventSerializerTests.cs similarity index 100% rename from src/OpenDDD/Tests/Infrastructure/Events/EventSerializerTests.cs rename to src/OpenDDD.Tests/Infrastructure/Events/EventSerializerTests.cs diff --git a/src/OpenDDD/Tests/Infrastructure/Events/InMemory/InMemoryMessagingProviderTests.cs b/src/OpenDDD.Tests/Infrastructure/Events/InMemory/InMemoryMessagingProviderTests.cs similarity index 100% rename from src/OpenDDD/Tests/Infrastructure/Events/InMemory/InMemoryMessagingProviderTests.cs rename to src/OpenDDD.Tests/Infrastructure/Events/InMemory/InMemoryMessagingProviderTests.cs diff --git a/src/OpenDDD/Tests/Infrastructure/Events/IntegrationPublisherTests.cs b/src/OpenDDD.Tests/Infrastructure/Events/IntegrationPublisherTests.cs similarity index 100% rename from src/OpenDDD/Tests/Infrastructure/Events/IntegrationPublisherTests.cs rename to src/OpenDDD.Tests/Infrastructure/Events/IntegrationPublisherTests.cs diff --git a/src/OpenDDD/Tests/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs b/src/OpenDDD.Tests/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs similarity index 100% rename from src/OpenDDD/Tests/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs rename to src/OpenDDD.Tests/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs diff --git a/src/OpenDDD/Tests/Infrastructure/Events/RabbitMq/RabbitMqMessagingProviderTests.cs b/src/OpenDDD.Tests/Infrastructure/Events/RabbitMq/RabbitMqMessagingProviderTests.cs similarity index 100% rename from src/OpenDDD/Tests/Infrastructure/Events/RabbitMq/RabbitMqMessagingProviderTests.cs rename to src/OpenDDD.Tests/Infrastructure/Events/RabbitMq/RabbitMqMessagingProviderTests.cs diff --git a/src/OpenDDD/Tests/Infrastructure/Events/TestEvent.cs b/src/OpenDDD.Tests/Infrastructure/Events/TestEvent.cs similarity index 100% rename from src/OpenDDD/Tests/Infrastructure/Events/TestEvent.cs rename to src/OpenDDD.Tests/Infrastructure/Events/TestEvent.cs diff --git a/src/OpenDDD/Tests/Infrastructure/Persistence/OpenDdd/Expressions/JsonbExpressionParserTests.cs b/src/OpenDDD.Tests/Infrastructure/Persistence/OpenDdd/Expressions/JsonbExpressionParserTests.cs similarity index 100% rename from src/OpenDDD/Tests/Infrastructure/Persistence/OpenDdd/Expressions/JsonbExpressionParserTests.cs rename to src/OpenDDD.Tests/Infrastructure/Persistence/OpenDdd/Expressions/JsonbExpressionParserTests.cs diff --git a/src/OpenDDD/Tests/Infrastructure/Persistence/OpenDdd/Serializers/OpenDddAggregateSerializerTests.cs b/src/OpenDDD.Tests/Infrastructure/Persistence/OpenDdd/Serializers/OpenDddAggregateSerializerTests.cs similarity index 100% rename from src/OpenDDD/Tests/Infrastructure/Persistence/OpenDdd/Serializers/OpenDddAggregateSerializerTests.cs rename to src/OpenDDD.Tests/Infrastructure/Persistence/OpenDdd/Serializers/OpenDddAggregateSerializerTests.cs diff --git a/src/OpenDDD/Tests/Infrastructure/Persistence/OpenDdd/Serializers/OpenDddSerializerTests.cs b/src/OpenDDD.Tests/Infrastructure/Persistence/OpenDdd/Serializers/OpenDddSerializerTests.cs similarity index 100% rename from src/OpenDDD/Tests/Infrastructure/Persistence/OpenDdd/Serializers/OpenDddSerializerTests.cs rename to src/OpenDDD.Tests/Infrastructure/Persistence/OpenDdd/Serializers/OpenDddSerializerTests.cs diff --git a/src/OpenDDD.Tests/OpenDDD.Tests.csproj b/src/OpenDDD.Tests/OpenDDD.Tests.csproj new file mode 100644 index 0000000..2070590 --- /dev/null +++ b/src/OpenDDD.Tests/OpenDDD.Tests.csproj @@ -0,0 +1,29 @@ + + + + net8.0 + enable + enable + + false + true + + + + + + + + + + + + + + + + + + + + diff --git a/src/OpenDDD.sln b/src/OpenDDD.sln index 47ad203..e6015c7 100644 --- a/src/OpenDDD.sln +++ b/src/OpenDDD.sln @@ -1,7 +1,10 @@  Microsoft Visual Studio Solution File, Format Version 12.00 +# Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "OpenDDD", "OpenDDD\OpenDDD.csproj", "{B5482293-BC97-4009-B3AD-2B0789526E52}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "OpenDDD.Tests", "OpenDDD.Tests\OpenDDD.Tests.csproj", "{75DD96D0-3849-4F8F-BC7D-CBE2560D9471}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -12,5 +15,9 @@ Global {B5482293-BC97-4009-B3AD-2B0789526E52}.Debug|Any CPU.Build.0 = Debug|Any CPU {B5482293-BC97-4009-B3AD-2B0789526E52}.Release|Any CPU.ActiveCfg = Release|Any CPU {B5482293-BC97-4009-B3AD-2B0789526E52}.Release|Any CPU.Build.0 = Release|Any CPU + {75DD96D0-3849-4F8F-BC7D-CBE2560D9471}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {75DD96D0-3849-4F8F-BC7D-CBE2560D9471}.Debug|Any CPU.Build.0 = Debug|Any CPU + {75DD96D0-3849-4F8F-BC7D-CBE2560D9471}.Release|Any CPU.ActiveCfg = Release|Any CPU + {75DD96D0-3849-4F8F-BC7D-CBE2560D9471}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection EndGlobal diff --git a/src/OpenDDD/OpenDDD.csproj b/src/OpenDDD/OpenDDD.csproj index b17d9ae..fd6eff6 100644 --- a/src/OpenDDD/OpenDDD.csproj +++ b/src/OpenDDD/OpenDDD.csproj @@ -19,13 +19,10 @@ - - - @@ -33,7 +30,6 @@ - From c3d183c885ec55efb75b271d3a15c01a83cd8ef2 Mon Sep 17 00:00:00 2001 From: David Runemalm Date: Thu, 20 Feb 2025 18:04:49 +0100 Subject: [PATCH 037/109] Filter tests by 'Trait'. --- .github/workflows/tests.yml | 4 ++-- Makefile | 4 ++-- src/OpenDDD.Tests/Base/IntegrationTests.cs | 1 + src/OpenDDD.Tests/Base/UnitTests.cs | 3 ++- .../Events/Azure/AzureServiceBusMessagingProviderTests.cs | 1 - 5 files changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 987b345..dda7961 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -32,7 +32,7 @@ jobs: - name: Run Unit Tests working-directory: src/OpenDDD.Tests - run: dotnet test --no-build --configuration Release --filter FullyQualifiedName~UnitTests + run: dotnet test --no-build --configuration Release --filter "Category=Unit" # integration-tests: # name: Integration Tests (.NET ${{ matrix.dotnet-version }}) @@ -84,4 +84,4 @@ jobs: # KAFKA_BROKER: localhost:9092 # RABBITMQ_HOST: localhost # AZURE_STORAGE_CONNECTION_STRING: "UseDevelopmentStorage=true;" - # run: dotnet test --no-build --configuration Release --filter FullyQualifiedName~IntegrationTests + # run: dotnet test --no-build --configuration Release --filter "Category=Integration" diff --git a/Makefile b/Makefile index 910e52a..31f2479 100644 --- a/Makefile +++ b/Makefile @@ -100,11 +100,11 @@ test: ##@Test Run all tests (unit & integration) .PHONY: test-unit test-unit: ##@Test Run only unit tests - cd $(TESTS_DIR) && dotnet test --configuration Release --filter FullyQualifiedName~UnitTests + cd $(TESTS_DIR) && dotnet test --configuration Release --filter "Category=Unit" .PHONY: test-integration test-integration: ##@Test Run only integration tests - cd $(TESTS_DIR) && dotnet test --configuration Release --filter FullyQualifiedName~IntegrationTests + cd $(TESTS_DIR) && dotnet test --configuration Release --filter "Category=Integration" ########################################################################## # BUILD diff --git a/src/OpenDDD.Tests/Base/IntegrationTests.cs b/src/OpenDDD.Tests/Base/IntegrationTests.cs index 8ae87aa..525f8f2 100644 --- a/src/OpenDDD.Tests/Base/IntegrationTests.cs +++ b/src/OpenDDD.Tests/Base/IntegrationTests.cs @@ -1,5 +1,6 @@ namespace OpenDDD.Tests.Base { + [Trait("Category", "Integration")] public class IntegrationTests { diff --git a/src/OpenDDD.Tests/Base/UnitTests.cs b/src/OpenDDD.Tests/Base/UnitTests.cs index ee4e2b8..67be01b 100644 --- a/src/OpenDDD.Tests/Base/UnitTests.cs +++ b/src/OpenDDD.Tests/Base/UnitTests.cs @@ -1,6 +1,7 @@ namespace OpenDDD.Tests.Base { - public class UnitTests + [Trait("Category", "Unit")] + public abstract class UnitTests { } diff --git a/src/OpenDDD.Tests/Infrastructure/Events/Azure/AzureServiceBusMessagingProviderTests.cs b/src/OpenDDD.Tests/Infrastructure/Events/Azure/AzureServiceBusMessagingProviderTests.cs index cd39801..01fd677 100644 --- a/src/OpenDDD.Tests/Infrastructure/Events/Azure/AzureServiceBusMessagingProviderTests.cs +++ b/src/OpenDDD.Tests/Infrastructure/Events/Azure/AzureServiceBusMessagingProviderTests.cs @@ -1,6 +1,5 @@ using System.Text; using Microsoft.Extensions.Logging; -using Xunit; using Moq; using Azure; using Azure.Messaging.ServiceBus; From e75c92263f879f9d907bfaf8311b41ae14c7ee17 Mon Sep 17 00:00:00 2001 From: David Runemalm Date: Thu, 20 Feb 2025 18:13:13 +0100 Subject: [PATCH 038/109] Publish test report. --- .github/workflows/tests.yml | 138 ++++++++++++++++++++++-------------- 1 file changed, 84 insertions(+), 54 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index dda7961..89667f9 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - dotnet-version: [8.0.x] + dotnet-version: [8.0.x, 9.0.x] steps: - name: Checkout Repository @@ -32,56 +32,86 @@ jobs: - name: Run Unit Tests working-directory: src/OpenDDD.Tests - run: dotnet test --no-build --configuration Release --filter "Category=Unit" - - # integration-tests: - # name: Integration Tests (.NET ${{ matrix.dotnet-version }}) - # runs-on: ubuntu-latest - # strategy: - # matrix: - # dotnet-version: [8.0.x, 9.0.x] - - # services: - # kafka: - # image: confluentinc/cp-kafka:latest - # env: - # KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:9092 - # KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 - # ports: - # - 9092:9092 - # options: --network-alias kafka - - # rabbitmq: - # image: rabbitmq:3-management - # ports: - # - 5672:5672 - # - 15672:15672 - - # azurite: - # image: mcr.microsoft.com/azure-storage/azurite - # ports: - # - 10000:10000 - # - 10001:10001 - # - 10002:10002 - - # steps: - # - name: Checkout Repository - # uses: actions/checkout@v4 - - # - name: Setup .NET - # uses: actions/setup-dotnet@v3 - # with: - # dotnet-version: ${{ matrix.dotnet-version }} - - # - name: Restore Dependencies - # run: dotnet restore - - # - name: Build Project - # run: dotnet build --no-restore --configuration Release - - # - name: Run Integration Tests - # env: - # KAFKA_BROKER: localhost:9092 - # RABBITMQ_HOST: localhost - # AZURE_STORAGE_CONNECTION_STRING: "UseDevelopmentStorage=true;" - # run: dotnet test --no-build --configuration Release --filter "Category=Integration" + run: dotnet test --no-build --configuration Release --filter "Category=Unit" --logger "trx;LogFileName=TestResults.trx" + + - name: Upload Unit Test Results + if: always() + uses: actions/upload-artifact@v4 + with: + name: UnitTestResults + path: src/OpenDDD.Tests/TestResults.trx + + - name: Publish Unit Test Report + if: always() + uses: dorny/test-reporter@v1 + with: + name: Unit Tests Report + path: src/OpenDDD.Tests/TestResults.trx + reporter: dotnet-trx + + integration-tests: + name: Integration Tests (.NET ${{ matrix.dotnet-version }}) + runs-on: ubuntu-latest + strategy: + matrix: + dotnet-version: [8.0.x, 9.0.x] + + services: + kafka: + image: confluentinc/cp-kafka:latest + env: + KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:9092 + KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + ports: + - 9092:9092 + options: --network-alias kafka + + rabbitmq: + image: rabbitmq:3-management + ports: + - 5672:5672 + - 15672:15672 + + azurite: + image: mcr.microsoft.com/azure-storage/azurite + ports: + - 10000:10000 + - 10001:10001 + - 10002:10002 + + steps: + - name: Checkout Repository + uses: actions/checkout@v4 + + - name: Setup .NET + uses: actions/setup-dotnet@v3 + with: + dotnet-version: ${{ matrix.dotnet-version }} + + - name: Restore Dependencies + run: dotnet restore + + - name: Build Project + run: dotnet build --no-restore --configuration Release + + - name: Run Integration Tests + env: + KAFKA_BROKER: localhost:9092 + RABBITMQ_HOST: localhost + AZURE_STORAGE_CONNECTION_STRING: "UseDevelopmentStorage=true;" + run: dotnet test --no-build --configuration Release --filter "Category=Integration" --logger "trx;LogFileName=TestResults.trx" + + - name: Upload Integration Test Results + if: always() + uses: actions/upload-artifact@v4 + with: + name: IntegrationTestResults + path: TestResults.trx + + - name: Publish Integration Test Report + if: always() + uses: dorny/test-reporter@v1 + with: + name: Integration Tests Report + path: TestResults.trx + reporter: dotnet-trx From ad62e568e9ba1bcb0910ae2268e3bad88793f59c Mon Sep 17 00:00:00 2001 From: David Runemalm Date: Thu, 20 Feb 2025 18:18:58 +0100 Subject: [PATCH 039/109] Remove .NET 9 from matrix. --- .github/workflows/tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 89667f9..a143651 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - dotnet-version: [8.0.x, 9.0.x] + dotnet-version: [8.0.x] steps: - name: Checkout Repository @@ -54,7 +54,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - dotnet-version: [8.0.x, 9.0.x] + dotnet-version: [8.0.x] services: kafka: From 24a7bad85ca52db064dafa4db5458dd4e7fcfdf1 Mon Sep 17 00:00:00 2001 From: David Runemalm Date: Thu, 20 Feb 2025 18:29:39 +0100 Subject: [PATCH 040/109] Update test reports path. --- .github/workflows/tests.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index a143651..da57b30 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -32,21 +32,21 @@ jobs: - name: Run Unit Tests working-directory: src/OpenDDD.Tests - run: dotnet test --no-build --configuration Release --filter "Category=Unit" --logger "trx;LogFileName=TestResults.trx" + run: dotnet test --no-build --configuration Release --filter "Category=Unit" --logger "trx;LogFileName=TestResults.trx" --results-directory TestResults - name: Upload Unit Test Results if: always() uses: actions/upload-artifact@v4 with: name: UnitTestResults - path: src/OpenDDD.Tests/TestResults.trx + path: src/OpenDDD.Tests/TestResults/TestResults.trx - name: Publish Unit Test Report if: always() uses: dorny/test-reporter@v1 with: name: Unit Tests Report - path: src/OpenDDD.Tests/TestResults.trx + path: src/OpenDDD.Tests/TestResults/TestResults.trx reporter: dotnet-trx integration-tests: @@ -99,19 +99,19 @@ jobs: KAFKA_BROKER: localhost:9092 RABBITMQ_HOST: localhost AZURE_STORAGE_CONNECTION_STRING: "UseDevelopmentStorage=true;" - run: dotnet test --no-build --configuration Release --filter "Category=Integration" --logger "trx;LogFileName=TestResults.trx" + run: dotnet test --no-build --configuration Release --filter "Category=Integration" --logger "trx;LogFileName=TestResults.trx" --results-directory TestResults - name: Upload Integration Test Results if: always() uses: actions/upload-artifact@v4 with: name: IntegrationTestResults - path: TestResults.trx + path: src/OpenDDD.Tests/TestResults/TestResults.trx - name: Publish Integration Test Report if: always() uses: dorny/test-reporter@v1 with: name: Integration Tests Report - path: TestResults.trx + path: src/OpenDDD.Tests/TestResults/TestResults.trx reporter: dotnet-trx From 127d70f9e62aeabda8ae60ac29ef292c60f2707f Mon Sep 17 00:00:00 2001 From: David Runemalm Date: Thu, 20 Feb 2025 18:34:04 +0100 Subject: [PATCH 041/109] Fix working directory in integration tests. --- .github/workflows/tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index da57b30..03731a0 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -89,12 +89,15 @@ jobs: dotnet-version: ${{ matrix.dotnet-version }} - name: Restore Dependencies + working-directory: src/OpenDDD.Tests run: dotnet restore - name: Build Project + working-directory: src/OpenDDD.Tests run: dotnet build --no-restore --configuration Release - name: Run Integration Tests + working-directory: src/OpenDDD.Tests env: KAFKA_BROKER: localhost:9092 RABBITMQ_HOST: localhost From 92481f33331decf3406380cf94e34326f42b6a34 Mon Sep 17 00:00:00 2001 From: David Runemalm Date: Thu, 20 Feb 2025 18:45:55 +0100 Subject: [PATCH 042/109] Add tests badge to README. --- .github/workflows/tests.yml | 2 +- README.md | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 03731a0..99acd08 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -1,4 +1,4 @@ -name: Run Tests +name: Tests on: push: diff --git a/README.md b/README.md index 86d5de4..1d8a5e8 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,6 @@ # OpenDDD.NET -[![License](https://img.shields.io/badge/License-GPLv3-blue.svg)](https://www.gnu.org/licenses/gpl-3.0.html) -[![NuGet](https://img.shields.io/nuget/v/OpenDDD.NET.svg)](https://www.nuget.org/packages/OpenDDD.NET/) +[![License](https://img.shields.io/badge/License-GPLv3-blue.svg)](https://www.gnu.org/licenses/gpl-3.0.html) [![NuGet](https://img.shields.io/nuget/v/OpenDDD.NET.svg)](https://www.nuget.org/packages/OpenDDD.NET/) [![Run Tests](https://github.com/runemalm/OpenDDD.NET/actions/workflows/tests.yml/badge.svg?branch=develop)](https://github.com/runemalm/OpenDDD.NET/actions/workflows/tests.yml) OpenDDD.NET is an open-source framework for domain-driven design (DDD) development using C# and ASP.NET Core. It provides a set of powerful tools and abstractions to help developers build scalable, maintainable, and testable applications following the principles of DDD. From ec46e8a2505bc423ad3b412d9b6c9d8442a381ae Mon Sep 17 00:00:00 2001 From: David Runemalm Date: Thu, 20 Feb 2025 19:09:59 +0100 Subject: [PATCH 043/109] Upload unit tests badge to shields.io. --- .github/workflows/tests.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 99acd08..8e0063d 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -49,6 +49,20 @@ jobs: path: src/OpenDDD.Tests/TestResults/TestResults.trx reporter: dotnet-trx + - name: Generate Test Summary + if: always() + run: | + PASSED=$(grep -oP '(?<=Passed=")[0-9]+' src/OpenDDD.Tests/TestResults/TestResults.trx || echo 0) + FAILED=$(grep -oP '(?<=Failed=")[0-9]+' src/OpenDDD.Tests/TestResults/TestResults.trx || echo 0) + echo "{ \"schemaVersion\": 1, \"label\": \"Unit Tests\", \"message\": \"✅ $PASSED | ❌ $FAILED\", \"color\": \"blue\" }" > unit-tests-badge.json + + - name: Upload Unit Test Badge + if: always() + uses: actions/upload-artifact@v4 + with: + name: unit-tests-badge + path: unit-tests-badge.json + integration-tests: name: Integration Tests (.NET ${{ matrix.dotnet-version }}) runs-on: ubuntu-latest From 02e0567273a428897f1b71dcd7cbb781726a2c09 Mon Sep 17 00:00:00 2001 From: David Runemalm Date: Thu, 20 Feb 2025 19:16:09 +0100 Subject: [PATCH 044/109] Upload badge to gh-pages. --- .github/workflows/tests.yml | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 8e0063d..52c616e 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -49,19 +49,23 @@ jobs: path: src/OpenDDD.Tests/TestResults/TestResults.trx reporter: dotnet-trx - - name: Generate Test Summary + - name: Generate Unit Test Badge if: always() run: | PASSED=$(grep -oP '(?<=Passed=")[0-9]+' src/OpenDDD.Tests/TestResults/TestResults.trx || echo 0) FAILED=$(grep -oP '(?<=Failed=")[0-9]+' src/OpenDDD.Tests/TestResults/TestResults.trx || echo 0) echo "{ \"schemaVersion\": 1, \"label\": \"Unit Tests\", \"message\": \"✅ $PASSED | ❌ $FAILED\", \"color\": \"blue\" }" > unit-tests-badge.json - - name: Upload Unit Test Badge + - name: Commit and Push Badge to GitHub Pages if: always() - uses: actions/upload-artifact@v4 - with: - name: unit-tests-badge - path: unit-tests-badge.json + run: | + git config --global user.name "github-actions" + git config --global user.email "github-actions@github.com" + git checkout gh-pages || git checkout -b gh-pages + mv unit-tests-badge.json badges/unit-tests-badge.json + git add badges/unit-tests-badge.json + git commit -m "Update unit test badge" + git push origin gh-pages integration-tests: name: Integration Tests (.NET ${{ matrix.dotnet-version }}) From 3b3b5558b0ac49d7d985249ac21b08fd7656a04b Mon Sep 17 00:00:00 2001 From: David Runemalm Date: Thu, 20 Feb 2025 19:26:48 +0100 Subject: [PATCH 045/109] Trigger build. --- .github/workflows/tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 52c616e..fda0ba9 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -56,6 +56,7 @@ jobs: FAILED=$(grep -oP '(?<=Failed=")[0-9]+' src/OpenDDD.Tests/TestResults/TestResults.trx || echo 0) echo "{ \"schemaVersion\": 1, \"label\": \"Unit Tests\", \"message\": \"✅ $PASSED | ❌ $FAILED\", \"color\": \"blue\" }" > unit-tests-badge.json + - name: Commit and Push Badge to GitHub Pages if: always() run: | From 997e6810f2452f73adf4b63cdbc3ff9714801c7e Mon Sep 17 00:00:00 2001 From: David Runemalm Date: Thu, 20 Feb 2025 19:30:04 +0100 Subject: [PATCH 046/109] Checkout gh-pages in workflow. --- .github/workflows/tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index fda0ba9..d57b523 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -56,13 +56,13 @@ jobs: FAILED=$(grep -oP '(?<=Failed=")[0-9]+' src/OpenDDD.Tests/TestResults/TestResults.trx || echo 0) echo "{ \"schemaVersion\": 1, \"label\": \"Unit Tests\", \"message\": \"✅ $PASSED | ❌ $FAILED\", \"color\": \"blue\" }" > unit-tests-badge.json - - name: Commit and Push Badge to GitHub Pages if: always() run: | git config --global user.name "github-actions" git config --global user.email "github-actions@github.com" - git checkout gh-pages || git checkout -b gh-pages + git fetch origin gh-pages || echo "No gh-pages branch found" + git checkout gh-pages mv unit-tests-badge.json badges/unit-tests-badge.json git add badges/unit-tests-badge.json git commit -m "Update unit test badge" From 37f1ef89312e0a98712c6402f38f4d38082d3266 Mon Sep 17 00:00:00 2001 From: David Runemalm Date: Thu, 20 Feb 2025 19:40:54 +0100 Subject: [PATCH 047/109] Remove badge generation from workflow. --- .github/workflows/tests.yml | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index d57b523..99acd08 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -49,25 +49,6 @@ jobs: path: src/OpenDDD.Tests/TestResults/TestResults.trx reporter: dotnet-trx - - name: Generate Unit Test Badge - if: always() - run: | - PASSED=$(grep -oP '(?<=Passed=")[0-9]+' src/OpenDDD.Tests/TestResults/TestResults.trx || echo 0) - FAILED=$(grep -oP '(?<=Failed=")[0-9]+' src/OpenDDD.Tests/TestResults/TestResults.trx || echo 0) - echo "{ \"schemaVersion\": 1, \"label\": \"Unit Tests\", \"message\": \"✅ $PASSED | ❌ $FAILED\", \"color\": \"blue\" }" > unit-tests-badge.json - - - name: Commit and Push Badge to GitHub Pages - if: always() - run: | - git config --global user.name "github-actions" - git config --global user.email "github-actions@github.com" - git fetch origin gh-pages || echo "No gh-pages branch found" - git checkout gh-pages - mv unit-tests-badge.json badges/unit-tests-badge.json - git add badges/unit-tests-badge.json - git commit -m "Update unit test badge" - git push origin gh-pages - integration-tests: name: Integration Tests (.NET ${{ matrix.dotnet-version }}) runs-on: ubuntu-latest From a59d82fb96fb53320a368bcb78a52810837bbb97 Mon Sep 17 00:00:00 2001 From: David Runemalm Date: Fri, 21 Feb 2025 12:41:59 +0100 Subject: [PATCH 048/109] Add integration tests for azure provider. --- .github/workflows/tests.yml | 40 +++++-- Makefile | 37 +++++- .../AzureServiceBusMessagingProviderTests.cs | 110 ++++++++++++++++++ .../Azure/AzureServiceBusTestsCollection.cs | 5 + .../Azure/AzureServiceBusMessagingProvider.cs | 7 ++ 5 files changed, 190 insertions(+), 9 deletions(-) create mode 100644 src/OpenDDD.Tests/Integration/Infrastructure/Events/Azure/AzureServiceBusMessagingProviderTests.cs create mode 100644 src/OpenDDD.Tests/Integration/Infrastructure/Events/Azure/AzureServiceBusTestsCollection.cs diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 99acd08..9ccb91f 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -72,13 +72,6 @@ jobs: - 5672:5672 - 15672:15672 - azurite: - image: mcr.microsoft.com/azure-storage/azurite - ports: - - 10000:10000 - - 10001:10001 - - 10002:10002 - steps: - name: Checkout Repository uses: actions/checkout@v4 @@ -96,14 +89,45 @@ jobs: working-directory: src/OpenDDD.Tests run: dotnet build --no-restore --configuration Release + - name: Log in to Azure + uses: azure/login@v1 + with: + creds: ${{ secrets.AZURE_CREDENTIALS }} + + - name: Create Azure Service Bus namespace for Testing + run: | + NAMESPACE_NAME="test-servicebus-${{ github.run_id }}-${{ github.run_attempt }}" + echo "NAMESPACE_NAME=${NAMESPACE_NAME}" >> $GITHUB_ENV + az servicebus namespace create \ + --resource-group opendddnet \ + --name $NAMESPACE_NAME \ + --location northeurope + CONNECTION_STRING=$(az servicebus namespace authorization-rule keys list \ + --resource-group opendddnet \ + --namespace-name $NAMESPACE_NAME \ + --name RootManageSharedAccessKey \ + --query primaryConnectionString \ + -o tsv) + echo "AZURE_SERVICE_BUS_CONNECTION_STRING=${CONNECTION_STRING}" >> $GITHUB_ENV + - name: Run Integration Tests working-directory: src/OpenDDD.Tests env: KAFKA_BROKER: localhost:9092 RABBITMQ_HOST: localhost - AZURE_STORAGE_CONNECTION_STRING: "UseDevelopmentStorage=true;" + AZURE_SERVICE_BUS_CONNECTION_STRING: ${{ env.AZURE_SERVICE_BUS_CONNECTION_STRING }} run: dotnet test --no-build --configuration Release --filter "Category=Integration" --logger "trx;LogFileName=TestResults.trx" --results-directory TestResults + - name: Delete Azure Service Bus namespace After Tests + if: always() + run: | + if [[ -n "${NAMESPACE_NAME}" ]]; then + echo "Deleting namespace: $NAMESPACE_NAME" + az servicebus namespace delete --resource-group opendddnet --name $NAMESPACE_NAME + else + echo "No namespace found, skipping deletion." + fi + - name: Upload Integration Test Results if: always() uses: actions/upload-artifact@v4 diff --git a/Makefile b/Makefile index 31f2479..72d8ca8 100644 --- a/Makefile +++ b/Makefile @@ -257,8 +257,43 @@ act-unit-tests: ##@Act Run only unit tests .PHONY: act-integration-tests act-integration-tests: ##@Act Run only integration tests - act -P ubuntu-latest=$(ACT_IMAGE) -j integration-tests --reuse + act -P ubuntu-latest=$(ACT_IMAGE) -j integration-tests --reuse -s AZURE_SERVICE_BUS_CONNECTION_STRING=$(AZURE_SERVICE_BUS_CONNECTION_STRING) .PHONY: act-debug act-debug: ##@Act Run act with verbose logging act -P ubuntu-latest=$(ACT_IMAGE) --verbose --reuse + +########################################################################## +# AZURE +########################################################################## + +.PHONY: azure-create-resource-group +azure-create-resource-group: ##@Azure Create the Azure Resource Group + az group create --name $(AZURE_RESOURCE_GROUP) --location $(AZURE_REGION) + +.PHONY: azure-create-service-principal +azure-create-service-principal: ##@Azure Create an Azure Service Principal for GitHub Actions + @echo "Creating Azure Service Principal..." + az ad sp create-for-rbac \ + --name "github-actions-opendddnet" \ + --role "Contributor" \ + --scopes /subscriptions/$(AZURE_SUBSCRIPTION_ID)/resourceGroups/$(AZURE_RESOURCE_GROUP) \ + --sdk-auth + @echo "✅ Copy the output above and add it as 'AZURE_CREDENTIALS' in GitHub Secrets." + +.PHONY: azure-create-servicebus-namespace +azure-create-servicebus-namespace: ##@Azure Create the Azure Service Bus namespace + az servicebus namespace create --name $(AZURE_SERVICEBUS_NAMESPACE) --resource-group $(AZURE_RESOURCE_GROUP) --location $(AZURE_REGION) --sku Standard + +.PHONY: azure-get-servicebus-connection +azure-get-servicebus-connection: ##@Azure Get the Service Bus connection string + az servicebus namespace authorization-rule keys list \ + --resource-group $(AZURE_RESOURCE_GROUP) \ + --namespace-name $(AZURE_SERVICEBUS_NAMESPACE) \ + --name RootManageSharedAccessKey \ + --query primaryConnectionString \ + --output tsv + +.PHONY: azure-delete-servicebus-namespace +azure-delete-servicebus: ##@Azure Delete the Azure Service Bus namespace + az servicebus namespace delete --resource-group $(AZURE_RESOURCE_GROUP) --name $(AZURE_SERVICE_BUS_NAMESPACE) diff --git a/src/OpenDDD.Tests/Integration/Infrastructure/Events/Azure/AzureServiceBusMessagingProviderTests.cs b/src/OpenDDD.Tests/Integration/Infrastructure/Events/Azure/AzureServiceBusMessagingProviderTests.cs new file mode 100644 index 0000000..47f591c --- /dev/null +++ b/src/OpenDDD.Tests/Integration/Infrastructure/Events/Azure/AzureServiceBusMessagingProviderTests.cs @@ -0,0 +1,110 @@ +using Microsoft.Extensions.Logging; +using Moq; +using OpenDDD.Infrastructure.Events.Azure; +using OpenDDD.Tests.Base; +using Azure.Messaging.ServiceBus; +using Azure.Messaging.ServiceBus.Administration; + +namespace OpenDDD.Tests.Integration.Infrastructure.Events.Azure +{ + [Collection("AzureServiceBusTests")] // Ensure tests run sequentially + public class AzureServiceBusMessagingProviderTests : IntegrationTests, IAsyncLifetime + { + private readonly string _connectionString; + private readonly ServiceBusAdministrationClient _adminClient; + private readonly Mock> _loggerMock; + private readonly ServiceBusClient _serviceBusClient; + private readonly AzureServiceBusMessagingProvider _messagingProvider; + + public AzureServiceBusMessagingProviderTests() + { + _connectionString = Environment.GetEnvironmentVariable("AZURE_SERVICE_BUS_CONNECTION_STRING") + ?? throw new InvalidOperationException("AZURE_SERVICE_BUS_CONNECTION_STRING is not set."); + + _adminClient = new ServiceBusAdministrationClient(_connectionString); + _serviceBusClient = new ServiceBusClient(_connectionString); + _loggerMock = new Mock>(); + + _messagingProvider = new AzureServiceBusMessagingProvider( + _serviceBusClient, + _adminClient, + autoCreateTopics: true, + _loggerMock.Object); + } + + public async Task InitializeAsync() + { + // Cleanup any test topics/subscriptions from previous runs + await CleanupTopicsAndSubscriptionsAsync(); + } + + public async Task DisposeAsync() + { + + } + + private async Task CleanupTopicsAndSubscriptionsAsync() + { + var topics = _adminClient.GetTopicsAsync(); + await foreach (var topic in topics) + { + if (topic.Name.StartsWith("test-topic-")) + { + var subscriptions = _adminClient.GetSubscriptionsAsync(topic.Name); + await foreach (var sub in subscriptions) + { + await _adminClient.DeleteSubscriptionAsync(topic.Name, sub.SubscriptionName); + } + + await _adminClient.DeleteTopicAsync(topic.Name); + } + } + } + + [Fact] + public async Task SubscribeAsync_WhenAutoCreateTopicsIsEnabled_ShouldCreateTopicIfNotExists() + { + // Arrange + var topicName = $"test-topic-{Guid.NewGuid()}"; + + // Ensure topic does not exist before the test + if (await _adminClient.TopicExistsAsync(topicName)) + { + await _adminClient.DeleteTopicAsync(topicName); + } + + // Act + await _messagingProvider.SubscribeAsync(topicName, "test-subscriber", (msg, token) => Task.CompletedTask); + + // Assert + Assert.True(await _adminClient.TopicExistsAsync(topicName)); + } + + [Fact] + public async Task SubscribeAsync_WhenAutoCreateTopicsIsDisabled_ShouldNotCreateTopic_AndLogErrorWhenTopicNotExist() + { + // Arrange + var topicName = $"test-topic-{Guid.NewGuid()}"; + + // Ensure topic does not exist before the test + if (await _adminClient.TopicExistsAsync(topicName)) + { + await _adminClient.DeleteTopicAsync(topicName); + } + + var messagingProvider = new AzureServiceBusMessagingProvider( + _serviceBusClient, + _adminClient, + autoCreateTopics: false, + _loggerMock.Object); + + // Act & Assert + var exception = await Assert.ThrowsAsync(async () => + { + await messagingProvider.SubscribeAsync(topicName, "test-subscriber", (msg, token) => Task.CompletedTask); + }); + + Assert.Equal($"Cannot subscribe to topic '{topicName}' because it does not exist and auto-creation is disabled.", exception.Message); + } + } +} diff --git a/src/OpenDDD.Tests/Integration/Infrastructure/Events/Azure/AzureServiceBusTestsCollection.cs b/src/OpenDDD.Tests/Integration/Infrastructure/Events/Azure/AzureServiceBusTestsCollection.cs new file mode 100644 index 0000000..f9cfe4f --- /dev/null +++ b/src/OpenDDD.Tests/Integration/Infrastructure/Events/Azure/AzureServiceBusTestsCollection.cs @@ -0,0 +1,5 @@ +namespace OpenDDD.Tests.Integration.Infrastructure.Events.Azure +{ + [CollectionDefinition("AzureServiceBusTests", DisableParallelization = true)] + public class AzureServiceBusTestsCollection { } +} diff --git a/src/OpenDDD/Infrastructure/Events/Azure/AzureServiceBusMessagingProvider.cs b/src/OpenDDD/Infrastructure/Events/Azure/AzureServiceBusMessagingProvider.cs index ef52942..3b68f12 100644 --- a/src/OpenDDD/Infrastructure/Events/Azure/AzureServiceBusMessagingProvider.cs +++ b/src/OpenDDD/Infrastructure/Events/Azure/AzureServiceBusMessagingProvider.cs @@ -48,6 +48,13 @@ public async Task SubscribeAsync(string topic, string consumerGroup, Func Date: Fri, 21 Feb 2025 12:49:46 +0100 Subject: [PATCH 049/109] Remove azure unit tests that we are replacing with integration tests. --- .../AzureServiceBusMessagingProviderTests.cs | 59 +------------------ 1 file changed, 1 insertion(+), 58 deletions(-) diff --git a/src/OpenDDD.Tests/Infrastructure/Events/Azure/AzureServiceBusMessagingProviderTests.cs b/src/OpenDDD.Tests/Infrastructure/Events/Azure/AzureServiceBusMessagingProviderTests.cs index 01fd677..4443d3a 100644 --- a/src/OpenDDD.Tests/Infrastructure/Events/Azure/AzureServiceBusMessagingProviderTests.cs +++ b/src/OpenDDD.Tests/Infrastructure/Events/Azure/AzureServiceBusMessagingProviderTests.cs @@ -1,5 +1,4 @@ -using System.Text; -using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging; using Moq; using Azure; using Azure.Messaging.ServiceBus; @@ -113,61 +112,5 @@ public async Task PublishAsync_ShouldThrowException_WhenMessageIsInvalid(string await Assert.ThrowsAsync(() => _provider.PublishAsync(_testTopic, invalidMessage, CancellationToken.None)); } - - [Fact] - public async Task SubscribeAsync_ShouldCreateTopicIfNotExists_WhenAutoCreateEnabled() - { - // Arrange: topic don't exist - _mockAdminClient.Setup(admin => admin.TopicExistsAsync(_testTopic, It.IsAny())) - .ReturnsAsync(Response.FromValue(false, Mock.Of())); - - // Act - await _provider.SubscribeAsync(_testTopic, _testSubscription, (msg, token) => Task.CompletedTask, CancellationToken.None); - - // Assert: create-topic was called - _mockAdminClient.Verify(admin => admin.CreateTopicAsync(_testTopic, It.IsAny()), Times.Once); - } - - [Fact] - public async Task SubscribeAsync_ShouldCreateSubscriptionIfNotExists() - { - // Arrange: subscription don't exist - _mockAdminClient.Setup(admin => admin.SubscriptionExistsAsync(_testTopic, _testSubscription, It.IsAny())) - .ReturnsAsync(Response.FromValue(false, Mock.Of())); - - // Act - await _provider.SubscribeAsync(_testTopic, _testSubscription, (msg, token) => Task.CompletedTask, CancellationToken.None); - - // Assert: create-subscription was called - _mockAdminClient.Verify(admin => admin.CreateSubscriptionAsync(_testTopic, _testSubscription, It.IsAny()), Times.Once); - } - - [Fact] - public async Task SubscribeAsync_ShouldStartProcessingMessages() - { - // Arrange: no-op handler - Func handler = (msg, token) => Task.CompletedTask; - - // Act - await _provider.SubscribeAsync(_testTopic, _testSubscription, handler, CancellationToken.None); - - // Assert: start-processing was called - _mockProcessor.Verify(processor => processor.StartProcessingAsync(It.IsAny()), Times.Once); - } - - [Fact] - public async Task PublishAsync_ShouldSendMessageToTopic() - { - // Arrange - var testMessage = "Hello, Azure Service Bus!"; - - // Act - await _provider.PublishAsync(_testTopic, testMessage, CancellationToken.None); - - // Assert: sender was called with the message - _mockSender.Verify(sender => sender.SendMessageAsync(It.Is( - msg => Encoding.UTF8.GetString(msg.Body.ToArray()) == testMessage), - It.IsAny()), Times.Once); - } } } From c07d6ad79739839f0a5671638d3fe8110124ce3f Mon Sep 17 00:00:00 2001 From: David Runemalm Date: Fri, 21 Feb 2025 13:00:24 +0100 Subject: [PATCH 050/109] Assert topic wasn't created in test. --- Makefile | 4 ++-- README.md | 2 +- .../Events/Azure/AzureServiceBusMessagingProviderTests.cs | 4 +++- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 72d8ca8..4ac4dc5 100644 --- a/Makefile +++ b/Makefile @@ -295,5 +295,5 @@ azure-get-servicebus-connection: ##@Azure Get the Service Bus connection string --output tsv .PHONY: azure-delete-servicebus-namespace -azure-delete-servicebus: ##@Azure Delete the Azure Service Bus namespace - az servicebus namespace delete --resource-group $(AZURE_RESOURCE_GROUP) --name $(AZURE_SERVICE_BUS_NAMESPACE) +azure-delete-servicebus-namespace: ##@Azure Delete the Azure Service Bus namespace + az servicebus namespace delete --resource-group $(AZURE_RESOURCE_GROUP) --name $(AZURE_SERVICEBUS_NAMESPACE) diff --git a/README.md b/README.md index 1d8a5e8..b07dbf0 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # OpenDDD.NET -[![License](https://img.shields.io/badge/License-GPLv3-blue.svg)](https://www.gnu.org/licenses/gpl-3.0.html) [![NuGet](https://img.shields.io/nuget/v/OpenDDD.NET.svg)](https://www.nuget.org/packages/OpenDDD.NET/) [![Run Tests](https://github.com/runemalm/OpenDDD.NET/actions/workflows/tests.yml/badge.svg?branch=develop)](https://github.com/runemalm/OpenDDD.NET/actions/workflows/tests.yml) +[![License](https://img.shields.io/badge/License-GPLv3-blue.svg)](https://www.gnu.org/licenses/gpl-3.0.html) [![NuGet](https://img.shields.io/nuget/v/OpenDDD.NET.svg)](https://www.nuget.org/packages/OpenDDD.NET/) [![Tests](https://github.com/runemalm/OpenDDD.NET/actions/workflows/tests.yml/badge.svg?branch=develop)](https://github.com/runemalm/OpenDDD.NET/actions/workflows/tests.yml) OpenDDD.NET is an open-source framework for domain-driven design (DDD) development using C# and ASP.NET Core. It provides a set of powerful tools and abstractions to help developers build scalable, maintainable, and testable applications following the principles of DDD. diff --git a/src/OpenDDD.Tests/Integration/Infrastructure/Events/Azure/AzureServiceBusMessagingProviderTests.cs b/src/OpenDDD.Tests/Integration/Infrastructure/Events/Azure/AzureServiceBusMessagingProviderTests.cs index 47f591c..1d2055a 100644 --- a/src/OpenDDD.Tests/Integration/Infrastructure/Events/Azure/AzureServiceBusMessagingProviderTests.cs +++ b/src/OpenDDD.Tests/Integration/Infrastructure/Events/Azure/AzureServiceBusMessagingProviderTests.cs @@ -40,7 +40,7 @@ public async Task InitializeAsync() public async Task DisposeAsync() { - + // Keep topics generated by tests after run, in case inspection is needed } private async Task CleanupTopicsAndSubscriptionsAsync() @@ -103,6 +103,8 @@ public async Task SubscribeAsync_WhenAutoCreateTopicsIsDisabled_ShouldNotCreateT { await messagingProvider.SubscribeAsync(topicName, "test-subscriber", (msg, token) => Task.CompletedTask); }); + + Assert.False(await _adminClient.TopicExistsAsync(topicName), "Topic should not have been created."); Assert.Equal($"Cannot subscribe to topic '{topicName}' because it does not exist and auto-creation is disabled.", exception.Message); } From 57120b8232c50e7d1eb4e16f077624a0445751c4 Mon Sep 17 00:00:00 2001 From: David Runemalm Date: Fri, 21 Feb 2025 18:46:23 +0100 Subject: [PATCH 051/109] Add RabbitMQ provider integration tests. --- .github/workflows/tests.yml | 19 +++ .../AzureServiceBusMessagingProviderTests.cs | 2 +- .../RabbitMqMessagingProviderTests.cs | 148 ++++++++++++++++++ .../RabbitMq/RabbitMqTestsCollection.cs | 5 + 4 files changed, 173 insertions(+), 1 deletion(-) create mode 100644 src/OpenDDD.Tests/Integration/Infrastructure/Events/RabbitMq/RabbitMqMessagingProviderTests.cs create mode 100644 src/OpenDDD.Tests/Integration/Infrastructure/Events/RabbitMq/RabbitMqTestsCollection.cs diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 9ccb91f..b42d658 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -68,9 +68,13 @@ jobs: rabbitmq: image: rabbitmq:3-management + env: + RABBITMQ_DEFAULT_USER: guest + RABBITMQ_DEFAULT_PASS: guest ports: - 5672:5672 - 15672:15672 + options: --health-cmd "rabbitmq-diagnostics check_port_connectivity" --health-interval 10s --health-timeout 5s --health-retries 5 steps: - name: Checkout Repository @@ -110,11 +114,26 @@ jobs: -o tsv) echo "AZURE_SERVICE_BUS_CONNECTION_STRING=${CONNECTION_STRING}" >> $GITHUB_ENV + - name: Wait for RabbitMQ to be Ready + run: | + for i in {1..10}; do + if curl -s -f http://localhost:15672 || nc -z localhost 5672; then + echo "RabbitMQ is up!" + exit 0 + fi + echo "Waiting for RabbitMQ..." + sleep 5 + done + echo "RabbitMQ did not start in time!" && exit 1 + - name: Run Integration Tests working-directory: src/OpenDDD.Tests env: KAFKA_BROKER: localhost:9092 RABBITMQ_HOST: localhost + RABBITMQ_PORT: 5672 + RABBITMQ_USERNAME: guest + RABBITMQ_PASSWORD: guest AZURE_SERVICE_BUS_CONNECTION_STRING: ${{ env.AZURE_SERVICE_BUS_CONNECTION_STRING }} run: dotnet test --no-build --configuration Release --filter "Category=Integration" --logger "trx;LogFileName=TestResults.trx" --results-directory TestResults diff --git a/src/OpenDDD.Tests/Integration/Infrastructure/Events/Azure/AzureServiceBusMessagingProviderTests.cs b/src/OpenDDD.Tests/Integration/Infrastructure/Events/Azure/AzureServiceBusMessagingProviderTests.cs index 1d2055a..0804b25 100644 --- a/src/OpenDDD.Tests/Integration/Infrastructure/Events/Azure/AzureServiceBusMessagingProviderTests.cs +++ b/src/OpenDDD.Tests/Integration/Infrastructure/Events/Azure/AzureServiceBusMessagingProviderTests.cs @@ -40,7 +40,7 @@ public async Task InitializeAsync() public async Task DisposeAsync() { - // Keep topics generated by tests after run, in case inspection is needed + // We don't delete the topics after tests, in case we need to inspect them. } private async Task CleanupTopicsAndSubscriptionsAsync() diff --git a/src/OpenDDD.Tests/Integration/Infrastructure/Events/RabbitMq/RabbitMqMessagingProviderTests.cs b/src/OpenDDD.Tests/Integration/Infrastructure/Events/RabbitMq/RabbitMqMessagingProviderTests.cs new file mode 100644 index 0000000..e903400 --- /dev/null +++ b/src/OpenDDD.Tests/Integration/Infrastructure/Events/RabbitMq/RabbitMqMessagingProviderTests.cs @@ -0,0 +1,148 @@ +using Microsoft.Extensions.Logging; +using Moq; +using OpenDDD.Infrastructure.Events.RabbitMq; +using OpenDDD.Infrastructure.Events.RabbitMq.Factories; +using OpenDDD.Tests.Base; +using RabbitMQ.Client; +using RabbitMQ.Client.Exceptions; + +namespace OpenDDD.Tests.Integration.Infrastructure.Events.RabbitMq +{ + [Collection("RabbitMqTests")] + public class RabbitMqMessagingProviderTests : IntegrationTests, IAsyncLifetime + { + private readonly RabbitMqMessagingProvider _messagingProvider; + private readonly IConnectionFactory _connectionFactory; + private readonly IRabbitMqConsumerFactory _consumerFactory; + private readonly Mock> _loggerMock; + private IConnection? _connection; + private IChannel? _channel; + + private readonly string _testTopic = "OpenDddTestTopic"; + private readonly string _testConsumerGroup = "OpenDddTestGroup"; + + public RabbitMqMessagingProviderTests() + { + _loggerMock = new Mock>(); + + _connectionFactory = new ConnectionFactory + { + HostName = Environment.GetEnvironmentVariable("RABBITMQ_HOST") ?? "localhost", + Port = int.Parse(Environment.GetEnvironmentVariable("RABBITMQ_PORT") ?? "5672"), + UserName = Environment.GetEnvironmentVariable("RABBITMQ_USERNAME") ?? "guest", + Password = Environment.GetEnvironmentVariable("RABBITMQ_PASSWORD") ?? "guest", + VirtualHost = Environment.GetEnvironmentVariable("RABBITMQ_VHOST") ?? "/" + }; + + _consumerFactory = new RabbitMqConsumerFactory(_loggerMock.Object); + _messagingProvider = new RabbitMqMessagingProvider(_connectionFactory, _consumerFactory, _loggerMock.Object); + } + + public async Task InitializeAsync() + { + await EnsureConnectionAndChannelOpenAsync(); + await DeleteExchangesAndQueuesAsync(); + } + + public async Task DisposeAsync() + { + await DeleteExchangesAndQueuesAsync(); + + if (_channel is not null) + { + await _channel.CloseAsync(); + await _channel.DisposeAsync(); + } + + if (_connection is not null) + { + await _connection.CloseAsync(); + await _connection.DisposeAsync(); + } + } + + private async Task VerifyExchangeAndQueueDoNotExist() + { + try + { + await _channel!.ExchangeDeclarePassiveAsync(_testTopic, CancellationToken.None); + Assert.Fail($"Exchange '{_testTopic}' already exists before test."); + } + catch (OperationInterruptedException ex) when (ex.ShutdownReason?.ReplyCode == 404) + { + // Expected: Exchange does not exist + } + + await EnsureConnectionAndChannelOpenAsync(); + + try + { + await _channel!.QueueDeclarePassiveAsync($"{_testConsumerGroup}.{_testTopic}", CancellationToken.None); + Assert.Fail($"Queue '{_testConsumerGroup}.{_testTopic}' already exists before test."); + } + catch (OperationInterruptedException ex) when (ex.ShutdownReason?.ReplyCode == 404) + { + // Expected: Queue does not exist + } + + await EnsureConnectionAndChannelOpenAsync(); + } + + private async Task EnsureConnectionAndChannelOpenAsync() + { + if (_connection is null || !_connection.IsOpen) + { + _connection = await _connectionFactory.CreateConnectionAsync(CancellationToken.None); + } + + if (_channel is null || !_channel.IsOpen) + { + _channel = await _connection.CreateChannelAsync(null, CancellationToken.None); + } + } + + private async Task DeleteExchangesAndQueuesAsync() + { + try + { + await _channel!.ExchangeDeleteAsync(_testTopic, ifUnused: false, cancellationToken: CancellationToken.None); + } + catch (OperationInterruptedException) { /* Exchange does not exist */ } + + try + { + await _channel!.QueueDeleteAsync($"{_testConsumerGroup}.{_testTopic}", ifUnused: false, ifEmpty: false, cancellationToken: CancellationToken.None); + } + catch (OperationInterruptedException) { /* Queue does not exist */ } + } + + [Fact] + public async Task SubscribeAsync_ShouldCreateTopicIfNotExists() + { + // Arrange + await VerifyExchangeAndQueueDoNotExist(); + + // Act + await _messagingProvider.SubscribeAsync(_testTopic, _testConsumerGroup, (msg, token) => Task.CompletedTask); + + // Assert + try + { + await _channel!.ExchangeDeclarePassiveAsync(_testTopic, CancellationToken.None); + } + catch (OperationInterruptedException) + { + Assert.Fail($"Exchange '{_testTopic}' does not exist."); + } + + try + { + await _channel!.QueueDeclarePassiveAsync($"{_testConsumerGroup}.{_testTopic}", CancellationToken.None); + } + catch (OperationInterruptedException) + { + Assert.Fail($"Queue '{_testConsumerGroup}.{_testTopic}' does not exist."); + } + } + } +} diff --git a/src/OpenDDD.Tests/Integration/Infrastructure/Events/RabbitMq/RabbitMqTestsCollection.cs b/src/OpenDDD.Tests/Integration/Infrastructure/Events/RabbitMq/RabbitMqTestsCollection.cs new file mode 100644 index 0000000..70668a3 --- /dev/null +++ b/src/OpenDDD.Tests/Integration/Infrastructure/Events/RabbitMq/RabbitMqTestsCollection.cs @@ -0,0 +1,5 @@ +namespace OpenDDD.Tests.Integration.Infrastructure.Events.RabbitMq +{ + [CollectionDefinition("RabbitMqTests", DisableParallelization = true)] + public class RabbitMqTestsCollection { } +} From 745f4a2ff3e084412787d2f08a8627f2f113cca6 Mon Sep 17 00:00:00 2001 From: David Runemalm Date: Fri, 21 Feb 2025 18:57:47 +0100 Subject: [PATCH 052/109] Add a couple of integration tests for rabbit mq provider. --- .../AzureServiceBusMessagingProviderTests.cs | 4 +- .../RabbitMqMessagingProviderTests.cs | 60 ++++++++++++++++++- 2 files changed, 60 insertions(+), 4 deletions(-) diff --git a/src/OpenDDD.Tests/Integration/Infrastructure/Events/Azure/AzureServiceBusMessagingProviderTests.cs b/src/OpenDDD.Tests/Integration/Infrastructure/Events/Azure/AzureServiceBusMessagingProviderTests.cs index 0804b25..97e3dc7 100644 --- a/src/OpenDDD.Tests/Integration/Infrastructure/Events/Azure/AzureServiceBusMessagingProviderTests.cs +++ b/src/OpenDDD.Tests/Integration/Infrastructure/Events/Azure/AzureServiceBusMessagingProviderTests.cs @@ -62,7 +62,7 @@ private async Task CleanupTopicsAndSubscriptionsAsync() } [Fact] - public async Task SubscribeAsync_WhenAutoCreateTopicsIsEnabled_ShouldCreateTopicIfNotExists() + public async Task Subscribe_WhenAutoCreateTopicsIsEnabled_ShouldCreateTopicIfNotExists() { // Arrange var topicName = $"test-topic-{Guid.NewGuid()}"; @@ -81,7 +81,7 @@ public async Task SubscribeAsync_WhenAutoCreateTopicsIsEnabled_ShouldCreateTopic } [Fact] - public async Task SubscribeAsync_WhenAutoCreateTopicsIsDisabled_ShouldNotCreateTopic_AndLogErrorWhenTopicNotExist() + public async Task Subscribe_WhenAutoCreateTopicsIsDisabled_ShouldNotCreateTopic_AndLogErrorWhenTopicNotExist() { // Arrange var topicName = $"test-topic-{Guid.NewGuid()}"; diff --git a/src/OpenDDD.Tests/Integration/Infrastructure/Events/RabbitMq/RabbitMqMessagingProviderTests.cs b/src/OpenDDD.Tests/Integration/Infrastructure/Events/RabbitMq/RabbitMqMessagingProviderTests.cs index e903400..5de233e 100644 --- a/src/OpenDDD.Tests/Integration/Infrastructure/Events/RabbitMq/RabbitMqMessagingProviderTests.cs +++ b/src/OpenDDD.Tests/Integration/Infrastructure/Events/RabbitMq/RabbitMqMessagingProviderTests.cs @@ -1,4 +1,5 @@ -using Microsoft.Extensions.Logging; +using System.Collections.Concurrent; +using Microsoft.Extensions.Logging; using Moq; using OpenDDD.Infrastructure.Events.RabbitMq; using OpenDDD.Infrastructure.Events.RabbitMq.Factories; @@ -117,7 +118,7 @@ private async Task DeleteExchangesAndQueuesAsync() } [Fact] - public async Task SubscribeAsync_ShouldCreateTopicIfNotExists() + public async Task Subscribe_ShouldCreateTopicIfNotExists() { // Arrange await VerifyExchangeAndQueueDoNotExist(); @@ -144,5 +145,60 @@ public async Task SubscribeAsync_ShouldCreateTopicIfNotExists() Assert.Fail($"Queue '{_testConsumerGroup}.{_testTopic}' does not exist."); } } + + [Fact] + public async Task SubscribeAsync_ShouldReceivePublishedMessage() + { + // Arrange + var receivedMessages = new ConcurrentBag(); + var messageToSend = "Hello, OpenDDD!"; + var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + + await _messagingProvider.SubscribeAsync(_testTopic, _testConsumerGroup, async (msg, token) => + { + receivedMessages.Add(msg); + }, cts.Token); + + await Task.Delay(500); // Allow time for the consumer to start + + // Act + await _messagingProvider.PublishAsync(_testTopic, messageToSend, cts.Token); + + // Wait for message to be received + await Task.Delay(1000); + + // Assert + Assert.Contains(messageToSend, receivedMessages); + } + + [Fact] + public async Task SubscribeAsync_ShouldDeliverMessageToOnlyOneCompetingConsumer() + { + // Arrange + var receivedMessages = new ConcurrentDictionary(); + var messageToSend = "Competing Consumer Test"; + var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + + async Task MessageHandler(string msg, CancellationToken token) + { + receivedMessages.AddOrUpdate("received", 1, (key, value) => value + 1); + } + + // Subscribe multiple competing consumers to the same topic + await _messagingProvider.SubscribeAsync(_testTopic, _testConsumerGroup, MessageHandler, cts.Token); + await _messagingProvider.SubscribeAsync(_testTopic, _testConsumerGroup, MessageHandler, cts.Token); + await _messagingProvider.SubscribeAsync(_testTopic, _testConsumerGroup, MessageHandler, cts.Token); + + await Task.Delay(500); // Allow time for consumers to start + + // Act + await _messagingProvider.PublishAsync(_testTopic, messageToSend, cts.Token); + + // Wait for message processing + await Task.Delay(1000); + + // Assert: Only one consumer should receive the message + Assert.Equal(1, receivedMessages.GetValueOrDefault("received", 0)); + } } } From dfdb5da620ab728ff19a0c8658255e2c513318ad Mon Sep 17 00:00:00 2001 From: David Runemalm Date: Sun, 23 Feb 2025 15:28:42 +0100 Subject: [PATCH 053/109] Add more rabbitmq provider integration tests. --- .../RabbitMqMessagingProviderTests.cs | 132 ++++++++++++++++-- .../Azure/AzureServiceBusMessagingProvider.cs | 5 + .../Events/IMessagingProvider.cs | 1 + .../InMemory/InMemoryMessagingProvider.cs | 5 + .../Events/Kafka/KafkaMessagingProvider.cs | 5 + .../RabbitMq/RabbitMqMessagingProvider.cs | 27 +++- 6 files changed, 159 insertions(+), 16 deletions(-) diff --git a/src/OpenDDD.Tests/Integration/Infrastructure/Events/RabbitMq/RabbitMqMessagingProviderTests.cs b/src/OpenDDD.Tests/Integration/Infrastructure/Events/RabbitMq/RabbitMqMessagingProviderTests.cs index 5de233e..aead1fb 100644 --- a/src/OpenDDD.Tests/Integration/Infrastructure/Events/RabbitMq/RabbitMqMessagingProviderTests.cs +++ b/src/OpenDDD.Tests/Integration/Infrastructure/Events/RabbitMq/RabbitMqMessagingProviderTests.cs @@ -116,9 +116,9 @@ private async Task DeleteExchangesAndQueuesAsync() } catch (OperationInterruptedException) { /* Queue does not exist */ } } - + [Fact] - public async Task Subscribe_ShouldCreateTopicIfNotExists() + public async Task Configuration_ShouldCreateTopic_WhenSubscribing_AndTopicDoNotExist_AndAutoCreateTopicIsEnabled() { // Arrange await VerifyExchangeAndQueueDoNotExist(); @@ -147,58 +147,160 @@ public async Task Subscribe_ShouldCreateTopicIfNotExists() } [Fact] - public async Task SubscribeAsync_ShouldReceivePublishedMessage() + public async Task AtLeastOnceGurantee_ShouldDeliverToLateSubscriber_WhenSubscribedBefore() { // Arrange var receivedMessages = new ConcurrentBag(); - var messageToSend = "Hello, OpenDDD!"; - var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + var messageToSend = "Persistent Message Test"; + var cts = new CancellationTokenSource(TimeSpan.FromSeconds(10)); + + // First subscription to establish the listener group + await _messagingProvider.SubscribeAsync(_testTopic, _testConsumerGroup, async (msg, token) => + { + Assert.Fail("First subscription should not receive the message."); + }, cts.Token); + await Task.Delay(500); + + // Unsubscribe + await _messagingProvider.UnsubscribeAsync(_testTopic, _testConsumerGroup, cts.Token); + await Task.Delay(500); + + // Act: Publish message + await _messagingProvider.PublishAsync(_testTopic, messageToSend, cts.Token); + + // Delay to simulate late subscriber + await Task.Delay(2000); + // Late subscriber await _messagingProvider.SubscribeAsync(_testTopic, _testConsumerGroup, async (msg, token) => { receivedMessages.Add(msg); }, cts.Token); - await Task.Delay(500); // Allow time for the consumer to start + // Wait for message processing + await Task.Delay(1000); - // Act + // Assert + Assert.Contains(messageToSend, receivedMessages); + } + + [Fact] + public async Task AtLeastOnceGurantee_ShouldNotDeliverToLateSubscriber_WhenNotSubscribedBefore() + { + // Arrange + var receivedMessages = new ConcurrentBag(); + var messageToSend = "Non-Persistent Message Test"; + var cts = new CancellationTokenSource(TimeSpan.FromSeconds(10)); + + // Act: Publish message before any subscription await _messagingProvider.PublishAsync(_testTopic, messageToSend, cts.Token); - // Wait for message to be received + // Delay to simulate late subscriber + await Task.Delay(2000); + + // Late subscriber + await _messagingProvider.SubscribeAsync(_testTopic, _testConsumerGroup, async (msg, token) => + { + receivedMessages.Add(msg); + }, cts.Token); + + // Wait for message processing await Task.Delay(1000); // Assert - Assert.Contains(messageToSend, receivedMessages); + Assert.DoesNotContain(messageToSend, receivedMessages); } + + [Fact] + public async Task AtLeastOnceGurantee_ShouldRedeliverLater_WhenMessageNotAcked() + { + // Arrange + var receivedMessages = new ConcurrentBag(); + var messageToSend = "Redelivery Test"; + var cts = new CancellationTokenSource(TimeSpan.FromSeconds(10)); + async Task FaultyHandler(string msg, CancellationToken token) + { + receivedMessages.Add(msg); + throw new Exception("Simulated consumer crash before acknowledgment."); + } + + await _messagingProvider.SubscribeAsync(_testTopic, _testConsumerGroup, FaultyHandler, cts.Token); + await Task.Delay(500); // Ensure setup + + // Act: Publish message + await _messagingProvider.PublishAsync(_testTopic, messageToSend, cts.Token); + + // Wait for redelivery + await Task.Delay(3000); + + // Assert: The message should be received multiple times due to reattempts + Assert.True(receivedMessages.Count > 1, "Message should be redelivered at least once."); + } + [Fact] - public async Task SubscribeAsync_ShouldDeliverMessageToOnlyOneCompetingConsumer() + public async Task CompetingConsumers_ShouldDeliverOnlyOnce_WhenMultipleConsumersInGroup() { // Arrange var receivedMessages = new ConcurrentDictionary(); var messageToSend = "Competing Consumer Test"; - var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + var cts = new CancellationTokenSource(TimeSpan.FromSeconds(10)); async Task MessageHandler(string msg, CancellationToken token) { receivedMessages.AddOrUpdate("received", 1, (key, value) => value + 1); } - // Subscribe multiple competing consumers to the same topic + // Multiple competing consumers in the same group await _messagingProvider.SubscribeAsync(_testTopic, _testConsumerGroup, MessageHandler, cts.Token); await _messagingProvider.SubscribeAsync(_testTopic, _testConsumerGroup, MessageHandler, cts.Token); await _messagingProvider.SubscribeAsync(_testTopic, _testConsumerGroup, MessageHandler, cts.Token); - await Task.Delay(500); // Allow time for consumers to start // Act await _messagingProvider.PublishAsync(_testTopic, messageToSend, cts.Token); - // Wait for message processing + // Wait for processing await Task.Delay(1000); - // Assert: Only one consumer should receive the message + // Assert: Only one of the competing consumers should receive the message Assert.Equal(1, receivedMessages.GetValueOrDefault("received", 0)); } + + [Fact] + public async Task CompetingConsumers_ShouldDistributeEvenly_WhenMultipleConsumersInGroup() + { + // Arrange + var receivedMessages = new ConcurrentDictionary(); + var totalMessages = 10; + var cts = new CancellationTokenSource(TimeSpan.FromSeconds(15)); + + async Task MessageHandler(string msg, CancellationToken token) + { + receivedMessages.AddOrUpdate(msg, 1, (key, value) => value + 1); + } + + // Multiple competing consumers in the same group + await _messagingProvider.SubscribeAsync(_testTopic, _testConsumerGroup, MessageHandler, cts.Token); + await _messagingProvider.SubscribeAsync(_testTopic, _testConsumerGroup, MessageHandler, cts.Token); + await _messagingProvider.SubscribeAsync(_testTopic, _testConsumerGroup, MessageHandler, cts.Token); + await Task.Delay(500); // Allow time for consumers to start + + // Act: Publish multiple messages + for (int i = 0; i < totalMessages; i++) + { + await _messagingProvider.PublishAsync(_testTopic, $"Message {i}", cts.Token); + } + + // Wait for processing + await Task.Delay(2000); + + // Assert: Messages should be evenly distributed across consumers + var messageCounts = receivedMessages.Values; + var minReceived = messageCounts.Min(); + var maxReceived = messageCounts.Max(); + + Assert.True(maxReceived - minReceived <= 1, "Messages should be evenly distributed among competing consumers."); + } } } diff --git a/src/OpenDDD/Infrastructure/Events/Azure/AzureServiceBusMessagingProvider.cs b/src/OpenDDD/Infrastructure/Events/Azure/AzureServiceBusMessagingProvider.cs index 3b68f12..0515581 100644 --- a/src/OpenDDD/Infrastructure/Events/Azure/AzureServiceBusMessagingProvider.cs +++ b/src/OpenDDD/Infrastructure/Events/Azure/AzureServiceBusMessagingProvider.cs @@ -77,6 +77,11 @@ public async Task SubscribeAsync(string topic, string consumerGroup, Func messageHandler, CancellationToken cancellationToken = default); + Task UnsubscribeAsync(string topic, string consumerGroup, CancellationToken cancellationToken = default); Task PublishAsync(string topic, string message, CancellationToken cancellationToken = default); } } diff --git a/src/OpenDDD/Infrastructure/Events/InMemory/InMemoryMessagingProvider.cs b/src/OpenDDD/Infrastructure/Events/InMemory/InMemoryMessagingProvider.cs index 9ec830c..b2bda45 100644 --- a/src/OpenDDD/Infrastructure/Events/InMemory/InMemoryMessagingProvider.cs +++ b/src/OpenDDD/Infrastructure/Events/InMemory/InMemoryMessagingProvider.cs @@ -25,6 +25,11 @@ public Task SubscribeAsync(string topic, string consumerGroup, Func key.StartsWith($"{topic}:")); diff --git a/src/OpenDDD/Infrastructure/Events/Kafka/KafkaMessagingProvider.cs b/src/OpenDDD/Infrastructure/Events/Kafka/KafkaMessagingProvider.cs index 1ac9083..792c253 100644 --- a/src/OpenDDD/Infrastructure/Events/Kafka/KafkaMessagingProvider.cs +++ b/src/OpenDDD/Infrastructure/Events/Kafka/KafkaMessagingProvider.cs @@ -68,6 +68,11 @@ public async Task SubscribeAsync( _consumerTasks.Add(consumerTask); } + public Task UnsubscribeAsync(string topic, string consumerGroup, CancellationToken cancellationToken = default) + { + throw new NotImplementedException(); + } + private async Task StartConsumerLoop( IConsumer consumer, Func messageHandler, diff --git a/src/OpenDDD/Infrastructure/Events/RabbitMq/RabbitMqMessagingProvider.cs b/src/OpenDDD/Infrastructure/Events/RabbitMq/RabbitMqMessagingProvider.cs index 83d99a8..6cf9b4f 100644 --- a/src/OpenDDD/Infrastructure/Events/RabbitMq/RabbitMqMessagingProvider.cs +++ b/src/OpenDDD/Infrastructure/Events/RabbitMq/RabbitMqMessagingProvider.cs @@ -14,6 +14,7 @@ public class RabbitMqMessagingProvider : IMessagingProvider, IAsyncDisposable private IConnection? _connection; private IChannel? _channel; private readonly ConcurrentBag _consumers = new(); + private readonly ConcurrentDictionary _consumerTags = new(); public RabbitMqMessagingProvider( IConnectionFactory factory, @@ -47,10 +48,34 @@ public async Task SubscribeAsync(string topic, string consumerGroup, Func Date: Sun, 23 Feb 2025 16:39:35 +0100 Subject: [PATCH 054/109] Add more azure targets to makefile. --- Makefile | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/Makefile b/Makefile index 4ac4dc5..9f87361 100644 --- a/Makefile +++ b/Makefile @@ -297,3 +297,27 @@ azure-get-servicebus-connection: ##@Azure Get the Service Bus connection string .PHONY: azure-delete-servicebus-namespace azure-delete-servicebus-namespace: ##@Azure Delete the Azure Service Bus namespace az servicebus namespace delete --resource-group $(AZURE_RESOURCE_GROUP) --name $(AZURE_SERVICEBUS_NAMESPACE) + +.PHONY: azure-list-servicebus-namespaces +azure-list-servicebus-namespaces: ##@Azure List all Azure Service Bus namespaces in the resource group + az servicebus namespace list --resource-group $(AZURE_RESOURCE_GROUP) --output table + +.PHONY: azure-list-servicebus-topics +azure-list-servicebus-topics: ##@Azure List all topics in the Azure Service Bus namespace + az servicebus topic list --resource-group $(AZURE_RESOURCE_GROUP) --namespace-name $(AZURE_SERVICEBUS_NAMESPACE) --output table + +.PHONY: azure-list-servicebus-subscriptions +azure-list-servicebus-subscriptions: ##@Azure List all subscriptions for a given topic (usage: make azure-list-servicebus-subscriptions TOPIC_NAME=) + @if [ -z "$(TOPIC_NAME)" ]; then \ + echo "Error: Specify the topic name using TOPIC_NAME="; \ + exit 1; \ + fi + az servicebus topic subscription list --resource-group $(AZURE_RESOURCE_GROUP) --namespace-name $(AZURE_SERVICEBUS_NAMESPACE) --topic-name $(TOPIC_NAME) --output table + +.PHONY: azure-list-servicebus-queues +azure-list-servicebus-queues: ##@Azure List all queues in the Azure Service Bus namespace + az servicebus queue list --resource-group $(AZURE_RESOURCE_GROUP) --namespace-name $(AZURE_SERVICEBUS_NAMESPACE) --output table + +.PHONY: azure-list-servicebus-authorization-rules +azure-list-servicebus-authorization-rules: ##@Azure List all authorization rules for the Azure Service Bus namespace + az servicebus namespace authorization-rule list --resource-group $(AZURE_RESOURCE_GROUP) --namespace-name $(AZURE_SERVICEBUS_NAMESPACE) --output table From 8622eda4787ad6c736d3c1f9bf0e51b60fc1c8b4 Mon Sep 17 00:00:00 2001 From: David Runemalm Date: Sun, 23 Feb 2025 16:39:57 +0100 Subject: [PATCH 055/109] Add more azure messaging provider integration tests. --- .../AzureServiceBusMessagingProviderTests.cs | 192 ++++++++++++++++-- .../RabbitMqMessagingProviderTests.cs | 2 +- .../Azure/AzureServiceBusMessagingProvider.cs | 32 ++- 3 files changed, 210 insertions(+), 16 deletions(-) diff --git a/src/OpenDDD.Tests/Integration/Infrastructure/Events/Azure/AzureServiceBusMessagingProviderTests.cs b/src/OpenDDD.Tests/Integration/Infrastructure/Events/Azure/AzureServiceBusMessagingProviderTests.cs index 97e3dc7..5c1af2c 100644 --- a/src/OpenDDD.Tests/Integration/Infrastructure/Events/Azure/AzureServiceBusMessagingProviderTests.cs +++ b/src/OpenDDD.Tests/Integration/Infrastructure/Events/Azure/AzureServiceBusMessagingProviderTests.cs @@ -1,4 +1,5 @@ -using Microsoft.Extensions.Logging; +using System.Collections.Concurrent; +using Microsoft.Extensions.Logging; using Moq; using OpenDDD.Infrastructure.Events.Azure; using OpenDDD.Tests.Base; @@ -34,13 +35,13 @@ public AzureServiceBusMessagingProviderTests() public async Task InitializeAsync() { - // Cleanup any test topics/subscriptions from previous runs + // Start each test with an empty service bus namespace await CleanupTopicsAndSubscriptionsAsync(); } public async Task DisposeAsync() { - // We don't delete the topics after tests, in case we need to inspect them. + // Don't delete after latest run test in case we need to inspect what was created } private async Task CleanupTopicsAndSubscriptionsAsync() @@ -60,44 +61,44 @@ private async Task CleanupTopicsAndSubscriptionsAsync() } } } - + [Fact] - public async Task Subscribe_WhenAutoCreateTopicsIsEnabled_ShouldCreateTopicIfNotExists() + public async Task AutoCreateTopic_ShouldCreateTopicOnSubscribe_WhenSettingEnabled() { // Arrange var topicName = $"test-topic-{Guid.NewGuid()}"; + var subscriptionName = "test-subscription"; - // Ensure topic does not exist before the test if (await _adminClient.TopicExistsAsync(topicName)) { await _adminClient.DeleteTopicAsync(topicName); } // Act - await _messagingProvider.SubscribeAsync(topicName, "test-subscriber", (msg, token) => Task.CompletedTask); + await _messagingProvider.SubscribeAsync(topicName, subscriptionName, (msg, token) => Task.CompletedTask); // Assert Assert.True(await _adminClient.TopicExistsAsync(topicName)); } - + [Fact] - public async Task Subscribe_WhenAutoCreateTopicsIsDisabled_ShouldNotCreateTopic_AndLogErrorWhenTopicNotExist() + public async Task AutoCreateTopic_ShouldNotCreateTopicOnSubscribe_WhenSettingDisabled() { // Arrange var topicName = $"test-topic-{Guid.NewGuid()}"; - + // Ensure topic does not exist before the test if (await _adminClient.TopicExistsAsync(topicName)) { await _adminClient.DeleteTopicAsync(topicName); } - + var messagingProvider = new AzureServiceBusMessagingProvider( _serviceBusClient, _adminClient, autoCreateTopics: false, _loggerMock.Object); - + // Act & Assert var exception = await Assert.ThrowsAsync(async () => { @@ -105,8 +106,173 @@ public async Task Subscribe_WhenAutoCreateTopicsIsDisabled_ShouldNotCreateTopic_ }); Assert.False(await _adminClient.TopicExistsAsync(topicName), "Topic should not have been created."); - + Assert.Equal($"Cannot subscribe to topic '{topicName}' because it does not exist and auto-creation is disabled.", exception.Message); } + + [Fact] + public async Task AtLeastOnceGurantee_ShouldDeliverToLateSubscriber_WhenSubscribedBefore() + { + // Arrange + var topicName = $"test-topic-{Guid.NewGuid()}"; + var subscriptionName = "test-subscription"; + var receivedMessages = new ConcurrentBag(); + var messageToSend = "Persistent Message Test"; + var cts = new CancellationTokenSource(TimeSpan.FromSeconds(30)); + + await _messagingProvider.SubscribeAsync(topicName, subscriptionName, async (msg, token) => + { + Assert.Fail("First subscription should not receive the message."); + }, cts.Token); + await Task.Delay(500); + + await _messagingProvider.UnsubscribeAsync(topicName, subscriptionName, cts.Token); + await Task.Delay(500); + + // Act: Publish message + await _messagingProvider.PublishAsync(topicName, messageToSend, cts.Token); + + // Delay to simulate late subscriber + await Task.Delay(2000); + + // Late subscriber + await _messagingProvider.SubscribeAsync(topicName, subscriptionName, async (msg, token) => + { + receivedMessages.Add(msg); + }, cts.Token); + + // Wait for message processing + await Task.Delay(1000); + + // Assert + Assert.Contains(messageToSend, receivedMessages); + } + + [Fact] + public async Task AtLeastOnceGurantee_ShouldNotDeliverToLateSubscriber_WhenNotSubscribedBefore() + { + // Arrange + var topicName = $"test-topic-{Guid.NewGuid()}"; + var subscriptionName = "test-subscription"; + var receivedMessages = new ConcurrentBag(); + var messageToSend = "Non-Persistent Message Test"; + var cts = new CancellationTokenSource(TimeSpan.FromSeconds(10)); + + // Act: Publish message before any subscription + await _messagingProvider.PublishAsync(topicName, messageToSend, cts.Token); + + await Task.Delay(500); + + // Late subscriber + await _messagingProvider.SubscribeAsync(topicName, subscriptionName, async (msg, token) => + { + receivedMessages.Add(msg); + }, cts.Token); + + // Wait for message processing + await Task.Delay(1000); + + // Assert + Assert.DoesNotContain(messageToSend, receivedMessages); + } + + [Fact] + public async Task AtLeastOnceGurantee_ShouldRedeliverLater_WhenMessageNotAcked() + { + // Arrange + var topicName = $"test-topic-{Guid.NewGuid()}"; + var subscriptionName = "test-subscription"; + var receivedMessages = new ConcurrentBag(); + var messageToSend = "Redelivery Test"; + var cts = new CancellationTokenSource(TimeSpan.FromSeconds(10)); + + async Task FaultyHandler(string msg, CancellationToken token) + { + receivedMessages.Add(msg); + throw new Exception("Simulated consumer crash before acknowledgment."); + } + + await _messagingProvider.SubscribeAsync(topicName, subscriptionName, FaultyHandler, cts.Token); + await Task.Delay(500); + + // Act: Publish message + await _messagingProvider.PublishAsync(topicName, messageToSend, cts.Token); + + // Wait for redelivery + await Task.Delay(3000); + + // Assert: The message should be received multiple times due to reattempts + Assert.True(receivedMessages.Count > 1, "Message should be redelivered at least once."); + } + + [Fact] + public async Task CompetingConsumers_ShouldDeliverOnlyOnce_WhenMultipleConsumersInGroup() + { + // Arrange + var topicName = $"test-topic-{Guid.NewGuid()}"; + var subscriptionName = "test-consumer-group"; + var receivedMessages = new ConcurrentDictionary(); + var messageToSend = "Competing Consumer Test"; + var cts = new CancellationTokenSource(TimeSpan.FromSeconds(10)); + + async Task MessageHandler(string msg, CancellationToken token) + { + receivedMessages.AddOrUpdate("received", 1, (key, value) => value + 1); + } + + // Multiple competing consumers in the same group + await _messagingProvider.SubscribeAsync(topicName, subscriptionName, MessageHandler, cts.Token); + await _messagingProvider.SubscribeAsync(topicName, subscriptionName, MessageHandler, cts.Token); + await _messagingProvider.SubscribeAsync(topicName, subscriptionName, MessageHandler, cts.Token); + await Task.Delay(500); + + // Act + await _messagingProvider.PublishAsync(topicName, messageToSend, cts.Token); + + // Wait for processing + await Task.Delay(1000); + + // Assert: Only one of the competing consumers should receive the message + Assert.Equal(1, receivedMessages.GetValueOrDefault("received", 0)); + } + + [Fact] + public async Task CompetingConsumers_ShouldDistributeEvenly_WhenMultipleConsumersInGroup() + { + // Arrange + var topicName = $"test-topic-{Guid.NewGuid()}"; + var subscriptionName = "test-consumer-group"; + var receivedMessages = new ConcurrentDictionary(); + var totalMessages = 10; + var cts = new CancellationTokenSource(TimeSpan.FromSeconds(30)); + + async Task MessageHandler(string msg, CancellationToken token) + { + receivedMessages.AddOrUpdate(msg, 1, (key, value) => value + 1); + } + + // Multiple competing consumers in the same group + await _messagingProvider.SubscribeAsync(topicName, subscriptionName, MessageHandler, cts.Token); + await _messagingProvider.SubscribeAsync(topicName, subscriptionName, MessageHandler, cts.Token); + await _messagingProvider.SubscribeAsync(topicName, subscriptionName, MessageHandler, cts.Token); + await Task.Delay(500); + + // Act: Publish multiple messages + for (int i = 0; i < totalMessages; i++) + { + await _messagingProvider.PublishAsync(topicName, $"Message {i}", cts.Token); + } + + // Wait for processing + await Task.Delay(2000); + + // Assert: Messages should be evenly distributed across consumers + var messageCounts = receivedMessages.Values; + var minReceived = messageCounts.Min(); + var maxReceived = messageCounts.Max(); + + Assert.True(maxReceived - minReceived <= 1, + "Messages should be evenly distributed among competing consumers."); + } } } diff --git a/src/OpenDDD.Tests/Integration/Infrastructure/Events/RabbitMq/RabbitMqMessagingProviderTests.cs b/src/OpenDDD.Tests/Integration/Infrastructure/Events/RabbitMq/RabbitMqMessagingProviderTests.cs index aead1fb..c0640bc 100644 --- a/src/OpenDDD.Tests/Integration/Infrastructure/Events/RabbitMq/RabbitMqMessagingProviderTests.cs +++ b/src/OpenDDD.Tests/Integration/Infrastructure/Events/RabbitMq/RabbitMqMessagingProviderTests.cs @@ -118,7 +118,7 @@ private async Task DeleteExchangesAndQueuesAsync() } [Fact] - public async Task Configuration_ShouldCreateTopic_WhenSubscribing_AndTopicDoNotExist_AndAutoCreateTopicIsEnabled() + public async Task AutoCreateTopic_ShouldCreateTopicOnSubscribe_WhenSettingEnabled() { // Arrange await VerifyExchangeAndQueueDoNotExist(); diff --git a/src/OpenDDD/Infrastructure/Events/Azure/AzureServiceBusMessagingProvider.cs b/src/OpenDDD/Infrastructure/Events/Azure/AzureServiceBusMessagingProvider.cs index 0515581..fc25fd4 100644 --- a/src/OpenDDD/Infrastructure/Events/Azure/AzureServiceBusMessagingProvider.cs +++ b/src/OpenDDD/Infrastructure/Events/Azure/AzureServiceBusMessagingProvider.cs @@ -77,9 +77,37 @@ public async Task SubscribeAsync(string topic, string consumerGroup, Func + p.EntityPath.Equals($"{topic}/Subscriptions/{subscriptionName}", StringComparison.OrdinalIgnoreCase)); + + if (processor != null) + { + _processors.Remove(processor); + _logger.LogInformation("Stopping and disposing message processor for topic '{Topic}' and subscription '{Subscription}'", topic, subscriptionName); + + await processor.StopProcessingAsync(cancellationToken); + await processor.DisposeAsync(); + } + else + { + _logger.LogWarning("No active subscription found for topic '{Topic}' and subscription '{Subscription}'", topic, subscriptionName); + } } public async Task PublishAsync(string topic, string message, CancellationToken cancellationToken = default) From ebcd11b8efe8d8d338368500f7a1efa0199c8ff2 Mon Sep 17 00:00:00 2001 From: David Runemalm Date: Sun, 23 Feb 2025 16:47:41 +0100 Subject: [PATCH 056/109] Add rabbit targets to makefile, for local integration testing. --- Makefile | 33 +++++++++++++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 9f87361..c081f55 100644 --- a/Makefile +++ b/Makefile @@ -221,8 +221,6 @@ templates-rebuild: templates-uninstall templates-pack templates-install ##@Templ # ACT ########################################################################## -# ACT_IMAGE := ghcr.io/catthehacker/ubuntu:full-latest -# ACT_IMAGE := ghcr.io/catthehacker/ubuntu:runner-latest ACT_IMAGE := ghcr.io/catthehacker/ubuntu:act-latest .PHONY: act-install @@ -321,3 +319,34 @@ azure-list-servicebus-queues: ##@Azure List all queues in the Azure Service Bus .PHONY: azure-list-servicebus-authorization-rules azure-list-servicebus-authorization-rules: ##@Azure List all authorization rules for the Azure Service Bus namespace az servicebus namespace authorization-rule list --resource-group $(AZURE_RESOURCE_GROUP) --namespace-name $(AZURE_SERVICEBUS_NAMESPACE) --output table + +########################################################################## +# RABBITMQ +########################################################################## + +RABBITMQ_PORT := 5672 + +.PHONY: rabbitmq-start +rabbitmq-start: ##@@RabbitMQ Start a RabbitMQ container + docker run --rm -d --name rabbitmq --hostname rabbitmq \ + -e RABBITMQ_DEFAULT_USER=$(RABBITMQ_DEFAULT_USER) \ + -e RABBITMQ_DEFAULT_PASS=$(RABBITMQ_DEFAULT_PASS) \ + -p 5672:$(RABBITMQ_PORT) -p 15672:15672 rabbitmq:management + @echo "RabbitMQ started. Management UI available at http://localhost:15672" + +.PHONY: rabbitmq-stop +rabbitmq-stop: ##@RabbitMQ Stop the RabbitMQ container + docker stop rabbitmq + @echo "RabbitMQ stopped." + +.PHONY: rabbitmq-status +rabbitmq-status: ##@RabbitMQ Check RabbitMQ container status + docker ps | grep rabbitmq || echo "RabbitMQ is not running." + +.PHONY: rabbitmq-get-connection +rabbitmq-get-connection: ##@RabbitMQ Get the RabbitMQ connection string + @echo "amqp://$(RABBITMQ_DEFAULT_USER):$(RABBITMQ_DEFAULT_PASS)@localhost:$(RABBITMQ_PORT)/" + +.PHONY: rabbitmq-logs +rabbitmq-logs: ##@RabbitMQ Show RabbitMQ logs + docker logs -f rabbitmq From 55d3a8d79ad983cdd78197bb2e9d293c8c5529c4 Mon Sep 17 00:00:00 2001 From: David Runemalm Date: Mon, 24 Feb 2025 17:27:11 +0100 Subject: [PATCH 057/109] Add kafka targets to makefile. --- Makefile | 76 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/Makefile b/Makefile index c081f55..57b3efc 100644 --- a/Makefile +++ b/Makefile @@ -33,6 +33,8 @@ FEED_DIR := $(HOME)/Projects/LocalFeed USER_NUGET_CONFIG_DIR=$(HOME)/.config/NuGet/NuGet.Config SPHINXDOC_IMG := openddd.net/sphinxdoc +NETWORK := opendddnet + BLUE := $(shell tput -Txterm setaf 4) GREEN := $(shell tput -Txterm setaf 2) TURQUOISE := $(shell tput -Txterm setaf 6) @@ -350,3 +352,77 @@ rabbitmq-get-connection: ##@RabbitMQ Get the RabbitMQ connection string .PHONY: rabbitmq-logs rabbitmq-logs: ##@RabbitMQ Show RabbitMQ logs docker logs -f rabbitmq + +########################################################################## +# KAFKA +########################################################################## + +ZOOKEEPER_CONTAINER := opendddnet-zookeeper + +KAFKA_NETWORK := $(NETWORK) +KAFKA_CONTAINER := opendddnet-kafka +KAFKA_BROKER := localhost:9092 +KAFKA_ZOOKEEPER := localhost:2181 + +.PHONY: kafka-start +kafka-start: ##@Kafka Start Kafka and Zookeeper using Docker + @docker network inspect $(KAFKA_NETWORK) >/dev/null 2>&1 || docker network create $(KAFKA_NETWORK) + @docker run -d --rm --name $(ZOOKEEPER_CONTAINER) --network $(KAFKA_NETWORK) -p 2181:2181 \ + wurstmeister/zookeeper:latest + @docker run -d --rm --name $(KAFKA_CONTAINER) --network $(KAFKA_NETWORK) -p 9092:9092 \ + -e KAFKA_BROKER_ID=1 \ + -e KAFKA_ZOOKEEPER_CONNECT=$(ZOOKEEPER_CONTAINER):2181 \ + -e KAFKA_LISTENERS=PLAINTEXT://0.0.0.0:9092 \ + -e KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://localhost:9092 \ + -e KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1 \ + wurstmeister/kafka:latest + +.PHONY: kafka-stop +kafka-stop: ##@Kafka Stop Kafka and Zookeeper + @docker stop $(KAFKA_CONTAINER) || true + @docker stop $(ZOOKEEPER_CONTAINER) || true + +.PHONY: kafka-logs +kafka-logs: ##@Kafka Show Kafka logs + @docker logs -f $(KAFKA_CONTAINER) + +.PHONY: kafka-shell +kafka-shell: ##@@Kafka Open a shell inside the Kafka container + docker exec -it $(KAFKA_CONTAINER) /bin/sh + +.PHONY: kafka-create-topic +kafka-create-topic: ##@Kafka Create a Kafka topic (uses NAME) +ifndef NAME + $(error Topic name not specified. Usage: make kafka-create-topic NAME=) +endif + @docker exec -it $(KAFKA_CONTAINER) kafka-topics.sh --create --topic $(NAME) --bootstrap-server $(KAFKA_BROKER) --replication-factor 1 --partitions 1 + +.PHONY: kafka-list-brokers +kafka-list-brokers: ##@Kafka List Kafka broker configurations + @docker exec -it $(KAFKA_CONTAINER) /opt/kafka/bin/kafka-configs.sh --bootstrap-server localhost:9092 --describe --entity-type brokers + +.PHONY: kafka-list-topics +kafka-list-topics: ##@Kafka List all Kafka topics + @docker exec -it $(KAFKA_CONTAINER) kafka-topics.sh --list --bootstrap-server $(KAFKA_BROKER) + +.PHONY: kafka-broker-status +kafka-broker-status: ##@Kafka Show Kafka broker status + @docker exec -it $(KAFKA_CONTAINER) /opt/kafka/bin/kafka-broker-api-versions.sh --bootstrap-server localhost:9092 + +.PHONY: kafka-list-consumer-groups +kafka-list-consumer-groups: ##@Kafka List active Kafka consumer groups + @docker exec -it $(KAFKA_CONTAINER) /opt/kafka/bin/kafka-consumer-groups.sh --bootstrap-server localhost:9092 --list + +.PHONY: kafka-consume +kafka-consume: ##@Kafka Consume messages from a Kafka topic (uses NAME) +ifndef NAME + $(error Topic name not specified. Usage: make kafka-consume NAME=) +endif + @docker exec -it $(KAFKA_CONTAINER) kafka-console-consumer.sh --bootstrap-server $(KAFKA_BROKER) --topic $(NAME) --from-beginning + +.PHONY: kafka-produce +kafka-produce: ##@Kafka Produce messages to a Kafka topic (uses NAME) +ifndef NAME + $(error Topic name not specified. Usage: make kafka-produce NAME=) +endif + @docker exec -it $(KAFKA_CONTAINER) kafka-console-producer.sh --broker-list $(KAFKA_BROKER) --topic $(NAME) From 0445adb87ee2e63752e616859c95103faa6711e9 Mon Sep 17 00:00:00 2001 From: David Runemalm Date: Mon, 24 Feb 2025 17:29:12 +0100 Subject: [PATCH 058/109] Add custom integration tests xunit logger. --- src/OpenDDD.Tests/Base/IntegrationTests.cs | 30 +++++++++++++++++-- .../AzureServiceBusMessagingProviderTests.cs | 6 ++-- .../RabbitMqMessagingProviderTests.cs | 9 +++--- src/OpenDDD.Tests/Logging/XunitLogger.cs | 29 ++++++++++++++++++ .../Logging/XunitLoggingProvider.cs | 22 ++++++++++++++ 5 files changed, 86 insertions(+), 10 deletions(-) create mode 100644 src/OpenDDD.Tests/Logging/XunitLogger.cs create mode 100644 src/OpenDDD.Tests/Logging/XunitLoggingProvider.cs diff --git a/src/OpenDDD.Tests/Base/IntegrationTests.cs b/src/OpenDDD.Tests/Base/IntegrationTests.cs index 525f8f2..81fda57 100644 --- a/src/OpenDDD.Tests/Base/IntegrationTests.cs +++ b/src/OpenDDD.Tests/Base/IntegrationTests.cs @@ -1,8 +1,32 @@ -namespace OpenDDD.Tests.Base +using Microsoft.Extensions.Logging; +using OpenDDD.Tests.Logging; +using Xunit.Abstractions; + +namespace OpenDDD.Tests.Base { [Trait("Category", "Integration")] - public class IntegrationTests + public class IntegrationTests : IDisposable { - + protected readonly ILoggerFactory LoggerFactory; + protected readonly ILogger Logger; + + public IntegrationTests(ITestOutputHelper testOutputHelper, bool enableLogging = false) + { + LoggerFactory = Microsoft.Extensions.Logging.LoggerFactory.Create(builder => + { + if (enableLogging) + { + builder.AddProvider(new XunitLoggerProvider(testOutputHelper)); + builder.SetMinimumLevel(LogLevel.Debug); + } + }); + + Logger = LoggerFactory.CreateLogger(GetType()); + } + + public void Dispose() + { + LoggerFactory.Dispose(); + } } } diff --git a/src/OpenDDD.Tests/Integration/Infrastructure/Events/Azure/AzureServiceBusMessagingProviderTests.cs b/src/OpenDDD.Tests/Integration/Infrastructure/Events/Azure/AzureServiceBusMessagingProviderTests.cs index 5c1af2c..3e17074 100644 --- a/src/OpenDDD.Tests/Integration/Infrastructure/Events/Azure/AzureServiceBusMessagingProviderTests.cs +++ b/src/OpenDDD.Tests/Integration/Infrastructure/Events/Azure/AzureServiceBusMessagingProviderTests.cs @@ -5,6 +5,7 @@ using OpenDDD.Tests.Base; using Azure.Messaging.ServiceBus; using Azure.Messaging.ServiceBus.Administration; +using Xunit.Abstractions; namespace OpenDDD.Tests.Integration.Infrastructure.Events.Azure { @@ -17,7 +18,7 @@ public class AzureServiceBusMessagingProviderTests : IntegrationTests, IAsyncLif private readonly ServiceBusClient _serviceBusClient; private readonly AzureServiceBusMessagingProvider _messagingProvider; - public AzureServiceBusMessagingProviderTests() + public AzureServiceBusMessagingProviderTests(ITestOutputHelper testOutputHelper) : base(testOutputHelper) { _connectionString = Environment.GetEnvironmentVariable("AZURE_SERVICE_BUS_CONNECTION_STRING") ?? throw new InvalidOperationException("AZURE_SERVICE_BUS_CONNECTION_STRING is not set."); @@ -35,13 +36,12 @@ public AzureServiceBusMessagingProviderTests() public async Task InitializeAsync() { - // Start each test with an empty service bus namespace await CleanupTopicsAndSubscriptionsAsync(); } public async Task DisposeAsync() { - // Don't delete after latest run test in case we need to inspect what was created + } private async Task CleanupTopicsAndSubscriptionsAsync() diff --git a/src/OpenDDD.Tests/Integration/Infrastructure/Events/RabbitMq/RabbitMqMessagingProviderTests.cs b/src/OpenDDD.Tests/Integration/Infrastructure/Events/RabbitMq/RabbitMqMessagingProviderTests.cs index c0640bc..e0b9030 100644 --- a/src/OpenDDD.Tests/Integration/Infrastructure/Events/RabbitMq/RabbitMqMessagingProviderTests.cs +++ b/src/OpenDDD.Tests/Integration/Infrastructure/Events/RabbitMq/RabbitMqMessagingProviderTests.cs @@ -6,6 +6,7 @@ using OpenDDD.Tests.Base; using RabbitMQ.Client; using RabbitMQ.Client.Exceptions; +using Xunit.Abstractions; namespace OpenDDD.Tests.Integration.Infrastructure.Events.RabbitMq { @@ -22,7 +23,7 @@ public class RabbitMqMessagingProviderTests : IntegrationTests, IAsyncLifetime private readonly string _testTopic = "OpenDddTestTopic"; private readonly string _testConsumerGroup = "OpenDddTestGroup"; - public RabbitMqMessagingProviderTests() + public RabbitMqMessagingProviderTests(ITestOutputHelper testOutputHelper) : base(testOutputHelper) { _loggerMock = new Mock>(); @@ -42,12 +43,12 @@ public RabbitMqMessagingProviderTests() public async Task InitializeAsync() { await EnsureConnectionAndChannelOpenAsync(); - await DeleteExchangesAndQueuesAsync(); + await CleanupExchangesAndQueuesAsync(); } public async Task DisposeAsync() { - await DeleteExchangesAndQueuesAsync(); + await CleanupExchangesAndQueuesAsync(); if (_channel is not null) { @@ -102,7 +103,7 @@ private async Task EnsureConnectionAndChannelOpenAsync() } } - private async Task DeleteExchangesAndQueuesAsync() + private async Task CleanupExchangesAndQueuesAsync() { try { diff --git a/src/OpenDDD.Tests/Logging/XunitLogger.cs b/src/OpenDDD.Tests/Logging/XunitLogger.cs new file mode 100644 index 0000000..3fed942 --- /dev/null +++ b/src/OpenDDD.Tests/Logging/XunitLogger.cs @@ -0,0 +1,29 @@ +using Microsoft.Extensions.Logging; +using Xunit.Abstractions; + +namespace OpenDDD.Tests.Logging +{ + public class XunitLogger : ILogger + { + private readonly ITestOutputHelper _testOutputHelper; + private readonly string _categoryName; + + public XunitLogger(ITestOutputHelper testOutputHelper, string categoryName) + { + _testOutputHelper = testOutputHelper; + _categoryName = categoryName; + } + + public IDisposable BeginScope(TState state) => null; + + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log(LogLevel logLevel, EventId eventId, TState state, Exception exception, Func formatter) + { + if (formatter != null) + { + _testOutputHelper.WriteLine($"[{logLevel}] {_categoryName}: {formatter(state, exception)}"); + } + } + } +} \ No newline at end of file diff --git a/src/OpenDDD.Tests/Logging/XunitLoggingProvider.cs b/src/OpenDDD.Tests/Logging/XunitLoggingProvider.cs new file mode 100644 index 0000000..e5283e4 --- /dev/null +++ b/src/OpenDDD.Tests/Logging/XunitLoggingProvider.cs @@ -0,0 +1,22 @@ +using Microsoft.Extensions.Logging; +using Xunit.Abstractions; + +namespace OpenDDD.Tests.Logging +{ + public class XunitLoggerProvider : ILoggerProvider + { + private readonly ITestOutputHelper _testOutputHelper; + + public XunitLoggerProvider(ITestOutputHelper testOutputHelper) + { + _testOutputHelper = testOutputHelper; + } + + public ILogger CreateLogger(string categoryName) + { + return new XunitLogger(_testOutputHelper, categoryName); + } + + public void Dispose() { } + } +} From 6f50ff7b42fdc1f2a0c806a7223fcc9d15794aba Mon Sep 17 00:00:00 2001 From: David Runemalm Date: Mon, 24 Feb 2025 18:57:29 +0100 Subject: [PATCH 059/109] Add kafka provider integration tests. --- .github/workflows/tests.yml | 25 +- .../Kafka/KafkaMessagingProviderTests.cs | 115 +------ .../Kafka/KafkaMessagingProviderTests.cs | 302 ++++++++++++++++++ .../Events/Kafka/KafkaTestsCollection.cs | 5 + .../OpenDddServiceCollectionExtensions.cs | 3 +- .../Kafka/Factories/IKafkaConsumerFactory.cs | 7 + .../Events/Kafka/Factories/KafkaConsumer.cs | 99 ++++++ .../Kafka/Factories/KafkaConsumerFactory.cs | 17 +- .../Events/Kafka/KafkaMessagingProvider.cs | 123 +++---- 9 files changed, 527 insertions(+), 169 deletions(-) create mode 100644 src/OpenDDD.Tests/Integration/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs create mode 100644 src/OpenDDD.Tests/Integration/Infrastructure/Events/Kafka/KafkaTestsCollection.cs create mode 100644 src/OpenDDD/Infrastructure/Events/Kafka/Factories/IKafkaConsumerFactory.cs create mode 100644 src/OpenDDD/Infrastructure/Events/Kafka/Factories/KafkaConsumer.cs diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index b42d658..c8e07e1 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -57,11 +57,20 @@ jobs: dotnet-version: [8.0.x] services: + zookeeper: + image: confluentinc/cp-zookeeper:latest + env: + ZOOKEEPER_CLIENT_PORT: 2181 + ports: + - 2181:2181 + kafka: image: confluentinc/cp-kafka:latest env: - KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:9092 + KAFKA_BROKER_ID: 1 KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:9092 + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 ports: - 9092:9092 options: --network-alias kafka @@ -126,10 +135,22 @@ jobs: done echo "RabbitMQ did not start in time!" && exit 1 + - name: Wait for Kafka to be Ready + run: | + for i in {1..10}; do + if nc -z localhost 9092; then + echo "Kafka is up!" + exit 0 + fi + echo "Waiting for Kafka..." + sleep 5 + done + echo "Kafka did not start in time!" && exit 1 + - name: Run Integration Tests working-directory: src/OpenDDD.Tests env: - KAFKA_BROKER: localhost:9092 + KAFKA_BOOTSTRAP_SERVERS: localhost:9092 RABBITMQ_HOST: localhost RABBITMQ_PORT: 5672 RABBITMQ_USERNAME: guest diff --git a/src/OpenDDD.Tests/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs b/src/OpenDDD.Tests/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs index 272e4c1..4c5224b 100644 --- a/src/OpenDDD.Tests/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs +++ b/src/OpenDDD.Tests/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs @@ -1,11 +1,9 @@ using Microsoft.Extensions.Logging; using Moq; -using Xunit; -using Confluent.Kafka; -using Confluent.Kafka.Admin; using OpenDDD.Infrastructure.Events.Kafka; using OpenDDD.Infrastructure.Events.Kafka.Factories; using OpenDDD.Tests.Base; +using Confluent.Kafka; namespace OpenDDD.Tests.Infrastructure.Events.Kafka { @@ -13,9 +11,10 @@ public class KafkaMessagingProviderTests : UnitTests { private readonly Mock> _mockProducer; private readonly Mock _mockAdminClient; - private readonly Mock _mockConsumerFactory; + private readonly Mock _mockConsumerFactory; private readonly Mock> _mockConsumer; private readonly Mock> _mockLogger; + private readonly Mock> _mockConsumerLogger; private readonly KafkaMessagingProvider _provider; private const string BootstrapServers = "localhost:9092"; private const string Topic = "test-topic"; @@ -26,9 +25,10 @@ public KafkaMessagingProviderTests() { _mockProducer = new Mock>(); _mockAdminClient = new Mock(); - _mockConsumerFactory = new Mock(BootstrapServers); + _mockConsumerFactory = new Mock(); _mockConsumer = new Mock>(); _mockLogger = new Mock>(); + _mockConsumerLogger = new Mock>(); _provider = new KafkaMessagingProvider( BootstrapServers, @@ -38,21 +38,22 @@ public KafkaMessagingProviderTests() autoCreateTopics: true, _mockLogger.Object); - // Mock factory to always return same consumer _mockConsumerFactory .Setup(f => f.Create(It.IsAny())) - .Returns(_mockConsumer.Object); + .Returns((string consumerGroup) => + { + var kafkaConsumer = new KafkaConsumer(_mockConsumer.Object, consumerGroup, _mockConsumerLogger.Object); + return kafkaConsumer; + }); - // Mock metadata retrieval for topics var metadata = new Metadata( - new List(), - new List(), + new List { new(1, "localhost", 9092) }, + new List { new(Topic, new List(), ErrorCode.NoError) }, // Ensure topic exists -1, "" ); - _mockAdminClient - .Setup(a => a.GetMetadata(Topic, It.IsAny())) + .Setup(a => a.GetMetadata(It.IsAny())) .Returns(metadata); } @@ -122,96 +123,6 @@ await Assert.ThrowsAsync(() => _provider.PublishAsync(Topic, invalidMessage, CancellationToken.None)); } - [Fact] - public async Task PublishAsync_ShouldCall_ProduceAsync() - { - // Arrange - var mockDeliveryResult = new DeliveryResult(); - _mockProducer - .Setup(p => p.ProduceAsync(It.IsAny(), It.IsAny>(), It.IsAny())) - .ReturnsAsync(mockDeliveryResult); - - // Act - await _provider.PublishAsync(Topic, Message, CancellationToken.None); - - // Assert - _mockProducer.Verify(p => p.ProduceAsync( - Topic, - It.Is>(m => m.Value == Message), - It.IsAny()), - Times.Once); - } - - [Fact] - public async Task PublishAsync_ShouldCall_CreateTopicIfNotExists_WhenAutoCreateTopicsIsEnabled() - { - // Act - await _provider.PublishAsync(Topic, Message, CancellationToken.None); - - // Assert - _mockAdminClient.Verify(a => a.CreateTopicsAsync(It.IsAny>(), null), Times.Once); - } - - [Fact] - public async Task SubscribeAsync_ShouldCreateConsumer_AndSubscribeToTopic() - { - // Act - await _provider.SubscribeAsync(Topic, ConsumerGroup, async (_, _) => await Task.CompletedTask, CancellationToken.None); - - // Assert - _mockConsumer.Verify(c => c.Subscribe(Topic), Times.Once); - } - - [Fact] - public async Task SubscribeAsync_ShouldProcessReceivedMessages() - { - // Arrange - _mockConsumer - .SetupSequence(c => c.Consume(It.IsAny())) - .Returns(new ConsumeResult - { - Message = new Message { Value = Message } - }) - .Throws(new OperationCanceledException()); // Stop after the first message - - var messageReceived = new TaskCompletionSource(); - - // Act: Start the subscription - await _provider.SubscribeAsync(Topic, ConsumerGroup, async (msg, _) => - { - if (msg == Message) messageReceived.SetResult(true); - await Task.CompletedTask; - }, CancellationToken.None); - - // Ensure the background task has enough time to consume the message - await Task.Delay(100); - - // Assert: Check if the message was received - Assert.True(await messageReceived.Task.WaitAsync(TimeSpan.FromSeconds(2)), "Message handler was not called."); - - // Ensure commit is called after processing - _mockConsumer.Verify(c => c.Commit(It.IsAny>()), Times.Once); - } - - [Fact] - public async Task SubscribeAsync_ShouldHandleConsumerExceptionsGracefully() - { - // Arrange - _mockConsumer.Setup(c => c.Consume(It.IsAny())) - .Throws(new KafkaException(ErrorCode.Local_Transport)); - - // Act - await _provider.SubscribeAsync(Topic, ConsumerGroup, async (_, _) => await Task.CompletedTask, CancellationToken.None); - - // Ensure the background task has enough time to consume the message - await Task.Delay(100); - - // Assert - _mockLogger.Verify( - l => l.Log(LogLevel.Error, It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>()), - Times.Once); - } - [Fact] public async Task DisposeAsync_ShouldDisposeAllConsumers_AndKafkaClients() { diff --git a/src/OpenDDD.Tests/Integration/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs b/src/OpenDDD.Tests/Integration/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs new file mode 100644 index 0000000..1077918 --- /dev/null +++ b/src/OpenDDD.Tests/Integration/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs @@ -0,0 +1,302 @@ +using System.Collections.Concurrent; +using Microsoft.Extensions.Logging; +using Xunit.Abstractions; +using FluentAssertions; +using OpenDDD.Infrastructure.Events.Kafka; +using OpenDDD.Infrastructure.Events.Kafka.Factories; +using OpenDDD.Tests.Base; +using Confluent.Kafka; + +namespace OpenDDD.Tests.Integration.Infrastructure.Events.Kafka +{ + [Collection("KafkaTests")] + public class KafkaMessagingProviderTests : IntegrationTests, IAsyncLifetime + { + private readonly string _bootstrapServers; + private readonly IAdminClient _adminClient; + private readonly IProducer _producer; + private readonly KafkaConsumerFactory _consumerFactory; + private readonly ILogger _logger; + private readonly ILogger _consumerLogger; + private readonly KafkaMessagingProvider _messagingProvider; + private readonly CancellationTokenSource _cts = new(); + + public KafkaMessagingProviderTests(ITestOutputHelper testOutputHelper) + : base(testOutputHelper, enableLogging: true) + { + _bootstrapServers = Environment.GetEnvironmentVariable("KAFKA_BOOTSTRAP_SERVERS") + ?? throw new InvalidOperationException("KAFKA_BOOTSTRAP_SERVERS is not set."); + + var adminClientConfig = new AdminClientConfig { BootstrapServers = _bootstrapServers }; + var producerConfig = new ProducerConfig { BootstrapServers = _bootstrapServers }; + + _adminClient = new AdminClientBuilder(adminClientConfig).Build(); + _producer = new ProducerBuilder(producerConfig).Build(); + _logger = LoggerFactory.CreateLogger(); + _consumerLogger = LoggerFactory.CreateLogger(); + _consumerFactory = new KafkaConsumerFactory(_bootstrapServers, _consumerLogger); + + _messagingProvider = new KafkaMessagingProvider( + _bootstrapServers, + _adminClient, + _producer, + _consumerFactory, + autoCreateTopics: true, + _logger); + } + + public async Task InitializeAsync() + { + await CleanupTopicsAndConsumerGroupsAsync(); + } + + public async Task DisposeAsync() + { + await _cts.CancelAsync(); + await _messagingProvider.DisposeAsync(); + } + + private async Task CleanupTopicsAndConsumerGroupsAsync() + { + try + { + // Delete test topics + var metadata = _adminClient.GetMetadata(TimeSpan.FromSeconds(5)); + var testTopics = metadata.Topics + .Where(t => t.Topic.StartsWith("test-topic-")) + .Select(t => t.Topic) + .ToList(); + + if (testTopics.Any()) + { + await _adminClient.DeleteTopicsAsync(testTopics); + _logger.LogInformation("Deleted test topics: {Topics}", string.Join(", ", testTopics)); + } + + // Delete consumer groups + var consumerGroups = _adminClient.ListGroups(TimeSpan.FromSeconds(5)) + .Where(g => g.Group.StartsWith("test-consumer-group")) + .Select(g => g.Group) + .ToList(); + + if (consumerGroups.Any()) + { + await _adminClient.DeleteGroupsAsync(consumerGroups); + _logger.LogInformation("Deleted test consumer groups: {Groups}", string.Join(", ", consumerGroups)); + } + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to clean up Kafka topics and consumer groups."); + } + } + + [Fact] + public async Task AutoCreateTopic_ShouldCreateTopicOnSubscribe_WhenSettingEnabled() + { + // Arrange + var topicName = $"test-topic-{Guid.NewGuid()}"; + var consumerGroup = "test-consumer-group"; + + // Ensure topic does not exist + var metadata = _adminClient.GetMetadata(TimeSpan.FromSeconds(5)); + metadata.Topics.Any(t => t.Topic == topicName).Should().BeFalse(); + + // Act + await _messagingProvider.SubscribeAsync(topicName, consumerGroup, async (msg, token) => + await Task.CompletedTask, _cts.Token); + + await Task.Delay(1000); + + // Assert + metadata = _adminClient.GetMetadata(TimeSpan.FromSeconds(5)); + metadata.Topics.Any(t => t.Topic == topicName).Should().BeTrue(); + } + + [Fact] + public async Task AutoCreateTopic_ShouldNotCreateTopicOnSubscribe_WhenSettingDisabled() + { + // Arrange + var topicName = $"test-topic-{Guid.NewGuid()}"; + var consumerGroup = "test-consumer-group"; + + var messagingProvider = new KafkaMessagingProvider( + _bootstrapServers, + _adminClient, + _producer, + _consumerFactory, + autoCreateTopics: false, + _logger); + + // Act & Assert + var exception = await Assert.ThrowsAsync(async () => + { + await messagingProvider.SubscribeAsync(topicName, consumerGroup, async (msg, token) => + await Task.CompletedTask, _cts.Token); + }); + + exception.Message.Should().Contain($"Topic '{topicName}' does not exist."); + } + + [Fact] + public async Task AtLeastOnceGurantee_ShouldDeliverToLateSubscriber_WhenSubscribedBefore() + { + // Arrange + var topicName = $"test-topic-{Guid.NewGuid()}"; + var consumerGroup = $"test-consumer-group-{Guid.NewGuid()}"; + var messageToSend = "Persistent Message Test"; + var firstSubscriberReceived = new TaskCompletionSource(); + var lateSubscriberReceived = new TaskCompletionSource(); + ConcurrentBag _receivedMessages = new(); + + await _messagingProvider.SubscribeAsync(topicName, consumerGroup, async (msg, token) => + { + firstSubscriberReceived.SetResult(true); + }, _cts.Token); + + await Task.Delay(500); + + await _messagingProvider.PublishAsync(topicName, messageToSend, _cts.Token); + + await firstSubscriberReceived.Task; + + await _messagingProvider.UnsubscribeAsync(topicName, consumerGroup, _cts.Token); + await Task.Delay(500); + + // Late subscriber + await _messagingProvider.SubscribeAsync(topicName, consumerGroup, async (msg, token) => + { + _receivedMessages.Add(msg); + lateSubscriberReceived.TrySetResult(true); + }, _cts.Token); + + await _messagingProvider.PublishAsync(topicName, messageToSend, _cts.Token); + + await lateSubscriberReceived.Task; + + // Assert + _receivedMessages.Should().Contain(messageToSend); + } + + [Fact] + public async Task AtLeastOnceGurantee_ShouldNotDeliverToLateSubscriber_WhenNotSubscribedBefore() + { + // Arrange + var topicName = $"test-topic-{Guid.NewGuid()}"; + var consumerGroup = "test-consumer-group"; + var messageToSend = "Non-Persistent Message Test"; + ConcurrentBag _receivedMessages = new(); + + // Act: Publish message before any subscription + await _messagingProvider.PublishAsync(topicName, messageToSend, _cts.Token); + await Task.Delay(500); + + // Late subscriber + await _messagingProvider.SubscribeAsync(topicName, consumerGroup, async (msg, token) => + { + _receivedMessages.Add(msg); + }, _cts.Token); + + await Task.Delay(1000); + + // Assert + _receivedMessages.Should().NotContain(messageToSend); + } + + [Fact] + public async Task AtLeastOnceGurantee_ShouldRedeliverLater_WhenMessageNotAcked() + { + // Arrange + var topicName = $"test-topic-{Guid.NewGuid()}"; + var consumerGroup = "test-consumer-group"; + var messageToSend = "Redelivery Test"; + ConcurrentBag _receivedMessages = new(); + + async Task FaultyHandler(string msg, CancellationToken token) + { + _receivedMessages.Add(msg); + throw new Exception("Simulated consumer crash before acknowledgment."); + } + + await _messagingProvider.SubscribeAsync(topicName, consumerGroup, FaultyHandler, _cts.Token); + await Task.Delay(500); + + // Act: Publish message + await _messagingProvider.PublishAsync(topicName, messageToSend, _cts.Token); + + // Wait for redelivery + for (int i = 0; i < 300; i++) + { + if (_receivedMessages.Count > 1) break; + await Task.Delay(1000); // Check every second + } + + // Assert: The message should be received multiple times due to reattempts + _receivedMessages.Count.Should().BeGreaterThan(1); + } + + [Fact] + public async Task CompetingConsumers_ShouldDeliverOnlyOnce_WhenMultipleConsumersInGroup() + { + // Arrange + var topicName = $"test-topic-{Guid.NewGuid()}"; + var consumerGroup = "test-consumer-group"; + var receivedMessages = new ConcurrentDictionary(); + var messageToSend = "Competing Consumer Test"; + + async Task MessageHandler(string msg, CancellationToken token) + { + receivedMessages.AddOrUpdate("received", 1, (key, value) => value + 1); + } + + // Multiple competing consumers in the same group + await _messagingProvider.SubscribeAsync(topicName, consumerGroup, MessageHandler, _cts.Token); + await _messagingProvider.SubscribeAsync(topicName, consumerGroup, MessageHandler, _cts.Token); + await Task.Delay(500); + + // Act + await _messagingProvider.PublishAsync(topicName, messageToSend, _cts.Token); + + // Wait for processing + await Task.Delay(5000); + + // Assert: Only one of the competing consumers should receive the message + receivedMessages.GetValueOrDefault("received", 0).Should().Be(1); + } + + [Fact] + public async Task CompetingConsumers_ShouldDistributeEvenly_WhenMultipleConsumersInGroup() + { + // Arrange + var topicName = $"test-topic-{Guid.NewGuid()}"; + var consumerGroup = "test-consumer-group"; + var receivedMessages = new ConcurrentDictionary(); + var totalMessages = 10; + + async Task MessageHandler(string msg, CancellationToken token) + { + receivedMessages.AddOrUpdate(msg, 1, (key, value) => value + 1); + } + + await _messagingProvider.SubscribeAsync(topicName, consumerGroup, MessageHandler, _cts.Token); + await _messagingProvider.SubscribeAsync(topicName, consumerGroup, MessageHandler, _cts.Token); + await Task.Delay(500); + + // Act: Publish multiple messages + for (int i = 0; i < totalMessages; i++) + { + await _messagingProvider.PublishAsync(topicName, $"Message {i}", _cts.Token); + } + + await Task.Delay(2000); + + // Assert: Messages should be evenly distributed across consumers + var messageCounts = receivedMessages.Values; + var minReceived = messageCounts.Min(); + var maxReceived = messageCounts.Max(); + + Assert.True(maxReceived - minReceived <= 1, + "Messages should be evenly distributed among competing consumers."); + } + } +} diff --git a/src/OpenDDD.Tests/Integration/Infrastructure/Events/Kafka/KafkaTestsCollection.cs b/src/OpenDDD.Tests/Integration/Infrastructure/Events/Kafka/KafkaTestsCollection.cs new file mode 100644 index 0000000..f5c5495 --- /dev/null +++ b/src/OpenDDD.Tests/Integration/Infrastructure/Events/Kafka/KafkaTestsCollection.cs @@ -0,0 +1,5 @@ +namespace OpenDDD.Tests.Integration.Infrastructure.Events.Kafka +{ + [CollectionDefinition("KafkaTests", DisableParallelization = true)] + public class KafkaTestsCollection { } +} diff --git a/src/OpenDDD/API/Extensions/OpenDddServiceCollectionExtensions.cs b/src/OpenDDD/API/Extensions/OpenDddServiceCollectionExtensions.cs index f4dd682..53dc91c 100644 --- a/src/OpenDDD/API/Extensions/OpenDddServiceCollectionExtensions.cs +++ b/src/OpenDDD/API/Extensions/OpenDddServiceCollectionExtensions.cs @@ -338,11 +338,12 @@ private static void AddKafka(this IServiceCollection services) throw new InvalidOperationException("Kafka bootstrap servers must be configured."); var logger = provider.GetRequiredService>(); + var consumerLogger = provider.GetRequiredService>(); return new KafkaMessagingProvider( kafkaOptions.BootstrapServers, new AdminClientBuilder(new AdminClientConfig { BootstrapServers = kafkaOptions.BootstrapServers, ClientId = "OpenDDD" }).Build(), new ProducerBuilder(new ProducerConfig { BootstrapServers = kafkaOptions.BootstrapServers, ClientId = "OpenDDD" }).Build(), - new KafkaConsumerFactory(kafkaOptions.BootstrapServers), + new KafkaConsumerFactory(kafkaOptions.BootstrapServers, consumerLogger), kafkaOptions.AutoCreateTopics, logger ); diff --git a/src/OpenDDD/Infrastructure/Events/Kafka/Factories/IKafkaConsumerFactory.cs b/src/OpenDDD/Infrastructure/Events/Kafka/Factories/IKafkaConsumerFactory.cs new file mode 100644 index 0000000..9410f36 --- /dev/null +++ b/src/OpenDDD/Infrastructure/Events/Kafka/Factories/IKafkaConsumerFactory.cs @@ -0,0 +1,7 @@ +namespace OpenDDD.Infrastructure.Events.Kafka.Factories +{ + public interface IKafkaConsumerFactory + { + KafkaConsumer Create(string consumerGroup); + } +} diff --git a/src/OpenDDD/Infrastructure/Events/Kafka/Factories/KafkaConsumer.cs b/src/OpenDDD/Infrastructure/Events/Kafka/Factories/KafkaConsumer.cs new file mode 100644 index 0000000..82a0885 --- /dev/null +++ b/src/OpenDDD/Infrastructure/Events/Kafka/Factories/KafkaConsumer.cs @@ -0,0 +1,99 @@ +using Confluent.Kafka; +using Microsoft.Extensions.Logging; + +namespace OpenDDD.Infrastructure.Events.Kafka.Factories +{ + public class KafkaConsumer : IDisposable + { + private readonly IConsumer _consumer; + private readonly ILogger _logger; + private readonly CancellationTokenSource _cts = new(); + private Task? _consumerTask; + private bool _disposed; + + public string ConsumerGroup { get; } + public HashSet SubscribedTopics { get; } = new(); + + public KafkaConsumer(IConsumer consumer, string consumerGroup, ILogger logger) + { + _consumer = consumer ?? throw new ArgumentNullException(nameof(consumer)); + ConsumerGroup = consumerGroup ?? throw new ArgumentNullException(nameof(consumerGroup)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public void Subscribe(string topic) + { + if (SubscribedTopics.Contains(topic)) return; + + _consumer.Subscribe(topic); + SubscribedTopics.Add(topic); + } + + public void StartProcessing(Func messageHandler, CancellationToken globalToken) + { + if (_consumerTask != null) return; + + var linkedCts = CancellationTokenSource.CreateLinkedTokenSource(globalToken, _cts.Token); + _consumerTask = Task.Run(() => ConsumeLoop(messageHandler, linkedCts.Token), linkedCts.Token); + } + + private async Task ConsumeLoop(Func messageHandler, CancellationToken token) + { + while (!token.IsCancellationRequested) + { + ConsumeResult? result = null; + + try + { + result = _consumer.Consume(token); + if (result?.Message == null) continue; + + await messageHandler(result.Message.Value, token); + _consumer.Commit(result); + } + catch (OperationCanceledException) + { + break; + } + catch (Exception ex) + { + _logger.LogError(ex, "Error in Kafka consumer loop. Retrying..."); + + try + { + await Task.Delay(5000, token); + } + catch (TaskCanceledException) + { + break; + } + + if (result != null) + { + _consumer.Seek(new TopicPartitionOffset(result.TopicPartition, result.Offset)); + } + } + } + } + + public async Task StopProcessingAsync() + { + _cts.Cancel(); + if (_consumerTask != null) + { + await _consumerTask; + _consumerTask = null; + } + } + + public void Dispose() + { + if (_disposed) return; + _disposed = true; + + _consumer.Close(); + _consumer.Dispose(); + _cts.Dispose(); + } + } +} diff --git a/src/OpenDDD/Infrastructure/Events/Kafka/Factories/KafkaConsumerFactory.cs b/src/OpenDDD/Infrastructure/Events/Kafka/Factories/KafkaConsumerFactory.cs index 648fdab..f042ee3 100644 --- a/src/OpenDDD/Infrastructure/Events/Kafka/Factories/KafkaConsumerFactory.cs +++ b/src/OpenDDD/Infrastructure/Events/Kafka/Factories/KafkaConsumerFactory.cs @@ -1,20 +1,23 @@ using Confluent.Kafka; +using Microsoft.Extensions.Logging; namespace OpenDDD.Infrastructure.Events.Kafka.Factories { - public class KafkaConsumerFactory + public class KafkaConsumerFactory : IKafkaConsumerFactory { private readonly string _bootstrapServers; + private readonly ILogger _logger; - public KafkaConsumerFactory(string bootstrapServers) + public KafkaConsumerFactory(string bootstrapServers, ILogger logger) { if (string.IsNullOrWhiteSpace(bootstrapServers)) throw new ArgumentException("Kafka bootstrap servers must be configured.", nameof(bootstrapServers)); _bootstrapServers = bootstrapServers; + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); } - public virtual IConsumer Create(string consumerGroup) + public virtual KafkaConsumer Create(string consumerGroup) { var consumerConfig = new ConsumerConfig { @@ -22,12 +25,16 @@ public virtual IConsumer Create(string consumerGroup) ClientId = "OpenDDD", GroupId = consumerGroup, EnableAutoCommit = false, - AutoOffsetReset = AutoOffsetReset.Earliest + AutoOffsetReset = AutoOffsetReset.Latest, + MaxPollIntervalMs = 300000, // Max time consumer can take to process a message before kafka removes it from group + SessionTimeoutMs = 45000, // Time before kafka assumes consumer is dead if it stops sending heartbeats + HeartbeatIntervalMs = 3000 // Frequence of heartbeats to kafka }; - return new ConsumerBuilder(consumerConfig) + var consumer = new ConsumerBuilder(consumerConfig) .SetValueDeserializer(Deserializers.Utf8) .Build(); + return new KafkaConsumer(consumer, consumerGroup, _logger); } } } diff --git a/src/OpenDDD/Infrastructure/Events/Kafka/KafkaMessagingProvider.cs b/src/OpenDDD/Infrastructure/Events/Kafka/KafkaMessagingProvider.cs index 792c253..4168d69 100644 --- a/src/OpenDDD/Infrastructure/Events/Kafka/KafkaMessagingProvider.cs +++ b/src/OpenDDD/Infrastructure/Events/Kafka/KafkaMessagingProvider.cs @@ -12,10 +12,9 @@ public class KafkaMessagingProvider : IMessagingProvider, IAsyncDisposable private readonly IProducer _producer; private readonly IAdminClient _adminClient; private readonly bool _autoCreateTopics; - private readonly KafkaConsumerFactory _consumerFactory; + private readonly IKafkaConsumerFactory _consumerFactory; private readonly ILogger _logger; - private readonly ConcurrentBag> _consumers = new(); - private readonly List _consumerTasks = new(); + private readonly ConcurrentDictionary _consumers = new(); private readonly CancellationTokenSource _cts = new(); private bool _disposed; @@ -23,7 +22,7 @@ public KafkaMessagingProvider( string bootstrapServers, IAdminClient adminClient, IProducer producer, - KafkaConsumerFactory consumerFactory, + IKafkaConsumerFactory consumerFactory, bool autoCreateTopics, ILogger logger) { @@ -52,61 +51,47 @@ public async Task SubscribeAsync( if (messageHandler is null) throw new ArgumentNullException(nameof(messageHandler)); - + if (_autoCreateTopics) { await CreateTopicIfNotExistsAsync(topic, cancellationToken); } + else + { + var metadata = _adminClient.GetMetadata(TimeSpan.FromSeconds(5)); + if (!metadata.Topics.Any(t => t.Topic == topic)) + { + _logger.LogError("Cannot subscribe to non-existent topic: {Topic}", topic); + throw new KafkaException(new Error(ErrorCode.UnknownTopicOrPart, $"Topic '{topic}' does not exist.")); + } + } - var consumer = _consumerFactory.Create(consumerGroup); - _consumers.Add(consumer); - consumer.Subscribe(topic); + var kafkaConsumer = _consumers.GetOrAdd(consumerGroup, _ => _consumerFactory.Create(consumerGroup)); + + kafkaConsumer.Subscribe(topic); _logger.LogDebug("Subscribed to Kafka topic '{Topic}' with consumer group '{ConsumerGroup}'", topic, consumerGroup); - - var consumerTask = Task.Run(() => StartConsumerLoop(consumer, messageHandler, _cts.Token), _cts.Token); - _consumerTasks.Add(consumerTask); - } - - public Task UnsubscribeAsync(string topic, string consumerGroup, CancellationToken cancellationToken = default) - { - throw new NotImplementedException(); + + kafkaConsumer.StartProcessing(messageHandler, _cts.Token); } - private async Task StartConsumerLoop( - IConsumer consumer, - Func messageHandler, - CancellationToken cancellationToken) + public async Task UnsubscribeAsync(string topic, string consumerGroup, CancellationToken cancellationToken) { - try - { - while (!cancellationToken.IsCancellationRequested) - { - // Seems like consume don't respect cancellation token. - // See: https://github.com/confluentinc/confluent-kafka-dotnet/issues/1085 - var result = consumer.Consume(cancellationToken); - if (result?.Message != null) - { - _logger.LogDebug("Received message from Kafka: {Message}", result.Message.Value); - await messageHandler(result.Message.Value, cancellationToken); - consumer.Commit(result); - _logger.LogDebug("Message processed and offset committed: {Offset}", result.Offset); - } - } - } - catch (OperationCanceledException) - { - _logger.LogDebug("Kafka consumer loop cancelled."); - } - catch (Exception ex) - { - _logger.LogError(ex, "Error occurred in Kafka consumer loop."); - } - finally + if (string.IsNullOrWhiteSpace(topic)) + throw new ArgumentException("Topic cannot be null or empty.", nameof(topic)); + + if (string.IsNullOrWhiteSpace(consumerGroup)) + throw new ArgumentException("Consumer group cannot be null or empty.", nameof(consumerGroup)); + + if (!_consumers.TryRemove(consumerGroup, out var kafkaConsumer)) { - _logger.LogDebug("Closing consumer."); - consumer.Close(); + _logger.LogWarning("No active consumer found for consumer group '{ConsumerGroup}'. It may have already stopped.", consumerGroup); + return; } + + _logger.LogInformation("Stopping consumer group '{ConsumerGroup}'...", consumerGroup); + await kafkaConsumer.StopProcessingAsync(); + kafkaConsumer.Dispose(); } public async Task PublishAsync(string topic, string message, CancellationToken cancellationToken) @@ -130,21 +115,46 @@ private async Task CreateTopicIfNotExistsAsync(string topic, CancellationToken c { try { - var metadata = _adminClient.GetMetadata(topic, TimeSpan.FromSeconds(5)); - if (metadata.Topics.Any(t => t.Topic == topic)) return; // Topic exists + var metadata = _adminClient.GetMetadata(TimeSpan.FromSeconds(5)); + + if (metadata.Topics.Any(t => t.Topic == topic)) + { + _logger.LogDebug("Topic '{Topic}' already exists. Skipping creation.", topic); + return; + } _logger.LogDebug("Creating Kafka topic: {Topic}", topic); await _adminClient.CreateTopicsAsync(new[] { new TopicSpecification { Name = topic, NumPartitions = 1, ReplicationFactor = 1 } }, null); + + for (int i = 0; i < 10; i++) + { + await Task.Delay(500, cancellationToken); + metadata = _adminClient.GetMetadata(TimeSpan.FromSeconds(1)); + + if (metadata.Topics.Any(t => t.Topic == topic)) + { + _logger.LogDebug("Kafka topic '{Topic}' is now available.", topic); + return; + } + } + + throw new KafkaException(new Error(ErrorCode.UnknownTopicOrPart, $"Failed to create topic '{topic}' within timeout.")); + } + catch (KafkaException ex) + { + _logger.LogError(ex, "Kafka error while creating topic {Topic}: {Message}", topic, ex.Message); + throw; } catch (Exception ex) { - _logger.LogError("Could not check or create Kafka topic {Topic}: {Message}", topic, ex.Message); + _logger.LogError(ex, "Unexpected error while creating Kafka topic {Topic}", topic); + throw new InvalidOperationException($"Failed to create topic '{topic}'", ex); } } - + public async ValueTask DisposeAsync() { if (_disposed) return; @@ -152,22 +162,17 @@ public async ValueTask DisposeAsync() _logger.LogDebug("Disposing KafkaMessagingProvider..."); - _logger.LogDebug("Cancelling consumer tasks..."); _cts.Cancel(); - _logger.LogDebug("Waiting for all consumer tasks to complete..."); - await Task.WhenAll(_consumerTasks); + var tasks = _consumers.Values.Select(c => c.StopProcessingAsync()).ToList(); + await Task.WhenAll(tasks); - foreach (var consumer in _consumers) + foreach (var consumer in _consumers.Values) { - _logger.LogDebug("Disposing consumer..."); consumer.Dispose(); } - _logger.LogDebug("Disposing producer..."); _producer.Dispose(); - - _logger.LogDebug("Disposing admin client..."); _adminClient.Dispose(); } } From 6382e10db5e47a8f60b17310f77b40fd3bd4abcf Mon Sep 17 00:00:00 2001 From: David Runemalm Date: Tue, 25 Feb 2025 09:58:53 +0100 Subject: [PATCH 060/109] Debug tests in github actions. --- .../Kafka/KafkaMessagingProviderTests.cs | 370 +++++++++--------- .../RabbitMqMessagingProviderTests.cs | 2 +- .../Events/Kafka/KafkaMessagingProvider.cs | 2 +- 3 files changed, 187 insertions(+), 187 deletions(-) diff --git a/src/OpenDDD.Tests/Integration/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs b/src/OpenDDD.Tests/Integration/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs index 1077918..1f80814 100644 --- a/src/OpenDDD.Tests/Integration/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs +++ b/src/OpenDDD.Tests/Integration/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs @@ -113,190 +113,190 @@ await _messagingProvider.SubscribeAsync(topicName, consumerGroup, async (msg, to metadata.Topics.Any(t => t.Topic == topicName).Should().BeTrue(); } - [Fact] - public async Task AutoCreateTopic_ShouldNotCreateTopicOnSubscribe_WhenSettingDisabled() - { - // Arrange - var topicName = $"test-topic-{Guid.NewGuid()}"; - var consumerGroup = "test-consumer-group"; - - var messagingProvider = new KafkaMessagingProvider( - _bootstrapServers, - _adminClient, - _producer, - _consumerFactory, - autoCreateTopics: false, - _logger); - - // Act & Assert - var exception = await Assert.ThrowsAsync(async () => - { - await messagingProvider.SubscribeAsync(topicName, consumerGroup, async (msg, token) => - await Task.CompletedTask, _cts.Token); - }); - - exception.Message.Should().Contain($"Topic '{topicName}' does not exist."); - } - - [Fact] - public async Task AtLeastOnceGurantee_ShouldDeliverToLateSubscriber_WhenSubscribedBefore() - { - // Arrange - var topicName = $"test-topic-{Guid.NewGuid()}"; - var consumerGroup = $"test-consumer-group-{Guid.NewGuid()}"; - var messageToSend = "Persistent Message Test"; - var firstSubscriberReceived = new TaskCompletionSource(); - var lateSubscriberReceived = new TaskCompletionSource(); - ConcurrentBag _receivedMessages = new(); - - await _messagingProvider.SubscribeAsync(topicName, consumerGroup, async (msg, token) => - { - firstSubscriberReceived.SetResult(true); - }, _cts.Token); - - await Task.Delay(500); - - await _messagingProvider.PublishAsync(topicName, messageToSend, _cts.Token); - - await firstSubscriberReceived.Task; - - await _messagingProvider.UnsubscribeAsync(topicName, consumerGroup, _cts.Token); - await Task.Delay(500); - - // Late subscriber - await _messagingProvider.SubscribeAsync(topicName, consumerGroup, async (msg, token) => - { - _receivedMessages.Add(msg); - lateSubscriberReceived.TrySetResult(true); - }, _cts.Token); - - await _messagingProvider.PublishAsync(topicName, messageToSend, _cts.Token); - - await lateSubscriberReceived.Task; - - // Assert - _receivedMessages.Should().Contain(messageToSend); - } - - [Fact] - public async Task AtLeastOnceGurantee_ShouldNotDeliverToLateSubscriber_WhenNotSubscribedBefore() - { - // Arrange - var topicName = $"test-topic-{Guid.NewGuid()}"; - var consumerGroup = "test-consumer-group"; - var messageToSend = "Non-Persistent Message Test"; - ConcurrentBag _receivedMessages = new(); - - // Act: Publish message before any subscription - await _messagingProvider.PublishAsync(topicName, messageToSend, _cts.Token); - await Task.Delay(500); - - // Late subscriber - await _messagingProvider.SubscribeAsync(topicName, consumerGroup, async (msg, token) => - { - _receivedMessages.Add(msg); - }, _cts.Token); - - await Task.Delay(1000); - - // Assert - _receivedMessages.Should().NotContain(messageToSend); - } - - [Fact] - public async Task AtLeastOnceGurantee_ShouldRedeliverLater_WhenMessageNotAcked() - { - // Arrange - var topicName = $"test-topic-{Guid.NewGuid()}"; - var consumerGroup = "test-consumer-group"; - var messageToSend = "Redelivery Test"; - ConcurrentBag _receivedMessages = new(); - - async Task FaultyHandler(string msg, CancellationToken token) - { - _receivedMessages.Add(msg); - throw new Exception("Simulated consumer crash before acknowledgment."); - } - - await _messagingProvider.SubscribeAsync(topicName, consumerGroup, FaultyHandler, _cts.Token); - await Task.Delay(500); - - // Act: Publish message - await _messagingProvider.PublishAsync(topicName, messageToSend, _cts.Token); - - // Wait for redelivery - for (int i = 0; i < 300; i++) - { - if (_receivedMessages.Count > 1) break; - await Task.Delay(1000); // Check every second - } - - // Assert: The message should be received multiple times due to reattempts - _receivedMessages.Count.Should().BeGreaterThan(1); - } - - [Fact] - public async Task CompetingConsumers_ShouldDeliverOnlyOnce_WhenMultipleConsumersInGroup() - { - // Arrange - var topicName = $"test-topic-{Guid.NewGuid()}"; - var consumerGroup = "test-consumer-group"; - var receivedMessages = new ConcurrentDictionary(); - var messageToSend = "Competing Consumer Test"; - - async Task MessageHandler(string msg, CancellationToken token) - { - receivedMessages.AddOrUpdate("received", 1, (key, value) => value + 1); - } - - // Multiple competing consumers in the same group - await _messagingProvider.SubscribeAsync(topicName, consumerGroup, MessageHandler, _cts.Token); - await _messagingProvider.SubscribeAsync(topicName, consumerGroup, MessageHandler, _cts.Token); - await Task.Delay(500); - - // Act - await _messagingProvider.PublishAsync(topicName, messageToSend, _cts.Token); - - // Wait for processing - await Task.Delay(5000); - - // Assert: Only one of the competing consumers should receive the message - receivedMessages.GetValueOrDefault("received", 0).Should().Be(1); - } - - [Fact] - public async Task CompetingConsumers_ShouldDistributeEvenly_WhenMultipleConsumersInGroup() - { - // Arrange - var topicName = $"test-topic-{Guid.NewGuid()}"; - var consumerGroup = "test-consumer-group"; - var receivedMessages = new ConcurrentDictionary(); - var totalMessages = 10; - - async Task MessageHandler(string msg, CancellationToken token) - { - receivedMessages.AddOrUpdate(msg, 1, (key, value) => value + 1); - } - - await _messagingProvider.SubscribeAsync(topicName, consumerGroup, MessageHandler, _cts.Token); - await _messagingProvider.SubscribeAsync(topicName, consumerGroup, MessageHandler, _cts.Token); - await Task.Delay(500); - - // Act: Publish multiple messages - for (int i = 0; i < totalMessages; i++) - { - await _messagingProvider.PublishAsync(topicName, $"Message {i}", _cts.Token); - } - - await Task.Delay(2000); - - // Assert: Messages should be evenly distributed across consumers - var messageCounts = receivedMessages.Values; - var minReceived = messageCounts.Min(); - var maxReceived = messageCounts.Max(); - - Assert.True(maxReceived - minReceived <= 1, - "Messages should be evenly distributed among competing consumers."); - } + // [Fact] + // public async Task AutoCreateTopic_ShouldNotCreateTopicOnSubscribe_WhenSettingDisabled() + // { + // // Arrange + // var topicName = $"test-topic-{Guid.NewGuid()}"; + // var consumerGroup = "test-consumer-group"; + // + // var messagingProvider = new KafkaMessagingProvider( + // _bootstrapServers, + // _adminClient, + // _producer, + // _consumerFactory, + // autoCreateTopics: false, + // _logger); + // + // // Act & Assert + // var exception = await Assert.ThrowsAsync(async () => + // { + // await messagingProvider.SubscribeAsync(topicName, consumerGroup, async (msg, token) => + // await Task.CompletedTask, _cts.Token); + // }); + // + // exception.Message.Should().Contain($"Topic '{topicName}' does not exist."); + // } + // + // [Fact] + // public async Task AtLeastOnceGurantee_ShouldDeliverToLateSubscriber_WhenSubscribedBefore() + // { + // // Arrange + // var topicName = $"test-topic-{Guid.NewGuid()}"; + // var consumerGroup = $"test-consumer-group-{Guid.NewGuid()}"; + // var messageToSend = "Persistent Message Test"; + // var firstSubscriberReceived = new TaskCompletionSource(); + // var lateSubscriberReceived = new TaskCompletionSource(); + // ConcurrentBag _receivedMessages = new(); + // + // await _messagingProvider.SubscribeAsync(topicName, consumerGroup, async (msg, token) => + // { + // firstSubscriberReceived.SetResult(true); + // }, _cts.Token); + // + // await Task.Delay(500); + // + // await _messagingProvider.PublishAsync(topicName, messageToSend, _cts.Token); + // + // await firstSubscriberReceived.Task; + // + // await _messagingProvider.UnsubscribeAsync(topicName, consumerGroup, _cts.Token); + // await Task.Delay(500); + // + // // Late subscriber + // await _messagingProvider.SubscribeAsync(topicName, consumerGroup, async (msg, token) => + // { + // _receivedMessages.Add(msg); + // lateSubscriberReceived.TrySetResult(true); + // }, _cts.Token); + // + // await _messagingProvider.PublishAsync(topicName, messageToSend, _cts.Token); + // + // await lateSubscriberReceived.Task; + // + // // Assert + // _receivedMessages.Should().Contain(messageToSend); + // } + // + // [Fact] + // public async Task AtLeastOnceGurantee_ShouldNotDeliverToLateSubscriber_WhenNotSubscribedBefore() + // { + // // Arrange + // var topicName = $"test-topic-{Guid.NewGuid()}"; + // var consumerGroup = "test-consumer-group"; + // var messageToSend = "Non-Persistent Message Test"; + // ConcurrentBag _receivedMessages = new(); + // + // // Act: Publish message before any subscription + // await _messagingProvider.PublishAsync(topicName, messageToSend, _cts.Token); + // await Task.Delay(500); + // + // // Late subscriber + // await _messagingProvider.SubscribeAsync(topicName, consumerGroup, async (msg, token) => + // { + // _receivedMessages.Add(msg); + // }, _cts.Token); + // + // await Task.Delay(1000); + // + // // Assert + // _receivedMessages.Should().NotContain(messageToSend); + // } + // + // [Fact] + // public async Task AtLeastOnceGurantee_ShouldRedeliverLater_WhenMessageNotAcked() + // { + // // Arrange + // var topicName = $"test-topic-{Guid.NewGuid()}"; + // var consumerGroup = "test-consumer-group"; + // var messageToSend = "Redelivery Test"; + // ConcurrentBag _receivedMessages = new(); + // + // async Task FaultyHandler(string msg, CancellationToken token) + // { + // _receivedMessages.Add(msg); + // throw new Exception("Simulated consumer crash before acknowledgment."); + // } + // + // await _messagingProvider.SubscribeAsync(topicName, consumerGroup, FaultyHandler, _cts.Token); + // await Task.Delay(500); + // + // // Act: Publish message + // await _messagingProvider.PublishAsync(topicName, messageToSend, _cts.Token); + // + // // Wait for redelivery + // for (int i = 0; i < 300; i++) + // { + // if (_receivedMessages.Count > 1) break; + // await Task.Delay(1000); // Check every second + // } + // + // // Assert: The message should be received multiple times due to reattempts + // _receivedMessages.Count.Should().BeGreaterThan(1); + // } + // + // [Fact] + // public async Task CompetingConsumers_ShouldDeliverOnlyOnce_WhenMultipleConsumersInGroup() + // { + // // Arrange + // var topicName = $"test-topic-{Guid.NewGuid()}"; + // var consumerGroup = "test-consumer-group"; + // var receivedMessages = new ConcurrentDictionary(); + // var messageToSend = "Competing Consumer Test"; + // + // async Task MessageHandler(string msg, CancellationToken token) + // { + // receivedMessages.AddOrUpdate("received", 1, (key, value) => value + 1); + // } + // + // // Multiple competing consumers in the same group + // await _messagingProvider.SubscribeAsync(topicName, consumerGroup, MessageHandler, _cts.Token); + // await _messagingProvider.SubscribeAsync(topicName, consumerGroup, MessageHandler, _cts.Token); + // await Task.Delay(500); + // + // // Act + // await _messagingProvider.PublishAsync(topicName, messageToSend, _cts.Token); + // + // // Wait for processing + // await Task.Delay(5000); + // + // // Assert: Only one of the competing consumers should receive the message + // receivedMessages.GetValueOrDefault("received", 0).Should().Be(1); + // } + // + // [Fact] + // public async Task CompetingConsumers_ShouldDistributeEvenly_WhenMultipleConsumersInGroup() + // { + // // Arrange + // var topicName = $"test-topic-{Guid.NewGuid()}"; + // var consumerGroup = "test-consumer-group"; + // var receivedMessages = new ConcurrentDictionary(); + // var totalMessages = 10; + // + // async Task MessageHandler(string msg, CancellationToken token) + // { + // receivedMessages.AddOrUpdate(msg, 1, (key, value) => value + 1); + // } + // + // await _messagingProvider.SubscribeAsync(topicName, consumerGroup, MessageHandler, _cts.Token); + // await _messagingProvider.SubscribeAsync(topicName, consumerGroup, MessageHandler, _cts.Token); + // await Task.Delay(500); + // + // // Act: Publish multiple messages + // for (int i = 0; i < totalMessages; i++) + // { + // await _messagingProvider.PublishAsync(topicName, $"Message {i}", _cts.Token); + // } + // + // await Task.Delay(2000); + // + // // Assert: Messages should be evenly distributed across consumers + // var messageCounts = receivedMessages.Values; + // var minReceived = messageCounts.Min(); + // var maxReceived = messageCounts.Max(); + // + // Assert.True(maxReceived - minReceived <= 1, + // "Messages should be evenly distributed among competing consumers."); + // } } } diff --git a/src/OpenDDD.Tests/Integration/Infrastructure/Events/RabbitMq/RabbitMqMessagingProviderTests.cs b/src/OpenDDD.Tests/Integration/Infrastructure/Events/RabbitMq/RabbitMqMessagingProviderTests.cs index e0b9030..eec960d 100644 --- a/src/OpenDDD.Tests/Integration/Infrastructure/Events/RabbitMq/RabbitMqMessagingProviderTests.cs +++ b/src/OpenDDD.Tests/Integration/Infrastructure/Events/RabbitMq/RabbitMqMessagingProviderTests.cs @@ -1,12 +1,12 @@ using System.Collections.Concurrent; using Microsoft.Extensions.Logging; +using Xunit.Abstractions; using Moq; using OpenDDD.Infrastructure.Events.RabbitMq; using OpenDDD.Infrastructure.Events.RabbitMq.Factories; using OpenDDD.Tests.Base; using RabbitMQ.Client; using RabbitMQ.Client.Exceptions; -using Xunit.Abstractions; namespace OpenDDD.Tests.Integration.Infrastructure.Events.RabbitMq { diff --git a/src/OpenDDD/Infrastructure/Events/Kafka/KafkaMessagingProvider.cs b/src/OpenDDD/Infrastructure/Events/Kafka/KafkaMessagingProvider.cs index 4168d69..dbac38b 100644 --- a/src/OpenDDD/Infrastructure/Events/Kafka/KafkaMessagingProvider.cs +++ b/src/OpenDDD/Infrastructure/Events/Kafka/KafkaMessagingProvider.cs @@ -129,7 +129,7 @@ await _adminClient.CreateTopicsAsync(new[] new TopicSpecification { Name = topic, NumPartitions = 1, ReplicationFactor = 1 } }, null); - for (int i = 0; i < 10; i++) + for (int i = 0; i < 30; i++) { await Task.Delay(500, cancellationToken); metadata = _adminClient.GetMetadata(TimeSpan.FromSeconds(1)); From f83657ece040fc635b8433ed3ccc4fe3cc9a6870 Mon Sep 17 00:00:00 2001 From: David Runemalm Date: Tue, 25 Feb 2025 10:05:33 +0100 Subject: [PATCH 061/109] Debug tests in github actions. --- .../Kafka/KafkaMessagingProviderTests.cs | 102 +++++++++--------- .../Events/Kafka/Factories/KafkaConsumer.cs | 1 + 2 files changed, 52 insertions(+), 51 deletions(-) diff --git a/src/OpenDDD.Tests/Integration/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs b/src/OpenDDD.Tests/Integration/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs index 1f80814..0782d03 100644 --- a/src/OpenDDD.Tests/Integration/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs +++ b/src/OpenDDD.Tests/Integration/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs @@ -91,27 +91,27 @@ private async Task CleanupTopicsAndConsumerGroupsAsync() } } - [Fact] - public async Task AutoCreateTopic_ShouldCreateTopicOnSubscribe_WhenSettingEnabled() - { - // Arrange - var topicName = $"test-topic-{Guid.NewGuid()}"; - var consumerGroup = "test-consumer-group"; - - // Ensure topic does not exist - var metadata = _adminClient.GetMetadata(TimeSpan.FromSeconds(5)); - metadata.Topics.Any(t => t.Topic == topicName).Should().BeFalse(); - - // Act - await _messagingProvider.SubscribeAsync(topicName, consumerGroup, async (msg, token) => - await Task.CompletedTask, _cts.Token); - - await Task.Delay(1000); - - // Assert - metadata = _adminClient.GetMetadata(TimeSpan.FromSeconds(5)); - metadata.Topics.Any(t => t.Topic == topicName).Should().BeTrue(); - } + // [Fact] + // public async Task AutoCreateTopic_ShouldCreateTopicOnSubscribe_WhenSettingEnabled() + // { + // // Arrange + // var topicName = $"test-topic-{Guid.NewGuid()}"; + // var consumerGroup = "test-consumer-group"; + // + // // Ensure topic does not exist + // var metadata = _adminClient.GetMetadata(TimeSpan.FromSeconds(5)); + // metadata.Topics.Any(t => t.Topic == topicName).Should().BeFalse(); + // + // // Act + // await _messagingProvider.SubscribeAsync(topicName, consumerGroup, async (msg, token) => + // await Task.CompletedTask, _cts.Token); + // + // await Task.Delay(1000); + // + // // Assert + // metadata = _adminClient.GetMetadata(TimeSpan.FromSeconds(5)); + // metadata.Topics.Any(t => t.Topic == topicName).Should().BeTrue(); + // } // [Fact] // public async Task AutoCreateTopic_ShouldNotCreateTopicOnSubscribe_WhenSettingDisabled() @@ -234,36 +234,36 @@ await _messagingProvider.SubscribeAsync(topicName, consumerGroup, async (msg, to // // Assert: The message should be received multiple times due to reattempts // _receivedMessages.Count.Should().BeGreaterThan(1); // } - // - // [Fact] - // public async Task CompetingConsumers_ShouldDeliverOnlyOnce_WhenMultipleConsumersInGroup() - // { - // // Arrange - // var topicName = $"test-topic-{Guid.NewGuid()}"; - // var consumerGroup = "test-consumer-group"; - // var receivedMessages = new ConcurrentDictionary(); - // var messageToSend = "Competing Consumer Test"; - // - // async Task MessageHandler(string msg, CancellationToken token) - // { - // receivedMessages.AddOrUpdate("received", 1, (key, value) => value + 1); - // } - // - // // Multiple competing consumers in the same group - // await _messagingProvider.SubscribeAsync(topicName, consumerGroup, MessageHandler, _cts.Token); - // await _messagingProvider.SubscribeAsync(topicName, consumerGroup, MessageHandler, _cts.Token); - // await Task.Delay(500); - // - // // Act - // await _messagingProvider.PublishAsync(topicName, messageToSend, _cts.Token); - // - // // Wait for processing - // await Task.Delay(5000); - // - // // Assert: Only one of the competing consumers should receive the message - // receivedMessages.GetValueOrDefault("received", 0).Should().Be(1); - // } - // + + [Fact] + public async Task CompetingConsumers_ShouldDeliverOnlyOnce_WhenMultipleConsumersInGroup() + { + // Arrange + var topicName = $"test-topic-{Guid.NewGuid()}"; + var consumerGroup = "test-consumer-group"; + var receivedMessages = new ConcurrentDictionary(); + var messageToSend = "Competing Consumer Test"; + + async Task MessageHandler(string msg, CancellationToken token) + { + receivedMessages.AddOrUpdate("received", 1, (key, value) => value + 1); + } + + // Multiple competing consumers in the same group + await _messagingProvider.SubscribeAsync(topicName, consumerGroup, MessageHandler, _cts.Token); + await _messagingProvider.SubscribeAsync(topicName, consumerGroup, MessageHandler, _cts.Token); + await Task.Delay(500); + + // Act + await _messagingProvider.PublishAsync(topicName, messageToSend, _cts.Token); + + // Wait for processing + await Task.Delay(5000); + + // Assert: Only one of the competing consumers should receive the message + receivedMessages.GetValueOrDefault("received", 0).Should().Be(1); + } + // [Fact] // public async Task CompetingConsumers_ShouldDistributeEvenly_WhenMultipleConsumersInGroup() // { diff --git a/src/OpenDDD/Infrastructure/Events/Kafka/Factories/KafkaConsumer.cs b/src/OpenDDD/Infrastructure/Events/Kafka/Factories/KafkaConsumer.cs index 82a0885..db0aa59 100644 --- a/src/OpenDDD/Infrastructure/Events/Kafka/Factories/KafkaConsumer.cs +++ b/src/OpenDDD/Infrastructure/Events/Kafka/Factories/KafkaConsumer.cs @@ -79,6 +79,7 @@ private async Task ConsumeLoop(Func messageHand public async Task StopProcessingAsync() { _cts.Cancel(); + if (_consumerTask != null) { await _consumerTask; From 3eaddc1ff5e36df7c758ec1098634c14248f1c10 Mon Sep 17 00:00:00 2001 From: David Runemalm Date: Tue, 25 Feb 2025 10:17:07 +0100 Subject: [PATCH 062/109] Debug tests in github actions. --- .../Events/Kafka/KafkaMessagingProviderTests.cs | 4 +++- .../Infrastructure/Events/Kafka/Factories/KafkaConsumer.cs | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/OpenDDD.Tests/Integration/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs b/src/OpenDDD.Tests/Integration/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs index 0782d03..61f59d3 100644 --- a/src/OpenDDD.Tests/Integration/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs +++ b/src/OpenDDD.Tests/Integration/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs @@ -243,10 +243,12 @@ public async Task CompetingConsumers_ShouldDeliverOnlyOnce_WhenMultipleConsumers var consumerGroup = "test-consumer-group"; var receivedMessages = new ConcurrentDictionary(); var messageToSend = "Competing Consumer Test"; + var subscriberReceived = new TaskCompletionSource(); async Task MessageHandler(string msg, CancellationToken token) { receivedMessages.AddOrUpdate("received", 1, (key, value) => value + 1); + subscriberReceived.SetResult(true); } // Multiple competing consumers in the same group @@ -258,7 +260,7 @@ async Task MessageHandler(string msg, CancellationToken token) await _messagingProvider.PublishAsync(topicName, messageToSend, _cts.Token); // Wait for processing - await Task.Delay(5000); + await subscriberReceived.Task.WaitAsync(TimeSpan.FromSeconds(20)); // Assert: Only one of the competing consumers should receive the message receivedMessages.GetValueOrDefault("received", 0).Should().Be(1); diff --git a/src/OpenDDD/Infrastructure/Events/Kafka/Factories/KafkaConsumer.cs b/src/OpenDDD/Infrastructure/Events/Kafka/Factories/KafkaConsumer.cs index db0aa59..f0236c0 100644 --- a/src/OpenDDD/Infrastructure/Events/Kafka/Factories/KafkaConsumer.cs +++ b/src/OpenDDD/Infrastructure/Events/Kafka/Factories/KafkaConsumer.cs @@ -79,7 +79,7 @@ private async Task ConsumeLoop(Func messageHand public async Task StopProcessingAsync() { _cts.Cancel(); - + if (_consumerTask != null) { await _consumerTask; From 2aaa114a02aa90de45c244c1fc0c93ae0c432c5c Mon Sep 17 00:00:00 2001 From: David Runemalm Date: Tue, 25 Feb 2025 10:27:05 +0100 Subject: [PATCH 063/109] Debug tests in github actions. --- .../Events/Kafka/KafkaMessagingProviderTests.cs | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/OpenDDD.Tests/Integration/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs b/src/OpenDDD.Tests/Integration/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs index 61f59d3..14988ce 100644 --- a/src/OpenDDD.Tests/Integration/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs +++ b/src/OpenDDD.Tests/Integration/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs @@ -19,7 +19,7 @@ public class KafkaMessagingProviderTests : IntegrationTests, IAsyncLifetime private readonly ILogger _logger; private readonly ILogger _consumerLogger; private readonly KafkaMessagingProvider _messagingProvider; - private readonly CancellationTokenSource _cts = new(); + private readonly CancellationTokenSource _cts = new(TimeSpan.FromSeconds(60)); public KafkaMessagingProviderTests(ITestOutputHelper testOutputHelper) : base(testOutputHelper, enableLogging: true) @@ -260,8 +260,15 @@ async Task MessageHandler(string msg, CancellationToken token) await _messagingProvider.PublishAsync(topicName, messageToSend, _cts.Token); // Wait for processing - await subscriberReceived.Task.WaitAsync(TimeSpan.FromSeconds(20)); - + try + { + await subscriberReceived.Task.WaitAsync(TimeSpan.FromSeconds(20)); + } + catch (TimeoutException) + { + _logger.LogDebug("Timed out waiting for subscriber to receive message."); + } + // Assert: Only one of the competing consumers should receive the message receivedMessages.GetValueOrDefault("received", 0).Should().Be(1); } From 4ba1cbb3396deffd2be0b79b2e8dd8d837581405 Mon Sep 17 00:00:00 2001 From: David Runemalm Date: Tue, 25 Feb 2025 10:38:52 +0100 Subject: [PATCH 064/109] Debug tests in github actions. --- .../Kafka/KafkaMessagingProviderTests.cs | 381 ++++++++++-------- 1 file changed, 202 insertions(+), 179 deletions(-) diff --git a/src/OpenDDD.Tests/Integration/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs b/src/OpenDDD.Tests/Integration/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs index 14988ce..4a343d6 100644 --- a/src/OpenDDD.Tests/Integration/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs +++ b/src/OpenDDD.Tests/Integration/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs @@ -91,149 +91,161 @@ private async Task CleanupTopicsAndConsumerGroupsAsync() } } - // [Fact] - // public async Task AutoCreateTopic_ShouldCreateTopicOnSubscribe_WhenSettingEnabled() - // { - // // Arrange - // var topicName = $"test-topic-{Guid.NewGuid()}"; - // var consumerGroup = "test-consumer-group"; - // - // // Ensure topic does not exist - // var metadata = _adminClient.GetMetadata(TimeSpan.FromSeconds(5)); - // metadata.Topics.Any(t => t.Topic == topicName).Should().BeFalse(); - // - // // Act - // await _messagingProvider.SubscribeAsync(topicName, consumerGroup, async (msg, token) => - // await Task.CompletedTask, _cts.Token); - // - // await Task.Delay(1000); - // - // // Assert - // metadata = _adminClient.GetMetadata(TimeSpan.FromSeconds(5)); - // metadata.Topics.Any(t => t.Topic == topicName).Should().BeTrue(); - // } + [Fact] + public async Task AutoCreateTopic_ShouldCreateTopicOnSubscribe_WhenSettingEnabled() + { + // Arrange + var topicName = $"test-topic-{Guid.NewGuid()}"; + var consumerGroup = "test-consumer-group"; + + // Ensure topic does not exist + var metadata = _adminClient.GetMetadata(TimeSpan.FromSeconds(5)); + metadata.Topics.Any(t => t.Topic == topicName).Should().BeFalse(); + + // Act + await _messagingProvider.SubscribeAsync(topicName, consumerGroup, async (msg, token) => + await Task.CompletedTask, _cts.Token); + + var timeout = TimeSpan.FromSeconds(10); + var pollingInterval = TimeSpan.FromMilliseconds(500); + var startTime = DateTime.UtcNow; + + bool topicExists = false; + while (DateTime.UtcNow - startTime < timeout) + { + metadata = _adminClient.GetMetadata(TimeSpan.FromSeconds(5)); + if (metadata.Topics.Any(t => t.Topic == topicName)) + { + topicExists = true; + break; + } + await Task.Delay(pollingInterval); + } + + // Assert + topicExists.Should().BeTrue("Kafka should create the topic automatically when subscribing."); + } - // [Fact] - // public async Task AutoCreateTopic_ShouldNotCreateTopicOnSubscribe_WhenSettingDisabled() - // { - // // Arrange - // var topicName = $"test-topic-{Guid.NewGuid()}"; - // var consumerGroup = "test-consumer-group"; - // - // var messagingProvider = new KafkaMessagingProvider( - // _bootstrapServers, - // _adminClient, - // _producer, - // _consumerFactory, - // autoCreateTopics: false, - // _logger); - // - // // Act & Assert - // var exception = await Assert.ThrowsAsync(async () => - // { - // await messagingProvider.SubscribeAsync(topicName, consumerGroup, async (msg, token) => - // await Task.CompletedTask, _cts.Token); - // }); - // - // exception.Message.Should().Contain($"Topic '{topicName}' does not exist."); - // } - // - // [Fact] - // public async Task AtLeastOnceGurantee_ShouldDeliverToLateSubscriber_WhenSubscribedBefore() - // { - // // Arrange - // var topicName = $"test-topic-{Guid.NewGuid()}"; - // var consumerGroup = $"test-consumer-group-{Guid.NewGuid()}"; - // var messageToSend = "Persistent Message Test"; - // var firstSubscriberReceived = new TaskCompletionSource(); - // var lateSubscriberReceived = new TaskCompletionSource(); - // ConcurrentBag _receivedMessages = new(); - // - // await _messagingProvider.SubscribeAsync(topicName, consumerGroup, async (msg, token) => - // { - // firstSubscriberReceived.SetResult(true); - // }, _cts.Token); - // - // await Task.Delay(500); - // - // await _messagingProvider.PublishAsync(topicName, messageToSend, _cts.Token); - // - // await firstSubscriberReceived.Task; - // - // await _messagingProvider.UnsubscribeAsync(topicName, consumerGroup, _cts.Token); - // await Task.Delay(500); - // - // // Late subscriber - // await _messagingProvider.SubscribeAsync(topicName, consumerGroup, async (msg, token) => - // { - // _receivedMessages.Add(msg); - // lateSubscriberReceived.TrySetResult(true); - // }, _cts.Token); - // - // await _messagingProvider.PublishAsync(topicName, messageToSend, _cts.Token); - // - // await lateSubscriberReceived.Task; - // - // // Assert - // _receivedMessages.Should().Contain(messageToSend); - // } - // - // [Fact] - // public async Task AtLeastOnceGurantee_ShouldNotDeliverToLateSubscriber_WhenNotSubscribedBefore() - // { - // // Arrange - // var topicName = $"test-topic-{Guid.NewGuid()}"; - // var consumerGroup = "test-consumer-group"; - // var messageToSend = "Non-Persistent Message Test"; - // ConcurrentBag _receivedMessages = new(); - // - // // Act: Publish message before any subscription - // await _messagingProvider.PublishAsync(topicName, messageToSend, _cts.Token); - // await Task.Delay(500); - // - // // Late subscriber - // await _messagingProvider.SubscribeAsync(topicName, consumerGroup, async (msg, token) => - // { - // _receivedMessages.Add(msg); - // }, _cts.Token); - // - // await Task.Delay(1000); - // - // // Assert - // _receivedMessages.Should().NotContain(messageToSend); - // } - // - // [Fact] - // public async Task AtLeastOnceGurantee_ShouldRedeliverLater_WhenMessageNotAcked() - // { - // // Arrange - // var topicName = $"test-topic-{Guid.NewGuid()}"; - // var consumerGroup = "test-consumer-group"; - // var messageToSend = "Redelivery Test"; - // ConcurrentBag _receivedMessages = new(); - // - // async Task FaultyHandler(string msg, CancellationToken token) - // { - // _receivedMessages.Add(msg); - // throw new Exception("Simulated consumer crash before acknowledgment."); - // } - // - // await _messagingProvider.SubscribeAsync(topicName, consumerGroup, FaultyHandler, _cts.Token); - // await Task.Delay(500); - // - // // Act: Publish message - // await _messagingProvider.PublishAsync(topicName, messageToSend, _cts.Token); - // - // // Wait for redelivery - // for (int i = 0; i < 300; i++) - // { - // if (_receivedMessages.Count > 1) break; - // await Task.Delay(1000); // Check every second - // } - // - // // Assert: The message should be received multiple times due to reattempts - // _receivedMessages.Count.Should().BeGreaterThan(1); - // } + [Fact] + public async Task AutoCreateTopic_ShouldNotCreateTopicOnSubscribe_WhenSettingDisabled() + { + // Arrange + var topicName = $"test-topic-{Guid.NewGuid()}"; + var consumerGroup = "test-consumer-group"; + + var messagingProvider = new KafkaMessagingProvider( + _bootstrapServers, + _adminClient, + _producer, + _consumerFactory, + autoCreateTopics: false, + _logger); + + // Act & Assert + var exception = await Assert.ThrowsAsync(async () => + { + await messagingProvider.SubscribeAsync(topicName, consumerGroup, async (msg, token) => + await Task.CompletedTask, _cts.Token); + }); + + exception.Message.Should().Contain($"Topic '{topicName}' does not exist."); + } + + [Fact] + public async Task AtLeastOnceGurantee_ShouldDeliverToLateSubscriber_WhenSubscribedBefore() + { + // Arrange + var topicName = $"test-topic-{Guid.NewGuid()}"; + var consumerGroup = $"test-consumer-group-{Guid.NewGuid()}"; + var messageToSend = "Persistent Message Test"; + var firstSubscriberReceived = new TaskCompletionSource(); + var lateSubscriberReceived = new TaskCompletionSource(); + ConcurrentBag _receivedMessages = new(); + + await _messagingProvider.SubscribeAsync(topicName, consumerGroup, async (msg, token) => + { + firstSubscriberReceived.SetResult(true); + }, _cts.Token); + + await Task.Delay(500, _cts.Token); + + await _messagingProvider.PublishAsync(topicName, messageToSend, _cts.Token); + + await firstSubscriberReceived.Task.WaitAsync(TimeSpan.FromSeconds(10)); + + await _messagingProvider.UnsubscribeAsync(topicName, consumerGroup, _cts.Token); + await Task.Delay(500, _cts.Token); + + // Late subscriber + await _messagingProvider.SubscribeAsync(topicName, consumerGroup, async (msg, token) => + { + _receivedMessages.Add(msg); + lateSubscriberReceived.TrySetResult(true); + }, _cts.Token); + + await _messagingProvider.PublishAsync(topicName, messageToSend, _cts.Token); + + await lateSubscriberReceived.Task.WaitAsync(TimeSpan.FromSeconds(10)); + + // Assert + _receivedMessages.Should().Contain(messageToSend); + } + + [Fact] + public async Task AtLeastOnceGurantee_ShouldNotDeliverToLateSubscriber_WhenNotSubscribedBefore() + { + // Arrange + var topicName = $"test-topic-{Guid.NewGuid()}"; + var consumerGroup = "test-consumer-group"; + var messageToSend = "Non-Persistent Message Test"; + ConcurrentBag _receivedMessages = new(); + + // Act + await _messagingProvider.PublishAsync(topicName, messageToSend, _cts.Token); + await Task.Delay(2000); + + // Late subscriber + await _messagingProvider.SubscribeAsync(topicName, consumerGroup, async (msg, token) => + { + _receivedMessages.Add(msg); + }, _cts.Token); + + await Task.Delay(10000); + + // Assert + _receivedMessages.Should().NotContain(messageToSend); + } + + [Fact] + public async Task AtLeastOnceGurantee_ShouldRedeliverLater_WhenMessageNotAcked() + { + // Arrange + var topicName = $"test-topic-{Guid.NewGuid()}"; + var consumerGroup = "test-consumer-group"; + var messageToSend = "Redelivery Test"; + ConcurrentBag _receivedMessages = new(); + + async Task FaultyHandler(string msg, CancellationToken token) + { + _receivedMessages.Add(msg); + throw new Exception("Simulated consumer crash before acknowledgment."); + } + + await _messagingProvider.SubscribeAsync(topicName, consumerGroup, FaultyHandler, _cts.Token); + await Task.Delay(2000); + + // Act + await _messagingProvider.PublishAsync(topicName, messageToSend, _cts.Token); + + for (int i = 0; i < 300; i++) + { + if (_receivedMessages.Count > 1) break; + await Task.Delay(1000); + } + + // Assert + _receivedMessages.Count.Should().BeGreaterThan(1); + } [Fact] public async Task CompetingConsumers_ShouldDeliverOnlyOnce_WhenMultipleConsumersInGroup() @@ -251,7 +263,6 @@ async Task MessageHandler(string msg, CancellationToken token) subscriberReceived.SetResult(true); } - // Multiple competing consumers in the same group await _messagingProvider.SubscribeAsync(topicName, consumerGroup, MessageHandler, _cts.Token); await _messagingProvider.SubscribeAsync(topicName, consumerGroup, MessageHandler, _cts.Token); await Task.Delay(500); @@ -259,7 +270,6 @@ async Task MessageHandler(string msg, CancellationToken token) // Act await _messagingProvider.PublishAsync(topicName, messageToSend, _cts.Token); - // Wait for processing try { await subscriberReceived.Task.WaitAsync(TimeSpan.FromSeconds(20)); @@ -269,43 +279,56 @@ async Task MessageHandler(string msg, CancellationToken token) _logger.LogDebug("Timed out waiting for subscriber to receive message."); } - // Assert: Only one of the competing consumers should receive the message + // Assert receivedMessages.GetValueOrDefault("received", 0).Should().Be(1); } - // [Fact] - // public async Task CompetingConsumers_ShouldDistributeEvenly_WhenMultipleConsumersInGroup() - // { - // // Arrange - // var topicName = $"test-topic-{Guid.NewGuid()}"; - // var consumerGroup = "test-consumer-group"; - // var receivedMessages = new ConcurrentDictionary(); - // var totalMessages = 10; - // - // async Task MessageHandler(string msg, CancellationToken token) - // { - // receivedMessages.AddOrUpdate(msg, 1, (key, value) => value + 1); - // } - // - // await _messagingProvider.SubscribeAsync(topicName, consumerGroup, MessageHandler, _cts.Token); - // await _messagingProvider.SubscribeAsync(topicName, consumerGroup, MessageHandler, _cts.Token); - // await Task.Delay(500); - // - // // Act: Publish multiple messages - // for (int i = 0; i < totalMessages; i++) - // { - // await _messagingProvider.PublishAsync(topicName, $"Message {i}", _cts.Token); - // } - // - // await Task.Delay(2000); - // - // // Assert: Messages should be evenly distributed across consumers - // var messageCounts = receivedMessages.Values; - // var minReceived = messageCounts.Min(); - // var maxReceived = messageCounts.Max(); - // - // Assert.True(maxReceived - minReceived <= 1, - // "Messages should be evenly distributed among competing consumers."); - // } + [Fact] + public async Task CompetingConsumers_ShouldDistributeEvenly_WhenMultipleConsumersInGroup() + { + // Arrange + var topicName = $"test-topic-{Guid.NewGuid()}"; + var consumerGroup = "test-consumer-group"; + var receivedMessages = new ConcurrentDictionary(); + var totalMessages = 10; + var allMessagesReceived = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + + async Task MessageHandler(string msg, CancellationToken token) + { + receivedMessages.AddOrUpdate(msg, 1, (key, value) => value + 1); + + if (receivedMessages.Count >= totalMessages) + { + allMessagesReceived.TrySetResult(true); + } + } + + await _messagingProvider.SubscribeAsync(topicName, consumerGroup, MessageHandler, _cts.Token); + await _messagingProvider.SubscribeAsync(topicName, consumerGroup, MessageHandler, _cts.Token); + await Task.Delay(500); + + // Act + for (int i = 0; i < totalMessages; i++) + { + await _messagingProvider.PublishAsync(topicName, $"Message {i}", _cts.Token); + } + + try + { + await allMessagesReceived.Task.WaitAsync(TimeSpan.FromSeconds(10)); + } + catch (TimeoutException) + { + _logger.LogDebug("Timed out waiting for subscriber to receive all messages."); + } + + // Assert: Messages should be evenly distributed across consumers + var messageCounts = receivedMessages.Values; + var minReceived = messageCounts.Min(); + var maxReceived = messageCounts.Max(); + + Assert.True(maxReceived - minReceived <= 1, + "Messages should be evenly distributed among competing consumers."); + } } } From 7aeaed8c1f725c1ace1d43f3407b118749e74b2b Mon Sep 17 00:00:00 2001 From: David Runemalm Date: Tue, 25 Feb 2025 12:31:18 +0100 Subject: [PATCH 065/109] Debug tests in github actions. --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index c8e07e1..3b864dc 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -65,7 +65,7 @@ jobs: - 2181:2181 kafka: - image: confluentinc/cp-kafka:latest + image: wurstmeister/kafka:latest env: KAFKA_BROKER_ID: 1 KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 From 19a9a9cad98884fbb6f848a46a2cea78072c56ba Mon Sep 17 00:00:00 2001 From: David Runemalm Date: Tue, 25 Feb 2025 12:41:09 +0100 Subject: [PATCH 066/109] Debug tests in github actions. --- .github/workflows/tests.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 3b864dc..741fa93 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -58,9 +58,7 @@ jobs: services: zookeeper: - image: confluentinc/cp-zookeeper:latest - env: - ZOOKEEPER_CLIENT_PORT: 2181 + image: wurstmeister/zookeeper:latest ports: - 2181:2181 @@ -70,6 +68,8 @@ jobs: KAFKA_BROKER_ID: 1 KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:9092 + KAFKA_CREATE_TOPICS: "test-topic:1:1" + KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true" KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 ports: - 9092:9092 From 83e64846fe37c4602f11b5840c9b7a356a9a5577 Mon Sep 17 00:00:00 2001 From: David Runemalm Date: Tue, 25 Feb 2025 19:10:30 +0100 Subject: [PATCH 067/109] Debug tests in github actions. --- .github/workflows/tests.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 741fa93..c8e07e1 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -58,18 +58,18 @@ jobs: services: zookeeper: - image: wurstmeister/zookeeper:latest + image: confluentinc/cp-zookeeper:latest + env: + ZOOKEEPER_CLIENT_PORT: 2181 ports: - 2181:2181 kafka: - image: wurstmeister/kafka:latest + image: confluentinc/cp-kafka:latest env: KAFKA_BROKER_ID: 1 KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:9092 - KAFKA_CREATE_TOPICS: "test-topic:1:1" - KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true" KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 ports: - 9092:9092 From d8d09b1acb3e7ae578ed70fbb4bd20ed82a3297a Mon Sep 17 00:00:00 2001 From: David Runemalm Date: Fri, 28 Feb 2025 08:52:57 +0100 Subject: [PATCH 068/109] Debug tests in github actions. --- .../Events/Kafka/KafkaMessagingProviderTests.cs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/OpenDDD.Tests/Integration/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs b/src/OpenDDD.Tests/Integration/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs index 4a343d6..f302e29 100644 --- a/src/OpenDDD.Tests/Integration/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs +++ b/src/OpenDDD.Tests/Integration/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs @@ -295,6 +295,8 @@ public async Task CompetingConsumers_ShouldDistributeEvenly_WhenMultipleConsumer async Task MessageHandler(string msg, CancellationToken token) { + _logger.LogDebug("Received a message."); + receivedMessages.AddOrUpdate(msg, 1, (key, value) => value + 1); if (receivedMessages.Count >= totalMessages) @@ -315,17 +317,18 @@ async Task MessageHandler(string msg, CancellationToken token) try { - await allMessagesReceived.Task.WaitAsync(TimeSpan.FromSeconds(10)); + await allMessagesReceived.Task.WaitAsync(TimeSpan.FromSeconds(700)); } catch (TimeoutException) { _logger.LogDebug("Timed out waiting for subscriber to receive all messages."); + Assert.Fail($"Consumers only received {receivedMessages.Count} of {totalMessages} messages."); } // Assert: Messages should be evenly distributed across consumers var messageCounts = receivedMessages.Values; - var minReceived = messageCounts.Min(); - var maxReceived = messageCounts.Max(); + var minReceived = messageCounts.Any() ? messageCounts.Min() : 0; + var maxReceived = messageCounts.Any() ? messageCounts.Max() : 0; Assert.True(maxReceived - minReceived <= 1, "Messages should be evenly distributed among competing consumers."); From 79f506b3f9992e03c2bb69792ddb4328d63ab604 Mon Sep 17 00:00:00 2001 From: David Runemalm Date: Fri, 28 Feb 2025 08:53:27 +0100 Subject: [PATCH 069/109] Debug tests in github actions. --- .../Kafka/KafkaMessagingProviderTests.cs | 382 +++++++++--------- 1 file changed, 191 insertions(+), 191 deletions(-) diff --git a/src/OpenDDD.Tests/Integration/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs b/src/OpenDDD.Tests/Integration/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs index f302e29..4cb911a 100644 --- a/src/OpenDDD.Tests/Integration/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs +++ b/src/OpenDDD.Tests/Integration/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs @@ -91,197 +91,197 @@ private async Task CleanupTopicsAndConsumerGroupsAsync() } } - [Fact] - public async Task AutoCreateTopic_ShouldCreateTopicOnSubscribe_WhenSettingEnabled() - { - // Arrange - var topicName = $"test-topic-{Guid.NewGuid()}"; - var consumerGroup = "test-consumer-group"; - - // Ensure topic does not exist - var metadata = _adminClient.GetMetadata(TimeSpan.FromSeconds(5)); - metadata.Topics.Any(t => t.Topic == topicName).Should().BeFalse(); - - // Act - await _messagingProvider.SubscribeAsync(topicName, consumerGroup, async (msg, token) => - await Task.CompletedTask, _cts.Token); - - var timeout = TimeSpan.FromSeconds(10); - var pollingInterval = TimeSpan.FromMilliseconds(500); - var startTime = DateTime.UtcNow; - - bool topicExists = false; - while (DateTime.UtcNow - startTime < timeout) - { - metadata = _adminClient.GetMetadata(TimeSpan.FromSeconds(5)); - if (metadata.Topics.Any(t => t.Topic == topicName)) - { - topicExists = true; - break; - } - await Task.Delay(pollingInterval); - } - - // Assert - topicExists.Should().BeTrue("Kafka should create the topic automatically when subscribing."); - } - - [Fact] - public async Task AutoCreateTopic_ShouldNotCreateTopicOnSubscribe_WhenSettingDisabled() - { - // Arrange - var topicName = $"test-topic-{Guid.NewGuid()}"; - var consumerGroup = "test-consumer-group"; - - var messagingProvider = new KafkaMessagingProvider( - _bootstrapServers, - _adminClient, - _producer, - _consumerFactory, - autoCreateTopics: false, - _logger); - - // Act & Assert - var exception = await Assert.ThrowsAsync(async () => - { - await messagingProvider.SubscribeAsync(topicName, consumerGroup, async (msg, token) => - await Task.CompletedTask, _cts.Token); - }); - - exception.Message.Should().Contain($"Topic '{topicName}' does not exist."); - } - - [Fact] - public async Task AtLeastOnceGurantee_ShouldDeliverToLateSubscriber_WhenSubscribedBefore() - { - // Arrange - var topicName = $"test-topic-{Guid.NewGuid()}"; - var consumerGroup = $"test-consumer-group-{Guid.NewGuid()}"; - var messageToSend = "Persistent Message Test"; - var firstSubscriberReceived = new TaskCompletionSource(); - var lateSubscriberReceived = new TaskCompletionSource(); - ConcurrentBag _receivedMessages = new(); - - await _messagingProvider.SubscribeAsync(topicName, consumerGroup, async (msg, token) => - { - firstSubscriberReceived.SetResult(true); - }, _cts.Token); - - await Task.Delay(500, _cts.Token); - - await _messagingProvider.PublishAsync(topicName, messageToSend, _cts.Token); - - await firstSubscriberReceived.Task.WaitAsync(TimeSpan.FromSeconds(10)); - - await _messagingProvider.UnsubscribeAsync(topicName, consumerGroup, _cts.Token); - await Task.Delay(500, _cts.Token); - - // Late subscriber - await _messagingProvider.SubscribeAsync(topicName, consumerGroup, async (msg, token) => - { - _receivedMessages.Add(msg); - lateSubscriberReceived.TrySetResult(true); - }, _cts.Token); - - await _messagingProvider.PublishAsync(topicName, messageToSend, _cts.Token); - - await lateSubscriberReceived.Task.WaitAsync(TimeSpan.FromSeconds(10)); - - // Assert - _receivedMessages.Should().Contain(messageToSend); - } - - [Fact] - public async Task AtLeastOnceGurantee_ShouldNotDeliverToLateSubscriber_WhenNotSubscribedBefore() - { - // Arrange - var topicName = $"test-topic-{Guid.NewGuid()}"; - var consumerGroup = "test-consumer-group"; - var messageToSend = "Non-Persistent Message Test"; - ConcurrentBag _receivedMessages = new(); - - // Act - await _messagingProvider.PublishAsync(topicName, messageToSend, _cts.Token); - await Task.Delay(2000); - - // Late subscriber - await _messagingProvider.SubscribeAsync(topicName, consumerGroup, async (msg, token) => - { - _receivedMessages.Add(msg); - }, _cts.Token); - - await Task.Delay(10000); - - // Assert - _receivedMessages.Should().NotContain(messageToSend); - } - - [Fact] - public async Task AtLeastOnceGurantee_ShouldRedeliverLater_WhenMessageNotAcked() - { - // Arrange - var topicName = $"test-topic-{Guid.NewGuid()}"; - var consumerGroup = "test-consumer-group"; - var messageToSend = "Redelivery Test"; - ConcurrentBag _receivedMessages = new(); - - async Task FaultyHandler(string msg, CancellationToken token) - { - _receivedMessages.Add(msg); - throw new Exception("Simulated consumer crash before acknowledgment."); - } - - await _messagingProvider.SubscribeAsync(topicName, consumerGroup, FaultyHandler, _cts.Token); - await Task.Delay(2000); - - // Act - await _messagingProvider.PublishAsync(topicName, messageToSend, _cts.Token); - - for (int i = 0; i < 300; i++) - { - if (_receivedMessages.Count > 1) break; - await Task.Delay(1000); - } - - // Assert - _receivedMessages.Count.Should().BeGreaterThan(1); - } - - [Fact] - public async Task CompetingConsumers_ShouldDeliverOnlyOnce_WhenMultipleConsumersInGroup() - { - // Arrange - var topicName = $"test-topic-{Guid.NewGuid()}"; - var consumerGroup = "test-consumer-group"; - var receivedMessages = new ConcurrentDictionary(); - var messageToSend = "Competing Consumer Test"; - var subscriberReceived = new TaskCompletionSource(); - - async Task MessageHandler(string msg, CancellationToken token) - { - receivedMessages.AddOrUpdate("received", 1, (key, value) => value + 1); - subscriberReceived.SetResult(true); - } - - await _messagingProvider.SubscribeAsync(topicName, consumerGroup, MessageHandler, _cts.Token); - await _messagingProvider.SubscribeAsync(topicName, consumerGroup, MessageHandler, _cts.Token); - await Task.Delay(500); - - // Act - await _messagingProvider.PublishAsync(topicName, messageToSend, _cts.Token); - - try - { - await subscriberReceived.Task.WaitAsync(TimeSpan.FromSeconds(20)); - } - catch (TimeoutException) - { - _logger.LogDebug("Timed out waiting for subscriber to receive message."); - } - - // Assert - receivedMessages.GetValueOrDefault("received", 0).Should().Be(1); - } + // [Fact] + // public async Task AutoCreateTopic_ShouldCreateTopicOnSubscribe_WhenSettingEnabled() + // { + // // Arrange + // var topicName = $"test-topic-{Guid.NewGuid()}"; + // var consumerGroup = "test-consumer-group"; + // + // // Ensure topic does not exist + // var metadata = _adminClient.GetMetadata(TimeSpan.FromSeconds(5)); + // metadata.Topics.Any(t => t.Topic == topicName).Should().BeFalse(); + // + // // Act + // await _messagingProvider.SubscribeAsync(topicName, consumerGroup, async (msg, token) => + // await Task.CompletedTask, _cts.Token); + // + // var timeout = TimeSpan.FromSeconds(10); + // var pollingInterval = TimeSpan.FromMilliseconds(500); + // var startTime = DateTime.UtcNow; + // + // bool topicExists = false; + // while (DateTime.UtcNow - startTime < timeout) + // { + // metadata = _adminClient.GetMetadata(TimeSpan.FromSeconds(5)); + // if (metadata.Topics.Any(t => t.Topic == topicName)) + // { + // topicExists = true; + // break; + // } + // await Task.Delay(pollingInterval); + // } + // + // // Assert + // topicExists.Should().BeTrue("Kafka should create the topic automatically when subscribing."); + // } + // + // [Fact] + // public async Task AutoCreateTopic_ShouldNotCreateTopicOnSubscribe_WhenSettingDisabled() + // { + // // Arrange + // var topicName = $"test-topic-{Guid.NewGuid()}"; + // var consumerGroup = "test-consumer-group"; + // + // var messagingProvider = new KafkaMessagingProvider( + // _bootstrapServers, + // _adminClient, + // _producer, + // _consumerFactory, + // autoCreateTopics: false, + // _logger); + // + // // Act & Assert + // var exception = await Assert.ThrowsAsync(async () => + // { + // await messagingProvider.SubscribeAsync(topicName, consumerGroup, async (msg, token) => + // await Task.CompletedTask, _cts.Token); + // }); + // + // exception.Message.Should().Contain($"Topic '{topicName}' does not exist."); + // } + // + // [Fact] + // public async Task AtLeastOnceGurantee_ShouldDeliverToLateSubscriber_WhenSubscribedBefore() + // { + // // Arrange + // var topicName = $"test-topic-{Guid.NewGuid()}"; + // var consumerGroup = $"test-consumer-group-{Guid.NewGuid()}"; + // var messageToSend = "Persistent Message Test"; + // var firstSubscriberReceived = new TaskCompletionSource(); + // var lateSubscriberReceived = new TaskCompletionSource(); + // ConcurrentBag _receivedMessages = new(); + // + // await _messagingProvider.SubscribeAsync(topicName, consumerGroup, async (msg, token) => + // { + // firstSubscriberReceived.SetResult(true); + // }, _cts.Token); + // + // await Task.Delay(500, _cts.Token); + // + // await _messagingProvider.PublishAsync(topicName, messageToSend, _cts.Token); + // + // await firstSubscriberReceived.Task.WaitAsync(TimeSpan.FromSeconds(10)); + // + // await _messagingProvider.UnsubscribeAsync(topicName, consumerGroup, _cts.Token); + // await Task.Delay(500, _cts.Token); + // + // // Late subscriber + // await _messagingProvider.SubscribeAsync(topicName, consumerGroup, async (msg, token) => + // { + // _receivedMessages.Add(msg); + // lateSubscriberReceived.TrySetResult(true); + // }, _cts.Token); + // + // await _messagingProvider.PublishAsync(topicName, messageToSend, _cts.Token); + // + // await lateSubscriberReceived.Task.WaitAsync(TimeSpan.FromSeconds(10)); + // + // // Assert + // _receivedMessages.Should().Contain(messageToSend); + // } + // + // [Fact] + // public async Task AtLeastOnceGurantee_ShouldNotDeliverToLateSubscriber_WhenNotSubscribedBefore() + // { + // // Arrange + // var topicName = $"test-topic-{Guid.NewGuid()}"; + // var consumerGroup = "test-consumer-group"; + // var messageToSend = "Non-Persistent Message Test"; + // ConcurrentBag _receivedMessages = new(); + // + // // Act + // await _messagingProvider.PublishAsync(topicName, messageToSend, _cts.Token); + // await Task.Delay(2000); + // + // // Late subscriber + // await _messagingProvider.SubscribeAsync(topicName, consumerGroup, async (msg, token) => + // { + // _receivedMessages.Add(msg); + // }, _cts.Token); + // + // await Task.Delay(10000); + // + // // Assert + // _receivedMessages.Should().NotContain(messageToSend); + // } + // + // [Fact] + // public async Task AtLeastOnceGurantee_ShouldRedeliverLater_WhenMessageNotAcked() + // { + // // Arrange + // var topicName = $"test-topic-{Guid.NewGuid()}"; + // var consumerGroup = "test-consumer-group"; + // var messageToSend = "Redelivery Test"; + // ConcurrentBag _receivedMessages = new(); + // + // async Task FaultyHandler(string msg, CancellationToken token) + // { + // _receivedMessages.Add(msg); + // throw new Exception("Simulated consumer crash before acknowledgment."); + // } + // + // await _messagingProvider.SubscribeAsync(topicName, consumerGroup, FaultyHandler, _cts.Token); + // await Task.Delay(2000); + // + // // Act + // await _messagingProvider.PublishAsync(topicName, messageToSend, _cts.Token); + // + // for (int i = 0; i < 300; i++) + // { + // if (_receivedMessages.Count > 1) break; + // await Task.Delay(1000); + // } + // + // // Assert + // _receivedMessages.Count.Should().BeGreaterThan(1); + // } + // + // [Fact] + // public async Task CompetingConsumers_ShouldDeliverOnlyOnce_WhenMultipleConsumersInGroup() + // { + // // Arrange + // var topicName = $"test-topic-{Guid.NewGuid()}"; + // var consumerGroup = "test-consumer-group"; + // var receivedMessages = new ConcurrentDictionary(); + // var messageToSend = "Competing Consumer Test"; + // var subscriberReceived = new TaskCompletionSource(); + // + // async Task MessageHandler(string msg, CancellationToken token) + // { + // receivedMessages.AddOrUpdate("received", 1, (key, value) => value + 1); + // subscriberReceived.SetResult(true); + // } + // + // await _messagingProvider.SubscribeAsync(topicName, consumerGroup, MessageHandler, _cts.Token); + // await _messagingProvider.SubscribeAsync(topicName, consumerGroup, MessageHandler, _cts.Token); + // await Task.Delay(500); + // + // // Act + // await _messagingProvider.PublishAsync(topicName, messageToSend, _cts.Token); + // + // try + // { + // await subscriberReceived.Task.WaitAsync(TimeSpan.FromSeconds(20)); + // } + // catch (TimeoutException) + // { + // _logger.LogDebug("Timed out waiting for subscriber to receive message."); + // } + // + // // Assert + // receivedMessages.GetValueOrDefault("received", 0).Should().Be(1); + // } [Fact] public async Task CompetingConsumers_ShouldDistributeEvenly_WhenMultipleConsumersInGroup() From 24677e57c5c04a09801642306ec3e7564e37ef6c Mon Sep 17 00:00:00 2001 From: David Runemalm Date: Fri, 28 Feb 2025 09:37:51 +0100 Subject: [PATCH 070/109] Add postgres repository integration tests. --- .github/workflows/tests.yml | 23 +++ Makefile | 41 +++++ .../Kafka/KafkaMessagingProviderTests.cs | 2 +- .../PostgresOpenDddRepositoryTests.cs | 169 ++++++++++++++++++ .../Postgres/PostgresTestsCollection.cs | 5 + 5 files changed, 239 insertions(+), 1 deletion(-) create mode 100644 src/OpenDDD.Tests/Integration/Infrastructure/Repository/OpenDdd/Postgres/PostgresOpenDddRepositoryTests.cs create mode 100644 src/OpenDDD.Tests/Integration/Infrastructure/Repository/OpenDdd/Postgres/PostgresTestsCollection.cs diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index c8e07e1..bacb3eb 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -85,6 +85,16 @@ jobs: - 15672:15672 options: --health-cmd "rabbitmq-diagnostics check_port_connectivity" --health-interval 10s --health-timeout 5s --health-retries 5 + postgres: + image: postgres:latest + env: + POSTGRES_DB: testdb + POSTGRES_USER: testuser + POSTGRES_PASSWORD: testpassword + ports: + - 5432:5432 + options: --health-cmd "pg_isready -U testuser" --health-interval 10s --health-timeout 5s --health-retries 5 + steps: - name: Checkout Repository uses: actions/checkout@v4 @@ -147,6 +157,18 @@ jobs: done echo "Kafka did not start in time!" && exit 1 + - name: Wait for PostgreSQL to be Ready + run: | + for i in {1..10}; do + if PGPASSWORD=testpassword psql -h localhost -U testuser -d testdb -c "SELECT 1" &> /dev/null; then + echo "PostgreSQL is up!" + exit 0 + fi + echo "Waiting for PostgreSQL..." + sleep 5 + done + echo "PostgreSQL did not start in time!" && exit 1 + - name: Run Integration Tests working-directory: src/OpenDDD.Tests env: @@ -156,6 +178,7 @@ jobs: RABBITMQ_USERNAME: guest RABBITMQ_PASSWORD: guest AZURE_SERVICE_BUS_CONNECTION_STRING: ${{ env.AZURE_SERVICE_BUS_CONNECTION_STRING }} + POSTGRES_TEST_CONNECTION_STRING: "Host=localhost;Port=5432;Database=testdb;Username=testuser;Password=testpassword" run: dotnet test --no-build --configuration Release --filter "Category=Integration" --logger "trx;LogFileName=TestResults.trx" --results-directory TestResults - name: Delete Azure Service Bus namespace After Tests diff --git a/Makefile b/Makefile index 57b3efc..7ccb7ec 100644 --- a/Makefile +++ b/Makefile @@ -426,3 +426,44 @@ ifndef NAME $(error Topic name not specified. Usage: make kafka-produce NAME=) endif @docker exec -it $(KAFKA_CONTAINER) kafka-console-producer.sh --broker-list $(KAFKA_BROKER) --topic $(NAME) + +########################################################################## +# POSTGRES +########################################################################## + +POSTGRES_CONTAINER := opendddnet-testspostgres +POSTGRES_PORT := 5432 +POSTGRES_DB := testdb + +.PHONY: postgres-start +postgres-start: ##@Postgres Start a PostgreSQL container + @docker run -d --name $(POSTGRES_CONTAINER) --network $(NETWORK) \ + -e POSTGRES_DB=$(POSTGRES_DB) \ + -e POSTGRES_USER=$(POSTGRES_USER) \ + -e POSTGRES_PASSWORD=$(POSTGRES_PASSWORD) \ + -p $(POSTGRES_PORT):5432 postgres:latest + @echo "PostgreSQL started on port $(POSTGRES_PORT)." + +.PHONY: postgres-stop +postgres-stop: ##@Postgres Stop the PostgreSQL container + @docker stop $(POSTGRES_CONTAINER) || true + @echo "PostgreSQL stopped." + +.PHONY: postgres-clean +postgres-clean: ##@Postgres Remove PostgreSQL container and its volumes + @docker rm -f $(POSTGRES_CONTAINER) || true + @echo "PostgreSQL container removed." + +.PHONY: postgres-logs +postgres-logs: ##@Postgres Show PostgreSQL logs + @docker logs -f $(POSTGRES_CONTAINER) + +.PHONY: postgres-shell +postgres-shell: ##@Postgres Open a shell inside the PostgreSQL container + docker exec -it $(POSTGRES_CONTAINER) psql -U $(POSTGRES_USER) -d $(POSTGRES_DB) + +.PHONY: postgres-connection-strings +postgres-connection-strings: ##@Postgres Display the connection strings for PostgreSQL + @echo "PostgreSQL Connection String (Key-Value/DSN): Host=localhost;Port=$(POSTGRES_PORT);Database=$(POSTGRES_DB);Username=$(POSTGRES_USER);Password=$(POSTGRES_PASSWORD)" + @echo "PostgreSQL Connection String (URI): postgresql://$(POSTGRES_USER):$(POSTGRES_PASSWORD)@localhost:$(POSTGRES_PORT)/$(POSTGRES_DB)" + @echo "PostgreSQL Connection String (JDBC): jdbc:postgresql://localhost:$(POSTGRES_PORT)/$(POSTGRES_DB)?user=$(POSTGRES_USER)&password=$(POSTGRES_PASSWORD)" diff --git a/src/OpenDDD.Tests/Integration/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs b/src/OpenDDD.Tests/Integration/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs index 4cb911a..4052327 100644 --- a/src/OpenDDD.Tests/Integration/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs +++ b/src/OpenDDD.Tests/Integration/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs @@ -308,7 +308,7 @@ async Task MessageHandler(string msg, CancellationToken token) await _messagingProvider.SubscribeAsync(topicName, consumerGroup, MessageHandler, _cts.Token); await _messagingProvider.SubscribeAsync(topicName, consumerGroup, MessageHandler, _cts.Token); await Task.Delay(500); - + // Act for (int i = 0; i < totalMessages; i++) { diff --git a/src/OpenDDD.Tests/Integration/Infrastructure/Repository/OpenDdd/Postgres/PostgresOpenDddRepositoryTests.cs b/src/OpenDDD.Tests/Integration/Infrastructure/Repository/OpenDdd/Postgres/PostgresOpenDddRepositoryTests.cs new file mode 100644 index 0000000..b0abe08 --- /dev/null +++ b/src/OpenDDD.Tests/Integration/Infrastructure/Repository/OpenDdd/Postgres/PostgresOpenDddRepositoryTests.cs @@ -0,0 +1,169 @@ +using System.Linq.Expressions; +using FluentAssertions; +using Npgsql; +using OpenDDD.API.Extensions; +using OpenDDD.Infrastructure.Persistence.OpenDdd.DatabaseSession.Postgres; +using OpenDDD.Infrastructure.Persistence.OpenDdd.Serializers; +using OpenDDD.Infrastructure.Persistence.Serializers; +using OpenDDD.Infrastructure.Repository.OpenDdd.Postgres; +using OpenDDD.Tests.Domain.Model; + +namespace OpenDDD.Tests.Integration.Infrastructure.Repository.OpenDdd.Postgres +{ + [Collection("PostgresTests")] + public class PostgresOpenDddRepositoryTests : IAsyncLifetime + { + private readonly string _connectionString; + private readonly PostgresDatabaseSession _session; + private readonly IAggregateSerializer _serializer; + private readonly PostgresOpenDddRepository _repository; + private readonly NpgsqlConnection _connection; + private readonly NpgsqlTransaction _transaction; + + public PostgresOpenDddRepositoryTests() + { + _connectionString = Environment.GetEnvironmentVariable("POSTGRES_TEST_CONNECTION_STRING") + ?? "Host=localhost;Port=5432;Database=testdb;Username=testuser;Password=testpassword"; + + _connection = new NpgsqlConnection(_connectionString); + _connection.Open(); + _transaction = _connection.BeginTransaction(); + + _session = new PostgresDatabaseSession(_connection); + _serializer = new OpenDddAggregateSerializer(); + _repository = new PostgresOpenDddRepository(_session, _serializer); + } + + public async Task InitializeAsync() + { + var tableName = typeof(TestAggregateRoot).Name.ToLower().Pluralize(); + + var createTableQuery = $@" + CREATE TABLE {tableName} ( + id UUID PRIMARY KEY, + data JSONB NOT NULL + );"; + + await using var createCmd = new NpgsqlCommand(createTableQuery, _connection, _transaction); + await createCmd.ExecuteNonQueryAsync(); + } + + public async Task DisposeAsync() + { + await _transaction.RollbackAsync(); + await _connection.CloseAsync(); + } + + [Fact] + public async Task SaveAsync_ShouldInsertOrUpdateEntity() + { + // Arrange + var aggregate = TestAggregateRoot.Create( + "Initial Root", + new List { TestEntity.Create("Entity 1"), TestEntity.Create("Entity 2") }, + new TestValueObject(100, "Value Object Data") + ); + + // Act + await _repository.SaveAsync(aggregate, CancellationToken.None); + var retrieved = await _repository.GetAsync(aggregate.Id, CancellationToken.None); + + // Assert + retrieved.Should().NotBeNull(); + retrieved.Id.Should().Be(aggregate.Id); + retrieved.Name.Should().Be("Initial Root"); + retrieved.Entities.Should().HaveCount(2); + retrieved.Value.Number.Should().Be(100); + + // Act (update) + aggregate = TestAggregateRoot.Create( + "Updated Root", + new List { TestEntity.Create("Updated Entity") }, + new TestValueObject(200, "Updated Value") + ); + + await _repository.SaveAsync(aggregate, CancellationToken.None); + var updated = await _repository.GetAsync(aggregate.Id, CancellationToken.None); + + // Assert (update) + updated.Name.Should().Be("Updated Root"); + updated.Entities.Should().HaveCount(1); + updated.Entities.First().Description.Should().Be("Updated Entity"); + updated.Value.Number.Should().Be(200); + } + + [Fact] + public async Task FindAsync_ShouldReturnEntityIfExists() + { + // Arrange + var aggregate = TestAggregateRoot.Create("Find Test", new List(), new TestValueObject(50, "Find Test Value")); + await _repository.SaveAsync(aggregate, CancellationToken.None); + + // Act + var result = await _repository.FindAsync(aggregate.Id, CancellationToken.None); + + // Assert + result.Should().NotBeNull(); + result!.Id.Should().Be(aggregate.Id); + } + + [Fact] + public async Task FindAsync_ShouldReturnNullIfNotExists() + { + // Act + var result = await _repository.FindAsync(Guid.NewGuid(), CancellationToken.None); + + // Assert + result.Should().BeNull(); + } + + [Fact] + public async Task FindWithAsync_ShouldReturnFilteredResults() + { + // Arrange + var aggregate1 = TestAggregateRoot.Create("Filter Match", new List(), new TestValueObject(300, "Match")); + var aggregate2 = TestAggregateRoot.Create("No Match", new List(), new TestValueObject(400, "Different")); + await _repository.SaveAsync(aggregate1, CancellationToken.None); + await _repository.SaveAsync(aggregate2, CancellationToken.None); + + // Act + Expression> filter = a => a.Value.Number == 300; + var results = (await _repository.FindWithAsync(filter, CancellationToken.None)).ToList(); + + // Assert + results.Should().HaveCount(1); + results[0].Name.Should().Be("Filter Match"); + } + + [Fact] + public async Task FindAllAsync_ShouldReturnAllEntities() + { + // Arrange + var aggregate1 = TestAggregateRoot.Create("Entity 1", new List(), new TestValueObject(10, "VO 1")); + var aggregate2 = TestAggregateRoot.Create("Entity 2", new List(), new TestValueObject(20, "VO 2")); + await _repository.SaveAsync(aggregate1, CancellationToken.None); + await _repository.SaveAsync(aggregate2, CancellationToken.None); + + // Act + var results = (await _repository.FindAllAsync(CancellationToken.None)).ToList(); + + // Assert + results.Should().HaveCount(2); + } + + [Fact] + public async Task DeleteAsync_ShouldRemoveEntity() + { + // Arrange + var aggregate = TestAggregateRoot.Create("To be deleted", new List(), new TestValueObject(99, "Delete")); + await _repository.SaveAsync(aggregate, CancellationToken.None); + + // Act + await _repository.DeleteAsync(aggregate, CancellationToken.None); + var result = await _repository.FindAsync(aggregate.Id, CancellationToken.None); + + // Assert + result.Should().BeNull(); + } + } +} diff --git a/src/OpenDDD.Tests/Integration/Infrastructure/Repository/OpenDdd/Postgres/PostgresTestsCollection.cs b/src/OpenDDD.Tests/Integration/Infrastructure/Repository/OpenDdd/Postgres/PostgresTestsCollection.cs new file mode 100644 index 0000000..98ae4fb --- /dev/null +++ b/src/OpenDDD.Tests/Integration/Infrastructure/Repository/OpenDdd/Postgres/PostgresTestsCollection.cs @@ -0,0 +1,5 @@ +namespace OpenDDD.Tests.Integration.Infrastructure.Repository.OpenDdd.Postgres +{ + [CollectionDefinition("PostgresTests", DisableParallelization = true)] + public class PostgresTestsCollection { } +} From 9c55a758de48592949084166998ad05b48f4a113 Mon Sep 17 00:00:00 2001 From: David Runemalm Date: Fri, 28 Feb 2025 11:46:47 +0100 Subject: [PATCH 071/109] Debug tests in github actions. --- .../OpenDdd/Postgres/PostgresOpenDddRepositoryTests.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/OpenDDD.Tests/Integration/Infrastructure/Repository/OpenDdd/Postgres/PostgresOpenDddRepositoryTests.cs b/src/OpenDDD.Tests/Integration/Infrastructure/Repository/OpenDdd/Postgres/PostgresOpenDddRepositoryTests.cs index b0abe08..1987e0b 100644 --- a/src/OpenDDD.Tests/Integration/Infrastructure/Repository/OpenDdd/Postgres/PostgresOpenDddRepositoryTests.cs +++ b/src/OpenDDD.Tests/Integration/Infrastructure/Repository/OpenDdd/Postgres/PostgresOpenDddRepositoryTests.cs @@ -6,12 +6,13 @@ using OpenDDD.Infrastructure.Persistence.OpenDdd.Serializers; using OpenDDD.Infrastructure.Persistence.Serializers; using OpenDDD.Infrastructure.Repository.OpenDdd.Postgres; +using OpenDDD.Tests.Base; using OpenDDD.Tests.Domain.Model; namespace OpenDDD.Tests.Integration.Infrastructure.Repository.OpenDdd.Postgres { [Collection("PostgresTests")] - public class PostgresOpenDddRepositoryTests : IAsyncLifetime + public class PostgresOpenDddRepositoryTests : IntegrationTests, IAsyncLifetime { private readonly string _connectionString; private readonly PostgresDatabaseSession _session; From 98028f0547c5eb3032a893653f40f33bb33f9313 Mon Sep 17 00:00:00 2001 From: David Runemalm Date: Fri, 28 Feb 2025 11:47:16 +0100 Subject: [PATCH 072/109] Debug tests in github actions. --- .../OpenDdd/Postgres/PostgresOpenDddRepositoryTests.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/OpenDDD.Tests/Integration/Infrastructure/Repository/OpenDdd/Postgres/PostgresOpenDddRepositoryTests.cs b/src/OpenDDD.Tests/Integration/Infrastructure/Repository/OpenDdd/Postgres/PostgresOpenDddRepositoryTests.cs index 1987e0b..8242ece 100644 --- a/src/OpenDDD.Tests/Integration/Infrastructure/Repository/OpenDdd/Postgres/PostgresOpenDddRepositoryTests.cs +++ b/src/OpenDDD.Tests/Integration/Infrastructure/Repository/OpenDdd/Postgres/PostgresOpenDddRepositoryTests.cs @@ -1,6 +1,7 @@ using System.Linq.Expressions; using FluentAssertions; using Npgsql; +using Xunit.Abstractions; using OpenDDD.API.Extensions; using OpenDDD.Infrastructure.Persistence.OpenDdd.DatabaseSession.Postgres; using OpenDDD.Infrastructure.Persistence.OpenDdd.Serializers; @@ -21,7 +22,8 @@ public class PostgresOpenDddRepositoryTests : IntegrationTests, IAsyncLifetime private readonly NpgsqlConnection _connection; private readonly NpgsqlTransaction _transaction; - public PostgresOpenDddRepositoryTests() + public PostgresOpenDddRepositoryTests(ITestOutputHelper testOutputHelper) + : base(testOutputHelper, enableLogging: true) { _connectionString = Environment.GetEnvironmentVariable("POSTGRES_TEST_CONNECTION_STRING") ?? "Host=localhost;Port=5432;Database=testdb;Username=testuser;Password=testpassword"; From 3b3ea7d265edabb25c0147d7e7dea0dd6e9ccbe1 Mon Sep 17 00:00:00 2001 From: David Runemalm Date: Fri, 28 Feb 2025 12:01:18 +0100 Subject: [PATCH 073/109] Add in-memory repository integration tests. --- .../Kafka/KafkaMessagingProviderTests.cs | 2 +- .../InMemoryOpenDddRepositoryTests.cs | 158 ++++++++++++++++++ .../InMemory/InMemoryTestsCollection.cs | 5 + .../Persistence/Storage/IKeyValueStorage.cs | 1 + .../InMemory/InMemoryKeyValueStorage.cs | 7 + 5 files changed, 172 insertions(+), 1 deletion(-) create mode 100644 src/OpenDDD.Tests/Integration/Infrastructure/Repository/OpenDdd/InMemory/InMemoryOpenDddRepositoryTests.cs create mode 100644 src/OpenDDD.Tests/Integration/Infrastructure/Repository/OpenDdd/InMemory/InMemoryTestsCollection.cs diff --git a/src/OpenDDD.Tests/Integration/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs b/src/OpenDDD.Tests/Integration/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs index 4052327..0b7ce7c 100644 --- a/src/OpenDDD.Tests/Integration/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs +++ b/src/OpenDDD.Tests/Integration/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs @@ -317,7 +317,7 @@ async Task MessageHandler(string msg, CancellationToken token) try { - await allMessagesReceived.Task.WaitAsync(TimeSpan.FromSeconds(700)); + await allMessagesReceived.Task.WaitAsync(TimeSpan.FromSeconds(10)); } catch (TimeoutException) { diff --git a/src/OpenDDD.Tests/Integration/Infrastructure/Repository/OpenDdd/InMemory/InMemoryOpenDddRepositoryTests.cs b/src/OpenDDD.Tests/Integration/Infrastructure/Repository/OpenDdd/InMemory/InMemoryOpenDddRepositoryTests.cs new file mode 100644 index 0000000..181e0aa --- /dev/null +++ b/src/OpenDDD.Tests/Integration/Infrastructure/Repository/OpenDdd/InMemory/InMemoryOpenDddRepositoryTests.cs @@ -0,0 +1,158 @@ +using System.Linq.Expressions; +using Microsoft.Extensions.Logging; +using FluentAssertions; +using Xunit.Abstractions; +using OpenDDD.Infrastructure.Persistence.OpenDdd.DatabaseSession.InMemory; +using OpenDDD.Infrastructure.Persistence.OpenDdd.Serializers; +using OpenDDD.Infrastructure.Persistence.Serializers; +using OpenDDD.Infrastructure.Persistence.Storage.InMemory; +using OpenDDD.Infrastructure.Repository.OpenDdd.InMemory; +using OpenDDD.Tests.Base; +using OpenDDD.Tests.Domain.Model; + +namespace OpenDDD.Tests.Integration.Infrastructure.Repository.OpenDdd.InMemory +{ + [Collection("InMemoryTests")] + public class InMemoryOpenDddRepositoryTests : IntegrationTests, IAsyncLifetime + { + private readonly ILogger _storageLogger; + private readonly ILogger _sessionLogger; + private readonly InMemoryKeyValueStorage _storage; + private readonly InMemoryDatabaseSession _session; + private readonly IAggregateSerializer _serializer; + private readonly InMemoryOpenDddRepository _repository; + + public InMemoryOpenDddRepositoryTests(ITestOutputHelper testOutputHelper) + : base(testOutputHelper, enableLogging: true) + { + _storageLogger = LoggerFactory.CreateLogger(); + _sessionLogger = LoggerFactory.CreateLogger(); + _storage = new InMemoryKeyValueStorage(_storageLogger); + _session = new InMemoryDatabaseSession(_storage, _sessionLogger); + _serializer = new OpenDddAggregateSerializer(); + _repository = new InMemoryOpenDddRepository(_session, _serializer); + } + + public async Task InitializeAsync() + { + await _storage.ClearAsync(CancellationToken.None); + } + + public Task DisposeAsync() + { + return Task.CompletedTask; + } + + [Fact] + public async Task SaveAsync_ShouldInsertOrUpdateEntity() + { + // Arrange + var aggregate = TestAggregateRoot.Create( + "Initial Root", + new List { TestEntity.Create("Entity 1"), TestEntity.Create("Entity 2") }, + new TestValueObject(100, "Value Object Data") + ); + + // Act + await _repository.SaveAsync(aggregate, CancellationToken.None); + var retrieved = await _repository.GetAsync(aggregate.Id, CancellationToken.None); + + // Assert + retrieved.Should().NotBeNull(); + retrieved.Id.Should().Be(aggregate.Id); + retrieved.Name.Should().Be("Initial Root"); + retrieved.Entities.Should().HaveCount(2); + retrieved.Value.Number.Should().Be(100); + + // Act (update) + aggregate = TestAggregateRoot.Create( + "Updated Root", + new List { TestEntity.Create("Updated Entity") }, + new TestValueObject(200, "Updated Value") + ); + + await _repository.SaveAsync(aggregate, CancellationToken.None); + var updated = await _repository.GetAsync(aggregate.Id, CancellationToken.None); + + // Assert (update) + updated.Name.Should().Be("Updated Root"); + updated.Entities.Should().HaveCount(1); + updated.Entities.First().Description.Should().Be("Updated Entity"); + updated.Value.Number.Should().Be(200); + } + + [Fact] + public async Task FindAsync_ShouldReturnEntityIfExists() + { + // Arrange + var aggregate = TestAggregateRoot.Create("Find Test", new List(), new TestValueObject(50, "Find Test Value")); + await _repository.SaveAsync(aggregate, CancellationToken.None); + + // Act + var result = await _repository.FindAsync(aggregate.Id, CancellationToken.None); + + // Assert + result.Should().NotBeNull(); + result!.Id.Should().Be(aggregate.Id); + } + + [Fact] + public async Task FindAsync_ShouldReturnNullIfNotExists() + { + // Act + var result = await _repository.FindAsync(Guid.NewGuid(), CancellationToken.None); + + // Assert + result.Should().BeNull(); + } + + [Fact] + public async Task FindWithAsync_ShouldReturnFilteredResults() + { + // Arrange + var aggregate1 = TestAggregateRoot.Create("Filter Match", new List(), new TestValueObject(300, "Match")); + var aggregate2 = TestAggregateRoot.Create("No Match", new List(), new TestValueObject(400, "Different")); + await _repository.SaveAsync(aggregate1, CancellationToken.None); + await _repository.SaveAsync(aggregate2, CancellationToken.None); + + // Act + Expression> filter = a => a.Value.Number == 300; + var results = (await _repository.FindWithAsync(filter, CancellationToken.None)).ToList(); + + // Assert + results.Should().HaveCount(1); + results[0].Name.Should().Be("Filter Match"); + } + + [Fact] + public async Task FindAllAsync_ShouldReturnAllEntities() + { + // Arrange + var aggregate1 = TestAggregateRoot.Create("Entity 1", new List(), new TestValueObject(10, "VO 1")); + var aggregate2 = TestAggregateRoot.Create("Entity 2", new List(), new TestValueObject(20, "VO 2")); + await _repository.SaveAsync(aggregate1, CancellationToken.None); + await _repository.SaveAsync(aggregate2, CancellationToken.None); + + // Act + var results = (await _repository.FindAllAsync(CancellationToken.None)).ToList(); + + // Assert + results.Should().HaveCount(2); + } + + [Fact] + public async Task DeleteAsync_ShouldRemoveEntity() + { + // Arrange + var aggregate = TestAggregateRoot.Create("To be deleted", new List(), new TestValueObject(99, "Delete")); + await _repository.SaveAsync(aggregate, CancellationToken.None); + + // Act + await _repository.DeleteAsync(aggregate, CancellationToken.None); + var result = await _repository.FindAsync(aggregate.Id, CancellationToken.None); + + // Assert + result.Should().BeNull(); + } + } +} diff --git a/src/OpenDDD.Tests/Integration/Infrastructure/Repository/OpenDdd/InMemory/InMemoryTestsCollection.cs b/src/OpenDDD.Tests/Integration/Infrastructure/Repository/OpenDdd/InMemory/InMemoryTestsCollection.cs new file mode 100644 index 0000000..0edce68 --- /dev/null +++ b/src/OpenDDD.Tests/Integration/Infrastructure/Repository/OpenDdd/InMemory/InMemoryTestsCollection.cs @@ -0,0 +1,5 @@ +namespace OpenDDD.Tests.Integration.Infrastructure.Repository.OpenDdd.InMemory +{ + [CollectionDefinition("InMemoryTests", DisableParallelization = true)] + public class InMemoryTestsCollection { } +} diff --git a/src/OpenDDD/Infrastructure/Persistence/Storage/IKeyValueStorage.cs b/src/OpenDDD/Infrastructure/Persistence/Storage/IKeyValueStorage.cs index 452e9a5..f32477b 100644 --- a/src/OpenDDD/Infrastructure/Persistence/Storage/IKeyValueStorage.cs +++ b/src/OpenDDD/Infrastructure/Persistence/Storage/IKeyValueStorage.cs @@ -6,5 +6,6 @@ public interface IKeyValueStorage Task GetAsync(string key, CancellationToken ct); Task> GetByPrefixAsync(string keyPrefix, CancellationToken ct); Task RemoveAsync(string key, CancellationToken ct); + Task ClearAsync(CancellationToken ct); } } diff --git a/src/OpenDDD/Infrastructure/Persistence/Storage/InMemory/InMemoryKeyValueStorage.cs b/src/OpenDDD/Infrastructure/Persistence/Storage/InMemory/InMemoryKeyValueStorage.cs index ad0a729..c0f120a 100644 --- a/src/OpenDDD/Infrastructure/Persistence/Storage/InMemory/InMemoryKeyValueStorage.cs +++ b/src/OpenDDD/Infrastructure/Persistence/Storage/InMemory/InMemoryKeyValueStorage.cs @@ -43,5 +43,12 @@ public Task RemoveAsync(string key, CancellationToken ct) _logger.LogDebug("Removed value with key '{Key}'", key); return Task.CompletedTask; } + + public Task ClearAsync(CancellationToken ct = default) + { + _storage.Clear(); + _logger.LogDebug("Cleared all in-memory storage."); + return Task.CompletedTask; + } } } From 3c562bb6c4a212d7514e8c72ba9aaa4396105c55 Mon Sep 17 00:00:00 2001 From: David Runemalm Date: Fri, 28 Feb 2025 13:55:02 +0100 Subject: [PATCH 074/109] Debug tests in github actions. --- .../Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/OpenDDD.Tests/Integration/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs b/src/OpenDDD.Tests/Integration/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs index 0b7ce7c..8c65cf3 100644 --- a/src/OpenDDD.Tests/Integration/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs +++ b/src/OpenDDD.Tests/Integration/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs @@ -307,7 +307,7 @@ async Task MessageHandler(string msg, CancellationToken token) await _messagingProvider.SubscribeAsync(topicName, consumerGroup, MessageHandler, _cts.Token); await _messagingProvider.SubscribeAsync(topicName, consumerGroup, MessageHandler, _cts.Token); - await Task.Delay(500); + await Task.Delay(20000); // Act for (int i = 0; i < totalMessages; i++) From 9449f1386a3a4e1690fc1be024e974ef6649a3ab Mon Sep 17 00:00:00 2001 From: David Runemalm Date: Fri, 28 Feb 2025 14:38:48 +0100 Subject: [PATCH 075/109] Make kafka integration test wait for group stabilization before publishing. --- .../Kafka/KafkaMessagingProviderTests.cs | 25 ++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/src/OpenDDD.Tests/Integration/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs b/src/OpenDDD.Tests/Integration/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs index 8c65cf3..d8ee088 100644 --- a/src/OpenDDD.Tests/Integration/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs +++ b/src/OpenDDD.Tests/Integration/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs @@ -307,7 +307,9 @@ async Task MessageHandler(string msg, CancellationToken token) await _messagingProvider.SubscribeAsync(topicName, consumerGroup, MessageHandler, _cts.Token); await _messagingProvider.SubscribeAsync(topicName, consumerGroup, MessageHandler, _cts.Token); - await Task.Delay(20000); + + // await Task.Delay(1000); + await WaitForKafkaConsumerGroupStable(consumerGroup, _cts.Token); // Act for (int i = 0; i < totalMessages; i++) @@ -333,5 +335,26 @@ async Task MessageHandler(string msg, CancellationToken token) Assert.True(maxReceived - minReceived <= 1, "Messages should be evenly distributed among competing consumers."); } + + private async Task WaitForKafkaConsumerGroupStable(string consumerGroup, CancellationToken cancellationToken) + { + var maxAttempts = 10; + for (int i = 0; i < maxAttempts; i++) + { + var groupInfo = _adminClient.ListGroups(TimeSpan.FromSeconds(5)) + .FirstOrDefault(g => g.Group == consumerGroup); + + if (groupInfo?.State == "Stable") + { + _logger.LogDebug("Consumer group {ConsumerGroup} is now stable.", consumerGroup); + return; + } + + _logger.LogDebug("Waiting for consumer group {ConsumerGroup} to stabilize...", consumerGroup); + await Task.Delay(500, cancellationToken); + } + + throw new TimeoutException($"Consumer group {consumerGroup} did not stabilize in time."); + } } } From 392a37e067dd017d76edac5f66d779ffcfd64742 Mon Sep 17 00:00:00 2001 From: David Runemalm Date: Fri, 28 Feb 2025 14:46:20 +0100 Subject: [PATCH 076/109] Wait for group stabilization in other kafka tests as well. --- .../Kafka/KafkaMessagingProviderTests.cs | 386 +++++++++--------- 1 file changed, 193 insertions(+), 193 deletions(-) diff --git a/src/OpenDDD.Tests/Integration/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs b/src/OpenDDD.Tests/Integration/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs index d8ee088..38f5d79 100644 --- a/src/OpenDDD.Tests/Integration/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs +++ b/src/OpenDDD.Tests/Integration/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs @@ -91,197 +91,198 @@ private async Task CleanupTopicsAndConsumerGroupsAsync() } } - // [Fact] - // public async Task AutoCreateTopic_ShouldCreateTopicOnSubscribe_WhenSettingEnabled() - // { - // // Arrange - // var topicName = $"test-topic-{Guid.NewGuid()}"; - // var consumerGroup = "test-consumer-group"; - // - // // Ensure topic does not exist - // var metadata = _adminClient.GetMetadata(TimeSpan.FromSeconds(5)); - // metadata.Topics.Any(t => t.Topic == topicName).Should().BeFalse(); - // - // // Act - // await _messagingProvider.SubscribeAsync(topicName, consumerGroup, async (msg, token) => - // await Task.CompletedTask, _cts.Token); - // - // var timeout = TimeSpan.FromSeconds(10); - // var pollingInterval = TimeSpan.FromMilliseconds(500); - // var startTime = DateTime.UtcNow; - // - // bool topicExists = false; - // while (DateTime.UtcNow - startTime < timeout) - // { - // metadata = _adminClient.GetMetadata(TimeSpan.FromSeconds(5)); - // if (metadata.Topics.Any(t => t.Topic == topicName)) - // { - // topicExists = true; - // break; - // } - // await Task.Delay(pollingInterval); - // } - // - // // Assert - // topicExists.Should().BeTrue("Kafka should create the topic automatically when subscribing."); - // } - // - // [Fact] - // public async Task AutoCreateTopic_ShouldNotCreateTopicOnSubscribe_WhenSettingDisabled() - // { - // // Arrange - // var topicName = $"test-topic-{Guid.NewGuid()}"; - // var consumerGroup = "test-consumer-group"; - // - // var messagingProvider = new KafkaMessagingProvider( - // _bootstrapServers, - // _adminClient, - // _producer, - // _consumerFactory, - // autoCreateTopics: false, - // _logger); - // - // // Act & Assert - // var exception = await Assert.ThrowsAsync(async () => - // { - // await messagingProvider.SubscribeAsync(topicName, consumerGroup, async (msg, token) => - // await Task.CompletedTask, _cts.Token); - // }); - // - // exception.Message.Should().Contain($"Topic '{topicName}' does not exist."); - // } - // - // [Fact] - // public async Task AtLeastOnceGurantee_ShouldDeliverToLateSubscriber_WhenSubscribedBefore() - // { - // // Arrange - // var topicName = $"test-topic-{Guid.NewGuid()}"; - // var consumerGroup = $"test-consumer-group-{Guid.NewGuid()}"; - // var messageToSend = "Persistent Message Test"; - // var firstSubscriberReceived = new TaskCompletionSource(); - // var lateSubscriberReceived = new TaskCompletionSource(); - // ConcurrentBag _receivedMessages = new(); - // - // await _messagingProvider.SubscribeAsync(topicName, consumerGroup, async (msg, token) => - // { - // firstSubscriberReceived.SetResult(true); - // }, _cts.Token); - // - // await Task.Delay(500, _cts.Token); - // - // await _messagingProvider.PublishAsync(topicName, messageToSend, _cts.Token); - // - // await firstSubscriberReceived.Task.WaitAsync(TimeSpan.FromSeconds(10)); - // - // await _messagingProvider.UnsubscribeAsync(topicName, consumerGroup, _cts.Token); - // await Task.Delay(500, _cts.Token); - // - // // Late subscriber - // await _messagingProvider.SubscribeAsync(topicName, consumerGroup, async (msg, token) => - // { - // _receivedMessages.Add(msg); - // lateSubscriberReceived.TrySetResult(true); - // }, _cts.Token); - // - // await _messagingProvider.PublishAsync(topicName, messageToSend, _cts.Token); - // - // await lateSubscriberReceived.Task.WaitAsync(TimeSpan.FromSeconds(10)); - // - // // Assert - // _receivedMessages.Should().Contain(messageToSend); - // } - // - // [Fact] - // public async Task AtLeastOnceGurantee_ShouldNotDeliverToLateSubscriber_WhenNotSubscribedBefore() - // { - // // Arrange - // var topicName = $"test-topic-{Guid.NewGuid()}"; - // var consumerGroup = "test-consumer-group"; - // var messageToSend = "Non-Persistent Message Test"; - // ConcurrentBag _receivedMessages = new(); - // - // // Act - // await _messagingProvider.PublishAsync(topicName, messageToSend, _cts.Token); - // await Task.Delay(2000); - // - // // Late subscriber - // await _messagingProvider.SubscribeAsync(topicName, consumerGroup, async (msg, token) => - // { - // _receivedMessages.Add(msg); - // }, _cts.Token); - // - // await Task.Delay(10000); - // - // // Assert - // _receivedMessages.Should().NotContain(messageToSend); - // } - // - // [Fact] - // public async Task AtLeastOnceGurantee_ShouldRedeliverLater_WhenMessageNotAcked() - // { - // // Arrange - // var topicName = $"test-topic-{Guid.NewGuid()}"; - // var consumerGroup = "test-consumer-group"; - // var messageToSend = "Redelivery Test"; - // ConcurrentBag _receivedMessages = new(); - // - // async Task FaultyHandler(string msg, CancellationToken token) - // { - // _receivedMessages.Add(msg); - // throw new Exception("Simulated consumer crash before acknowledgment."); - // } - // - // await _messagingProvider.SubscribeAsync(topicName, consumerGroup, FaultyHandler, _cts.Token); - // await Task.Delay(2000); - // - // // Act - // await _messagingProvider.PublishAsync(topicName, messageToSend, _cts.Token); - // - // for (int i = 0; i < 300; i++) - // { - // if (_receivedMessages.Count > 1) break; - // await Task.Delay(1000); - // } - // - // // Assert - // _receivedMessages.Count.Should().BeGreaterThan(1); - // } - // - // [Fact] - // public async Task CompetingConsumers_ShouldDeliverOnlyOnce_WhenMultipleConsumersInGroup() - // { - // // Arrange - // var topicName = $"test-topic-{Guid.NewGuid()}"; - // var consumerGroup = "test-consumer-group"; - // var receivedMessages = new ConcurrentDictionary(); - // var messageToSend = "Competing Consumer Test"; - // var subscriberReceived = new TaskCompletionSource(); - // - // async Task MessageHandler(string msg, CancellationToken token) - // { - // receivedMessages.AddOrUpdate("received", 1, (key, value) => value + 1); - // subscriberReceived.SetResult(true); - // } - // - // await _messagingProvider.SubscribeAsync(topicName, consumerGroup, MessageHandler, _cts.Token); - // await _messagingProvider.SubscribeAsync(topicName, consumerGroup, MessageHandler, _cts.Token); - // await Task.Delay(500); - // - // // Act - // await _messagingProvider.PublishAsync(topicName, messageToSend, _cts.Token); - // - // try - // { - // await subscriberReceived.Task.WaitAsync(TimeSpan.FromSeconds(20)); - // } - // catch (TimeoutException) - // { - // _logger.LogDebug("Timed out waiting for subscriber to receive message."); - // } - // - // // Assert - // receivedMessages.GetValueOrDefault("received", 0).Should().Be(1); - // } + [Fact] + public async Task AutoCreateTopic_ShouldCreateTopicOnSubscribe_WhenSettingEnabled() + { + // Arrange + var topicName = $"test-topic-{Guid.NewGuid()}"; + var consumerGroup = "test-consumer-group"; + + // Ensure topic does not exist + var metadata = _adminClient.GetMetadata(TimeSpan.FromSeconds(5)); + metadata.Topics.Any(t => t.Topic == topicName).Should().BeFalse(); + + // Act + await _messagingProvider.SubscribeAsync(topicName, consumerGroup, async (msg, token) => + await Task.CompletedTask, _cts.Token); + + var timeout = TimeSpan.FromSeconds(10); + var pollingInterval = TimeSpan.FromMilliseconds(500); + var startTime = DateTime.UtcNow; + + bool topicExists = false; + while (DateTime.UtcNow - startTime < timeout) + { + metadata = _adminClient.GetMetadata(TimeSpan.FromSeconds(5)); + if (metadata.Topics.Any(t => t.Topic == topicName)) + { + topicExists = true; + break; + } + await Task.Delay(pollingInterval); + } + + // Assert + topicExists.Should().BeTrue("Kafka should create the topic automatically when subscribing."); + } + + [Fact] + public async Task AutoCreateTopic_ShouldNotCreateTopicOnSubscribe_WhenSettingDisabled() + { + // Arrange + var topicName = $"test-topic-{Guid.NewGuid()}"; + var consumerGroup = "test-consumer-group"; + + var messagingProvider = new KafkaMessagingProvider( + _bootstrapServers, + _adminClient, + _producer, + _consumerFactory, + autoCreateTopics: false, + _logger); + + // Act & Assert + var exception = await Assert.ThrowsAsync(async () => + { + await messagingProvider.SubscribeAsync(topicName, consumerGroup, async (msg, token) => + await Task.CompletedTask, _cts.Token); + }); + + exception.Message.Should().Contain($"Topic '{topicName}' does not exist."); + } + + [Fact] + public async Task AtLeastOnceGurantee_ShouldDeliverToLateSubscriber_WhenSubscribedBefore() + { + // Arrange + var topicName = $"test-topic-{Guid.NewGuid()}"; + var consumerGroup = $"test-consumer-group-{Guid.NewGuid()}"; + var messageToSend = "Persistent Message Test"; + var firstSubscriberReceived = new TaskCompletionSource(); + var lateSubscriberReceived = new TaskCompletionSource(); + ConcurrentBag _receivedMessages = new(); + + await _messagingProvider.SubscribeAsync(topicName, consumerGroup, async (msg, token) => + { + firstSubscriberReceived.SetResult(true); + }, _cts.Token); + await WaitForKafkaConsumerGroupStable(consumerGroup, _cts.Token); + + await _messagingProvider.PublishAsync(topicName, messageToSend, _cts.Token); + await firstSubscriberReceived.Task.WaitAsync(TimeSpan.FromSeconds(10)); + + await _messagingProvider.UnsubscribeAsync(topicName, consumerGroup, _cts.Token); + + // Late subscriber + await _messagingProvider.SubscribeAsync(topicName, consumerGroup, async (msg, token) => + { + _receivedMessages.Add(msg); + lateSubscriberReceived.TrySetResult(true); + }, _cts.Token); + + await _messagingProvider.PublishAsync(topicName, messageToSend, _cts.Token); + + await lateSubscriberReceived.Task.WaitAsync(TimeSpan.FromSeconds(10)); + + // Assert + _receivedMessages.Should().Contain(messageToSend); + } + + [Fact] + public async Task AtLeastOnceGurantee_ShouldNotDeliverToLateSubscriber_WhenNotSubscribedBefore() + { + // Arrange + var topicName = $"test-topic-{Guid.NewGuid()}"; + var consumerGroup = "test-consumer-group"; + var messageToSend = "Non-Persistent Message Test"; + ConcurrentBag _receivedMessages = new(); + + // Act + await _messagingProvider.PublishAsync(topicName, messageToSend, _cts.Token); + await Task.Delay(2000); + + // Late subscriber + await _messagingProvider.SubscribeAsync(topicName, consumerGroup, async (msg, token) => + { + _receivedMessages.Add(msg); + }, _cts.Token); + + await WaitForKafkaConsumerGroupStable(consumerGroup, _cts.Token); + + await Task.Delay(1000); + + // Assert + _receivedMessages.Should().NotContain(messageToSend); + } + + [Fact] + public async Task AtLeastOnceGurantee_ShouldRedeliverLater_WhenMessageNotAcked() + { + // Arrange + var topicName = $"test-topic-{Guid.NewGuid()}"; + var consumerGroup = "test-consumer-group"; + var messageToSend = "Redelivery Test"; + ConcurrentBag _receivedMessages = new(); + + async Task FaultyHandler(string msg, CancellationToken token) + { + _receivedMessages.Add(msg); + throw new Exception("Simulated consumer crash before acknowledgment."); + } + + await _messagingProvider.SubscribeAsync(topicName, consumerGroup, FaultyHandler, _cts.Token); + + await WaitForKafkaConsumerGroupStable(consumerGroup, _cts.Token); + + // Act + await _messagingProvider.PublishAsync(topicName, messageToSend, _cts.Token); + + for (int i = 0; i < 300; i++) + { + if (_receivedMessages.Count > 1) break; + await Task.Delay(1000); + } + + // Assert + _receivedMessages.Count.Should().BeGreaterThan(1); + } + + [Fact] + public async Task CompetingConsumers_ShouldDeliverOnlyOnce_WhenMultipleConsumersInGroup() + { + // Arrange + var topicName = $"test-topic-{Guid.NewGuid()}"; + var consumerGroup = "test-consumer-group"; + var receivedMessages = new ConcurrentDictionary(); + var messageToSend = "Competing Consumer Test"; + var subscriberReceived = new TaskCompletionSource(); + + async Task MessageHandler(string msg, CancellationToken token) + { + receivedMessages.AddOrUpdate("received", 1, (key, value) => value + 1); + subscriberReceived.SetResult(true); + } + + await _messagingProvider.SubscribeAsync(topicName, consumerGroup, MessageHandler, _cts.Token); + await _messagingProvider.SubscribeAsync(topicName, consumerGroup, MessageHandler, _cts.Token); + + await WaitForKafkaConsumerGroupStable(consumerGroup, _cts.Token); + + // Act + await _messagingProvider.PublishAsync(topicName, messageToSend, _cts.Token); + + try + { + await subscriberReceived.Task.WaitAsync(TimeSpan.FromSeconds(20)); + } + catch (TimeoutException) + { + _logger.LogDebug("Timed out waiting for subscriber to receive message."); + } + + // Assert + receivedMessages.GetValueOrDefault("received", 0).Should().Be(1); + } [Fact] public async Task CompetingConsumers_ShouldDistributeEvenly_WhenMultipleConsumersInGroup() @@ -308,7 +309,6 @@ async Task MessageHandler(string msg, CancellationToken token) await _messagingProvider.SubscribeAsync(topicName, consumerGroup, MessageHandler, _cts.Token); await _messagingProvider.SubscribeAsync(topicName, consumerGroup, MessageHandler, _cts.Token); - // await Task.Delay(1000); await WaitForKafkaConsumerGroupStable(consumerGroup, _cts.Token); // Act @@ -338,7 +338,7 @@ async Task MessageHandler(string msg, CancellationToken token) private async Task WaitForKafkaConsumerGroupStable(string consumerGroup, CancellationToken cancellationToken) { - var maxAttempts = 10; + var maxAttempts = 30; for (int i = 0; i < maxAttempts; i++) { var groupInfo = _adminClient.ListGroups(TimeSpan.FromSeconds(5)) From b713b54584b1d85f983aff9961a38d80148972fa Mon Sep 17 00:00:00 2001 From: David Runemalm Date: Tue, 4 Mar 2025 09:22:42 +0700 Subject: [PATCH 077/109] Add integration tests for postgres & sqlite efcore repositories. --- Makefile | 20 +- .../TestAggregateRootConfiguration.cs | 24 ++ .../Configurations/TestEntityConfiguration.cs | 16 ++ .../TestValueObjectConfiguration.cs | 14 ++ .../Postgres/PostgresTestDbContext.cs | 18 ++ .../Postgres/PostgresTestDbContextFactory.cs | 23 ++ .../DbContext/Sqlite/SqliteTestDbContext.cs | 18 ++ .../Sqlite/SqliteTestDbContextFactory.cs | 24 ++ ...4021847_Postgres_InitialCreate.Designer.cs | 140 +++++++++++ .../20250304021847_Postgres_InitialCreate.cs | 84 +++++++ .../PostgresTestDbContextModelSnapshot.cs | 137 ++++++++++ ...304021905_Sqlite_InitialCreate.Designer.cs | 135 ++++++++++ .../20250304021905_Sqlite_InitialCreate.cs | 84 +++++++ .../SqliteTestDbContextModelSnapshot.cs | 132 ++++++++++ .../EfCore/EfCoreTestsCollection.cs | 5 + .../Postgres/EfCorePostgresRepositoryTests.cs | 233 ++++++++++++++++++ .../Sqlite/EfCoreSqliteRepositoryTests.cs | 198 +++++++++++++++ src/OpenDDD.Tests/OpenDDD.Tests.csproj | 5 + .../EfCore/Base/OpenDddDbContextBase.cs | 2 +- .../Infrastructure/Utils/TypeScanner.cs | 6 +- 20 files changed, 1313 insertions(+), 5 deletions(-) create mode 100644 src/OpenDDD.Tests/Integration/Infrastructure/Persistence/EfCore/Configurations/TestAggregateRootConfiguration.cs create mode 100644 src/OpenDDD.Tests/Integration/Infrastructure/Persistence/EfCore/Configurations/TestEntityConfiguration.cs create mode 100644 src/OpenDDD.Tests/Integration/Infrastructure/Persistence/EfCore/Configurations/TestValueObjectConfiguration.cs create mode 100644 src/OpenDDD.Tests/Integration/Infrastructure/Persistence/EfCore/DbContext/Postgres/PostgresTestDbContext.cs create mode 100644 src/OpenDDD.Tests/Integration/Infrastructure/Persistence/EfCore/DbContext/Postgres/PostgresTestDbContextFactory.cs create mode 100644 src/OpenDDD.Tests/Integration/Infrastructure/Persistence/EfCore/DbContext/Sqlite/SqliteTestDbContext.cs create mode 100644 src/OpenDDD.Tests/Integration/Infrastructure/Persistence/EfCore/DbContext/Sqlite/SqliteTestDbContextFactory.cs create mode 100644 src/OpenDDD.Tests/Integration/Infrastructure/Persistence/EfCore/Migrations/Postgres/20250304021847_Postgres_InitialCreate.Designer.cs create mode 100644 src/OpenDDD.Tests/Integration/Infrastructure/Persistence/EfCore/Migrations/Postgres/20250304021847_Postgres_InitialCreate.cs create mode 100644 src/OpenDDD.Tests/Integration/Infrastructure/Persistence/EfCore/Migrations/Postgres/PostgresTestDbContextModelSnapshot.cs create mode 100644 src/OpenDDD.Tests/Integration/Infrastructure/Persistence/EfCore/Migrations/Sqlite/20250304021905_Sqlite_InitialCreate.Designer.cs create mode 100644 src/OpenDDD.Tests/Integration/Infrastructure/Persistence/EfCore/Migrations/Sqlite/20250304021905_Sqlite_InitialCreate.cs create mode 100644 src/OpenDDD.Tests/Integration/Infrastructure/Persistence/EfCore/Migrations/Sqlite/SqliteTestDbContextModelSnapshot.cs create mode 100644 src/OpenDDD.Tests/Integration/Infrastructure/Repository/EfCore/EfCoreTestsCollection.cs create mode 100644 src/OpenDDD.Tests/Integration/Infrastructure/Repository/EfCore/Postgres/EfCorePostgresRepositoryTests.cs create mode 100644 src/OpenDDD.Tests/Integration/Infrastructure/Repository/EfCore/Sqlite/EfCoreSqliteRepositoryTests.cs diff --git a/Makefile b/Makefile index 7ccb7ec..5eb43fb 100644 --- a/Makefile +++ b/Makefile @@ -108,6 +108,24 @@ test-unit: ##@Test Run only unit tests test-integration: ##@Test Run only integration tests cd $(TESTS_DIR) && dotnet test --configuration Release --filter "Category=Integration" +.PHONY: test-ef-migrations-create-postgres +test-ef-migrations-create-postgres: ##@Test Create PostgreSQL migrations for PostgresTestDbContext + cd $(TESTS_DIR) && \ + dotnet ef migrations add Postgres_InitialCreate \ + --context PostgresTestDbContext \ + --output-dir Integration/Infrastructure/Persistence/EfCore/Migrations/Postgres \ + --project $(TESTS_DIR) \ + -- --database-provider postgres + +.PHONY: test-ef-migrations-create-sqlite +test-ef-migrations-create-sqlite: ##@Test Create SQLite migrations for SqliteTestDbContext + cd $(TESTS_DIR) && \ + dotnet ef migrations add Sqlite_InitialCreate \ + --context SqliteTestDbContext \ + --output-dir Integration/Infrastructure/Persistence/EfCore/Migrations/Sqlite \ + --project $(TESTS_DIR) \ + -- --database-provider sqlite + ########################################################################## # BUILD ########################################################################## @@ -437,7 +455,7 @@ POSTGRES_DB := testdb .PHONY: postgres-start postgres-start: ##@Postgres Start a PostgreSQL container - @docker run -d --name $(POSTGRES_CONTAINER) --network $(NETWORK) \ + @docker run --rm -d --name $(POSTGRES_CONTAINER) --network $(NETWORK) \ -e POSTGRES_DB=$(POSTGRES_DB) \ -e POSTGRES_USER=$(POSTGRES_USER) \ -e POSTGRES_PASSWORD=$(POSTGRES_PASSWORD) \ diff --git a/src/OpenDDD.Tests/Integration/Infrastructure/Persistence/EfCore/Configurations/TestAggregateRootConfiguration.cs b/src/OpenDDD.Tests/Integration/Infrastructure/Persistence/EfCore/Configurations/TestAggregateRootConfiguration.cs new file mode 100644 index 0000000..49aa41f --- /dev/null +++ b/src/OpenDDD.Tests/Integration/Infrastructure/Persistence/EfCore/Configurations/TestAggregateRootConfiguration.cs @@ -0,0 +1,24 @@ +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Metadata.Builders; +using OpenDDD.Infrastructure.Persistence.EfCore.Base; +using OpenDDD.Tests.Domain.Model; + +namespace OpenDDD.Tests.Integration.Infrastructure.Persistence.EfCore.Configurations +{ + public class TestAggregateRootConfiguration : EfAggregateRootConfigurationBase + { + public override void Configure(EntityTypeBuilder builder) + { + base.Configure(builder); + + builder.OwnsOne(a => a.Value); + + builder.HasMany(a => a.Entities) + .WithOne() + .HasForeignKey("TestAggregateRootId") + .OnDelete(DeleteBehavior.Cascade); + + builder.Navigation(a => a.Entities).AutoInclude(); + } + } +} diff --git a/src/OpenDDD.Tests/Integration/Infrastructure/Persistence/EfCore/Configurations/TestEntityConfiguration.cs b/src/OpenDDD.Tests/Integration/Infrastructure/Persistence/EfCore/Configurations/TestEntityConfiguration.cs new file mode 100644 index 0000000..db4830b --- /dev/null +++ b/src/OpenDDD.Tests/Integration/Infrastructure/Persistence/EfCore/Configurations/TestEntityConfiguration.cs @@ -0,0 +1,16 @@ +using Microsoft.EntityFrameworkCore.Metadata.Builders; +using OpenDDD.Infrastructure.Persistence.EfCore.Base; +using OpenDDD.Tests.Domain.Model; + +namespace OpenDDD.Tests.Integration.Infrastructure.Persistence.EfCore.Configurations +{ + public class TestEntityConfiguration : EfEntityConfigurationBase + { + public override void Configure(EntityTypeBuilder builder) + { + base.Configure(builder); + + builder.Property("TestAggregateRootId").IsRequired(); + } + } +} diff --git a/src/OpenDDD.Tests/Integration/Infrastructure/Persistence/EfCore/Configurations/TestValueObjectConfiguration.cs b/src/OpenDDD.Tests/Integration/Infrastructure/Persistence/EfCore/Configurations/TestValueObjectConfiguration.cs new file mode 100644 index 0000000..07447cc --- /dev/null +++ b/src/OpenDDD.Tests/Integration/Infrastructure/Persistence/EfCore/Configurations/TestValueObjectConfiguration.cs @@ -0,0 +1,14 @@ +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Metadata.Builders; +using OpenDDD.Tests.Domain.Model; + +namespace OpenDDD.Tests.Integration.Infrastructure.Persistence.EfCore.Configurations +{ + public class TestValueObjectConfiguration : IEntityTypeConfiguration + { + public void Configure(EntityTypeBuilder builder) + { + builder.HasNoKey(); + } + } +} diff --git a/src/OpenDDD.Tests/Integration/Infrastructure/Persistence/EfCore/DbContext/Postgres/PostgresTestDbContext.cs b/src/OpenDDD.Tests/Integration/Infrastructure/Persistence/EfCore/DbContext/Postgres/PostgresTestDbContext.cs new file mode 100644 index 0000000..0178a59 --- /dev/null +++ b/src/OpenDDD.Tests/Integration/Infrastructure/Persistence/EfCore/DbContext/Postgres/PostgresTestDbContext.cs @@ -0,0 +1,18 @@ +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Logging; +using OpenDDD.API.Options; +using OpenDDD.Infrastructure.Persistence.EfCore.Base; +using OpenDDD.Tests.Domain.Model; + +namespace OpenDDD.Tests.Integration.Infrastructure.Persistence.EfCore.DbContext.Postgres +{ + public class PostgresTestDbContext : OpenDddDbContextBase + { + public PostgresTestDbContext(DbContextOptions options, OpenDddOptions openDddOptions, ILogger logger) + : base(options, openDddOptions, logger) + { + } + + public DbSet TestAggregates { get; set; } = null!; + } +} diff --git a/src/OpenDDD.Tests/Integration/Infrastructure/Persistence/EfCore/DbContext/Postgres/PostgresTestDbContextFactory.cs b/src/OpenDDD.Tests/Integration/Infrastructure/Persistence/EfCore/DbContext/Postgres/PostgresTestDbContextFactory.cs new file mode 100644 index 0000000..bec47b7 --- /dev/null +++ b/src/OpenDDD.Tests/Integration/Infrastructure/Persistence/EfCore/DbContext/Postgres/PostgresTestDbContextFactory.cs @@ -0,0 +1,23 @@ +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Design; +using Microsoft.Extensions.Logging; +using OpenDDD.API.Options; + +namespace OpenDDD.Tests.Integration.Infrastructure.Persistence.EfCore.DbContext.Postgres +{ + public class PostgresTestDbContextFactory : IDesignTimeDbContextFactory + { + public PostgresTestDbContext CreateDbContext(string[] args) + { + var optionsBuilder = new DbContextOptionsBuilder() + .UseNpgsql("Host=localhost;Port=5432;Database=testdb;Username=testuser;Password=testpassword") + .EnableSensitiveDataLogging(); + + var openDddOptions = new OpenDddOptions(); + var loggerFactory = LoggerFactory.Create(builder => builder.AddConsole()); + var logger = loggerFactory.CreateLogger(); + + return new PostgresTestDbContext(optionsBuilder.Options, openDddOptions, logger); + } + } +} diff --git a/src/OpenDDD.Tests/Integration/Infrastructure/Persistence/EfCore/DbContext/Sqlite/SqliteTestDbContext.cs b/src/OpenDDD.Tests/Integration/Infrastructure/Persistence/EfCore/DbContext/Sqlite/SqliteTestDbContext.cs new file mode 100644 index 0000000..1be7873 --- /dev/null +++ b/src/OpenDDD.Tests/Integration/Infrastructure/Persistence/EfCore/DbContext/Sqlite/SqliteTestDbContext.cs @@ -0,0 +1,18 @@ +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Logging; +using OpenDDD.API.Options; +using OpenDDD.Infrastructure.Persistence.EfCore.Base; +using OpenDDD.Tests.Domain.Model; + +namespace OpenDDD.Tests.Integration.Infrastructure.Persistence.EfCore.DbContext.Sqlite +{ + public class SqliteTestDbContext : OpenDddDbContextBase + { + public SqliteTestDbContext(DbContextOptions options, OpenDddOptions openDddOptions, ILogger logger) + : base(options, openDddOptions, logger) + { + } + + public DbSet TestAggregates { get; set; } = null!; + } +} diff --git a/src/OpenDDD.Tests/Integration/Infrastructure/Persistence/EfCore/DbContext/Sqlite/SqliteTestDbContextFactory.cs b/src/OpenDDD.Tests/Integration/Infrastructure/Persistence/EfCore/DbContext/Sqlite/SqliteTestDbContextFactory.cs new file mode 100644 index 0000000..4f6c2e3 --- /dev/null +++ b/src/OpenDDD.Tests/Integration/Infrastructure/Persistence/EfCore/DbContext/Sqlite/SqliteTestDbContextFactory.cs @@ -0,0 +1,24 @@ +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Design; +using Microsoft.Extensions.Logging; +using OpenDDD.API.Options; + +namespace OpenDDD.Tests.Integration.Infrastructure.Persistence.EfCore.DbContext.Sqlite +{ + public class SqliteTestDbContextFactory : IDesignTimeDbContextFactory + { + public SqliteTestDbContext CreateDbContext(string[] args) + { + var optionsBuilder = new DbContextOptionsBuilder() + .UseSqlite("DataSource=:memory:") + .EnableSensitiveDataLogging(); + + var openDddOptions = new OpenDddOptions(); + var loggerFactory = LoggerFactory.Create(builder => builder.AddConsole()); + var logger = loggerFactory.CreateLogger(); + + return new SqliteTestDbContext(optionsBuilder.Options, openDddOptions, logger); + } + + } +} diff --git a/src/OpenDDD.Tests/Integration/Infrastructure/Persistence/EfCore/Migrations/Postgres/20250304021847_Postgres_InitialCreate.Designer.cs b/src/OpenDDD.Tests/Integration/Infrastructure/Persistence/EfCore/Migrations/Postgres/20250304021847_Postgres_InitialCreate.Designer.cs new file mode 100644 index 0000000..86b891d --- /dev/null +++ b/src/OpenDDD.Tests/Integration/Infrastructure/Persistence/EfCore/Migrations/Postgres/20250304021847_Postgres_InitialCreate.Designer.cs @@ -0,0 +1,140 @@ +// +using System; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; +using OpenDDD.Tests.Integration.Infrastructure.Persistence.EfCore.DbContext.Postgres; + +#nullable disable + +namespace OpenDDD.Tests.Integration.Infrastructure.Persistence.EfCore.Migrations.Postgres +{ + [DbContext(typeof(PostgresTestDbContext))] + [Migration("20250304021847_Postgres_InitialCreate")] + partial class Postgres_InitialCreate + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "9.0.2") + .HasAnnotation("Relational:MaxIdentifierLength", 63); + + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + + modelBuilder.Entity("OpenDDD.Infrastructure.TransactionalOutbox.OutboxEntry", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("EventName") + .IsRequired() + .HasColumnType("text"); + + b.Property("EventType") + .IsRequired() + .HasColumnType("text"); + + b.Property("Payload") + .IsRequired() + .HasColumnType("text"); + + b.Property("ProcessedAt") + .HasColumnType("timestamp with time zone"); + + b.HasKey("Id"); + + b.ToTable("OutboxEntries"); + }); + + modelBuilder.Entity("OpenDDD.Tests.Domain.Model.TestAggregateRoot", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text"); + + b.Property("UpdatedAt") + .HasColumnType("timestamp with time zone"); + + b.HasKey("Id"); + + b.ToTable("TestAggregateRoots", (string)null); + }); + + modelBuilder.Entity("OpenDDD.Tests.Domain.Model.TestEntity", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("Description") + .IsRequired() + .HasColumnType("text"); + + b.Property("TestAggregateRootId") + .HasColumnType("uuid"); + + b.HasKey("Id"); + + b.HasIndex("TestAggregateRootId"); + + b.ToTable("TestEntities", (string)null); + }); + + modelBuilder.Entity("OpenDDD.Tests.Domain.Model.TestAggregateRoot", b => + { + b.OwnsOne("OpenDDD.Tests.Domain.Model.TestValueObject", "Value", b1 => + { + b1.Property("TestAggregateRootId") + .HasColumnType("uuid"); + + b1.Property("Number") + .HasColumnType("integer"); + + b1.Property("Text") + .IsRequired() + .HasColumnType("text"); + + b1.HasKey("TestAggregateRootId"); + + b1.ToTable("TestAggregateRoots"); + + b1.WithOwner() + .HasForeignKey("TestAggregateRootId"); + }); + + b.Navigation("Value") + .IsRequired(); + }); + + modelBuilder.Entity("OpenDDD.Tests.Domain.Model.TestEntity", b => + { + b.HasOne("OpenDDD.Tests.Domain.Model.TestAggregateRoot", null) + .WithMany("Entities") + .HasForeignKey("TestAggregateRootId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("OpenDDD.Tests.Domain.Model.TestAggregateRoot", b => + { + b.Navigation("Entities"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/src/OpenDDD.Tests/Integration/Infrastructure/Persistence/EfCore/Migrations/Postgres/20250304021847_Postgres_InitialCreate.cs b/src/OpenDDD.Tests/Integration/Infrastructure/Persistence/EfCore/Migrations/Postgres/20250304021847_Postgres_InitialCreate.cs new file mode 100644 index 0000000..7e861b0 --- /dev/null +++ b/src/OpenDDD.Tests/Integration/Infrastructure/Persistence/EfCore/Migrations/Postgres/20250304021847_Postgres_InitialCreate.cs @@ -0,0 +1,84 @@ +using System; +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace OpenDDD.Tests.Integration.Infrastructure.Persistence.EfCore.Migrations.Postgres +{ + /// + public partial class Postgres_InitialCreate : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.CreateTable( + name: "OutboxEntries", + columns: table => new + { + Id = table.Column(type: "uuid", nullable: false), + EventType = table.Column(type: "text", nullable: false), + EventName = table.Column(type: "text", nullable: false), + Payload = table.Column(type: "text", nullable: false), + CreatedAt = table.Column(type: "timestamp with time zone", nullable: false), + ProcessedAt = table.Column(type: "timestamp with time zone", nullable: true) + }, + constraints: table => + { + table.PrimaryKey("PK_OutboxEntries", x => x.Id); + }); + + migrationBuilder.CreateTable( + name: "TestAggregateRoots", + columns: table => new + { + Id = table.Column(type: "uuid", nullable: false), + Name = table.Column(type: "text", nullable: false), + Value_Number = table.Column(type: "integer", nullable: false), + Value_Text = table.Column(type: "text", nullable: false), + CreatedAt = table.Column(type: "timestamp with time zone", nullable: false), + UpdatedAt = table.Column(type: "timestamp with time zone", nullable: false) + }, + constraints: table => + { + table.PrimaryKey("PK_TestAggregateRoots", x => x.Id); + }); + + migrationBuilder.CreateTable( + name: "TestEntities", + columns: table => new + { + Id = table.Column(type: "uuid", nullable: false), + Description = table.Column(type: "text", nullable: false), + TestAggregateRootId = table.Column(type: "uuid", nullable: false) + }, + constraints: table => + { + table.PrimaryKey("PK_TestEntities", x => x.Id); + table.ForeignKey( + name: "FK_TestEntities_TestAggregateRoots_TestAggregateRootId", + column: x => x.TestAggregateRootId, + principalTable: "TestAggregateRoots", + principalColumn: "Id", + onDelete: ReferentialAction.Cascade); + }); + + migrationBuilder.CreateIndex( + name: "IX_TestEntities_TestAggregateRootId", + table: "TestEntities", + column: "TestAggregateRootId"); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropTable( + name: "OutboxEntries"); + + migrationBuilder.DropTable( + name: "TestEntities"); + + migrationBuilder.DropTable( + name: "TestAggregateRoots"); + } + } +} diff --git a/src/OpenDDD.Tests/Integration/Infrastructure/Persistence/EfCore/Migrations/Postgres/PostgresTestDbContextModelSnapshot.cs b/src/OpenDDD.Tests/Integration/Infrastructure/Persistence/EfCore/Migrations/Postgres/PostgresTestDbContextModelSnapshot.cs new file mode 100644 index 0000000..8105d48 --- /dev/null +++ b/src/OpenDDD.Tests/Integration/Infrastructure/Persistence/EfCore/Migrations/Postgres/PostgresTestDbContextModelSnapshot.cs @@ -0,0 +1,137 @@ +// +using System; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; +using OpenDDD.Tests.Integration.Infrastructure.Persistence.EfCore.DbContext.Postgres; + +#nullable disable + +namespace OpenDDD.Tests.Integration.Infrastructure.Persistence.EfCore.Migrations.Postgres +{ + [DbContext(typeof(PostgresTestDbContext))] + partial class PostgresTestDbContextModelSnapshot : ModelSnapshot + { + protected override void BuildModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "9.0.2") + .HasAnnotation("Relational:MaxIdentifierLength", 63); + + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + + modelBuilder.Entity("OpenDDD.Infrastructure.TransactionalOutbox.OutboxEntry", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("EventName") + .IsRequired() + .HasColumnType("text"); + + b.Property("EventType") + .IsRequired() + .HasColumnType("text"); + + b.Property("Payload") + .IsRequired() + .HasColumnType("text"); + + b.Property("ProcessedAt") + .HasColumnType("timestamp with time zone"); + + b.HasKey("Id"); + + b.ToTable("OutboxEntries"); + }); + + modelBuilder.Entity("OpenDDD.Tests.Domain.Model.TestAggregateRoot", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text"); + + b.Property("UpdatedAt") + .HasColumnType("timestamp with time zone"); + + b.HasKey("Id"); + + b.ToTable("TestAggregateRoots", (string)null); + }); + + modelBuilder.Entity("OpenDDD.Tests.Domain.Model.TestEntity", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("Description") + .IsRequired() + .HasColumnType("text"); + + b.Property("TestAggregateRootId") + .HasColumnType("uuid"); + + b.HasKey("Id"); + + b.HasIndex("TestAggregateRootId"); + + b.ToTable("TestEntities", (string)null); + }); + + modelBuilder.Entity("OpenDDD.Tests.Domain.Model.TestAggregateRoot", b => + { + b.OwnsOne("OpenDDD.Tests.Domain.Model.TestValueObject", "Value", b1 => + { + b1.Property("TestAggregateRootId") + .HasColumnType("uuid"); + + b1.Property("Number") + .HasColumnType("integer"); + + b1.Property("Text") + .IsRequired() + .HasColumnType("text"); + + b1.HasKey("TestAggregateRootId"); + + b1.ToTable("TestAggregateRoots"); + + b1.WithOwner() + .HasForeignKey("TestAggregateRootId"); + }); + + b.Navigation("Value") + .IsRequired(); + }); + + modelBuilder.Entity("OpenDDD.Tests.Domain.Model.TestEntity", b => + { + b.HasOne("OpenDDD.Tests.Domain.Model.TestAggregateRoot", null) + .WithMany("Entities") + .HasForeignKey("TestAggregateRootId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("OpenDDD.Tests.Domain.Model.TestAggregateRoot", b => + { + b.Navigation("Entities"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/src/OpenDDD.Tests/Integration/Infrastructure/Persistence/EfCore/Migrations/Sqlite/20250304021905_Sqlite_InitialCreate.Designer.cs b/src/OpenDDD.Tests/Integration/Infrastructure/Persistence/EfCore/Migrations/Sqlite/20250304021905_Sqlite_InitialCreate.Designer.cs new file mode 100644 index 0000000..3b35708 --- /dev/null +++ b/src/OpenDDD.Tests/Integration/Infrastructure/Persistence/EfCore/Migrations/Sqlite/20250304021905_Sqlite_InitialCreate.Designer.cs @@ -0,0 +1,135 @@ +// +using System; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using OpenDDD.Tests.Integration.Infrastructure.Persistence.EfCore.DbContext.Sqlite; + +#nullable disable + +namespace OpenDDD.Tests.Integration.Infrastructure.Persistence.EfCore.Migrations.Sqlite +{ + [DbContext(typeof(SqliteTestDbContext))] + [Migration("20250304021905_Sqlite_InitialCreate")] + partial class Sqlite_InitialCreate + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder.HasAnnotation("ProductVersion", "9.0.2"); + + modelBuilder.Entity("OpenDDD.Infrastructure.TransactionalOutbox.OutboxEntry", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("TEXT"); + + b.Property("CreatedAt") + .HasColumnType("TEXT"); + + b.Property("EventName") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("EventType") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("Payload") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("ProcessedAt") + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.ToTable("OutboxEntries"); + }); + + modelBuilder.Entity("OpenDDD.Tests.Domain.Model.TestAggregateRoot", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("TEXT"); + + b.Property("CreatedAt") + .HasColumnType("TEXT"); + + b.Property("Name") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("UpdatedAt") + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.ToTable("TestAggregateRoots", (string)null); + }); + + modelBuilder.Entity("OpenDDD.Tests.Domain.Model.TestEntity", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("TEXT"); + + b.Property("Description") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("TestAggregateRootId") + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("TestAggregateRootId"); + + b.ToTable("TestEntities", (string)null); + }); + + modelBuilder.Entity("OpenDDD.Tests.Domain.Model.TestAggregateRoot", b => + { + b.OwnsOne("OpenDDD.Tests.Domain.Model.TestValueObject", "Value", b1 => + { + b1.Property("TestAggregateRootId") + .HasColumnType("TEXT"); + + b1.Property("Number") + .HasColumnType("INTEGER"); + + b1.Property("Text") + .IsRequired() + .HasColumnType("TEXT"); + + b1.HasKey("TestAggregateRootId"); + + b1.ToTable("TestAggregateRoots"); + + b1.WithOwner() + .HasForeignKey("TestAggregateRootId"); + }); + + b.Navigation("Value") + .IsRequired(); + }); + + modelBuilder.Entity("OpenDDD.Tests.Domain.Model.TestEntity", b => + { + b.HasOne("OpenDDD.Tests.Domain.Model.TestAggregateRoot", null) + .WithMany("Entities") + .HasForeignKey("TestAggregateRootId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("OpenDDD.Tests.Domain.Model.TestAggregateRoot", b => + { + b.Navigation("Entities"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/src/OpenDDD.Tests/Integration/Infrastructure/Persistence/EfCore/Migrations/Sqlite/20250304021905_Sqlite_InitialCreate.cs b/src/OpenDDD.Tests/Integration/Infrastructure/Persistence/EfCore/Migrations/Sqlite/20250304021905_Sqlite_InitialCreate.cs new file mode 100644 index 0000000..a01dd6e --- /dev/null +++ b/src/OpenDDD.Tests/Integration/Infrastructure/Persistence/EfCore/Migrations/Sqlite/20250304021905_Sqlite_InitialCreate.cs @@ -0,0 +1,84 @@ +using System; +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace OpenDDD.Tests.Integration.Infrastructure.Persistence.EfCore.Migrations.Sqlite +{ + /// + public partial class Sqlite_InitialCreate : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.CreateTable( + name: "OutboxEntries", + columns: table => new + { + Id = table.Column(type: "TEXT", nullable: false), + EventType = table.Column(type: "TEXT", nullable: false), + EventName = table.Column(type: "TEXT", nullable: false), + Payload = table.Column(type: "TEXT", nullable: false), + CreatedAt = table.Column(type: "TEXT", nullable: false), + ProcessedAt = table.Column(type: "TEXT", nullable: true) + }, + constraints: table => + { + table.PrimaryKey("PK_OutboxEntries", x => x.Id); + }); + + migrationBuilder.CreateTable( + name: "TestAggregateRoots", + columns: table => new + { + Id = table.Column(type: "TEXT", nullable: false), + Name = table.Column(type: "TEXT", nullable: false), + Value_Number = table.Column(type: "INTEGER", nullable: false), + Value_Text = table.Column(type: "TEXT", nullable: false), + CreatedAt = table.Column(type: "TEXT", nullable: false), + UpdatedAt = table.Column(type: "TEXT", nullable: false) + }, + constraints: table => + { + table.PrimaryKey("PK_TestAggregateRoots", x => x.Id); + }); + + migrationBuilder.CreateTable( + name: "TestEntities", + columns: table => new + { + Id = table.Column(type: "TEXT", nullable: false), + Description = table.Column(type: "TEXT", nullable: false), + TestAggregateRootId = table.Column(type: "TEXT", nullable: false) + }, + constraints: table => + { + table.PrimaryKey("PK_TestEntities", x => x.Id); + table.ForeignKey( + name: "FK_TestEntities_TestAggregateRoots_TestAggregateRootId", + column: x => x.TestAggregateRootId, + principalTable: "TestAggregateRoots", + principalColumn: "Id", + onDelete: ReferentialAction.Cascade); + }); + + migrationBuilder.CreateIndex( + name: "IX_TestEntities_TestAggregateRootId", + table: "TestEntities", + column: "TestAggregateRootId"); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropTable( + name: "OutboxEntries"); + + migrationBuilder.DropTable( + name: "TestEntities"); + + migrationBuilder.DropTable( + name: "TestAggregateRoots"); + } + } +} diff --git a/src/OpenDDD.Tests/Integration/Infrastructure/Persistence/EfCore/Migrations/Sqlite/SqliteTestDbContextModelSnapshot.cs b/src/OpenDDD.Tests/Integration/Infrastructure/Persistence/EfCore/Migrations/Sqlite/SqliteTestDbContextModelSnapshot.cs new file mode 100644 index 0000000..5ae467c --- /dev/null +++ b/src/OpenDDD.Tests/Integration/Infrastructure/Persistence/EfCore/Migrations/Sqlite/SqliteTestDbContextModelSnapshot.cs @@ -0,0 +1,132 @@ +// +using System; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using OpenDDD.Tests.Integration.Infrastructure.Persistence.EfCore.DbContext.Sqlite; + +#nullable disable + +namespace OpenDDD.Tests.Integration.Infrastructure.Persistence.EfCore.Migrations.Sqlite +{ + [DbContext(typeof(SqliteTestDbContext))] + partial class SqliteTestDbContextModelSnapshot : ModelSnapshot + { + protected override void BuildModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder.HasAnnotation("ProductVersion", "9.0.2"); + + modelBuilder.Entity("OpenDDD.Infrastructure.TransactionalOutbox.OutboxEntry", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("TEXT"); + + b.Property("CreatedAt") + .HasColumnType("TEXT"); + + b.Property("EventName") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("EventType") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("Payload") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("ProcessedAt") + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.ToTable("OutboxEntries"); + }); + + modelBuilder.Entity("OpenDDD.Tests.Domain.Model.TestAggregateRoot", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("TEXT"); + + b.Property("CreatedAt") + .HasColumnType("TEXT"); + + b.Property("Name") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("UpdatedAt") + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.ToTable("TestAggregateRoots", (string)null); + }); + + modelBuilder.Entity("OpenDDD.Tests.Domain.Model.TestEntity", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("TEXT"); + + b.Property("Description") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("TestAggregateRootId") + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("TestAggregateRootId"); + + b.ToTable("TestEntities", (string)null); + }); + + modelBuilder.Entity("OpenDDD.Tests.Domain.Model.TestAggregateRoot", b => + { + b.OwnsOne("OpenDDD.Tests.Domain.Model.TestValueObject", "Value", b1 => + { + b1.Property("TestAggregateRootId") + .HasColumnType("TEXT"); + + b1.Property("Number") + .HasColumnType("INTEGER"); + + b1.Property("Text") + .IsRequired() + .HasColumnType("TEXT"); + + b1.HasKey("TestAggregateRootId"); + + b1.ToTable("TestAggregateRoots"); + + b1.WithOwner() + .HasForeignKey("TestAggregateRootId"); + }); + + b.Navigation("Value") + .IsRequired(); + }); + + modelBuilder.Entity("OpenDDD.Tests.Domain.Model.TestEntity", b => + { + b.HasOne("OpenDDD.Tests.Domain.Model.TestAggregateRoot", null) + .WithMany("Entities") + .HasForeignKey("TestAggregateRootId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("OpenDDD.Tests.Domain.Model.TestAggregateRoot", b => + { + b.Navigation("Entities"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/src/OpenDDD.Tests/Integration/Infrastructure/Repository/EfCore/EfCoreTestsCollection.cs b/src/OpenDDD.Tests/Integration/Infrastructure/Repository/EfCore/EfCoreTestsCollection.cs new file mode 100644 index 0000000..8f2f8f4 --- /dev/null +++ b/src/OpenDDD.Tests/Integration/Infrastructure/Repository/EfCore/EfCoreTestsCollection.cs @@ -0,0 +1,5 @@ +namespace OpenDDD.Tests.Integration.Infrastructure.Repository.EfCore +{ + [CollectionDefinition("EfCoreTests", DisableParallelization = true)] + public class EfCoreTestsCollection { } +} diff --git a/src/OpenDDD.Tests/Integration/Infrastructure/Repository/EfCore/Postgres/EfCorePostgresRepositoryTests.cs b/src/OpenDDD.Tests/Integration/Infrastructure/Repository/EfCore/Postgres/EfCorePostgresRepositoryTests.cs new file mode 100644 index 0000000..7e1eb46 --- /dev/null +++ b/src/OpenDDD.Tests/Integration/Infrastructure/Repository/EfCore/Postgres/EfCorePostgresRepositoryTests.cs @@ -0,0 +1,233 @@ +using System.Linq.Expressions; +using FluentAssertions; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Xunit.Abstractions; +using Moq; +using Npgsql; +using OpenDDD.API.Options; +using OpenDDD.Domain.Model; +using OpenDDD.Infrastructure.Persistence.EfCore.Base; +using OpenDDD.Infrastructure.Persistence.EfCore.DatabaseSession; +using OpenDDD.Infrastructure.Persistence.EfCore.UoW; +using OpenDDD.Infrastructure.Repository.EfCore; +using OpenDDD.Infrastructure.TransactionalOutbox; +using OpenDDD.Tests.Base; +using OpenDDD.Tests.Domain.Model; +using OpenDDD.Tests.Integration.Infrastructure.Persistence.EfCore.DbContext.Postgres; + +namespace OpenDDD.Tests.Integration.Infrastructure.Repository.EfCore.Postgres +{ + [Collection("EfCoreTests")] + public class EfCorePostgresRepositoryTests : IntegrationTests, IAsyncLifetime + { + private readonly string _connectionString; + private readonly EfCoreDatabaseSession _session; + private readonly Mock _mockDomainPublisher; + private readonly Mock _mockIntegrationPublisher; + private readonly Mock _mockOutboxRepository; + private readonly EfCoreUnitOfWork _unitOfWork; + private readonly EfCoreRepository _repository; + private readonly PostgresTestDbContext _dbContext; + private readonly NpgsqlConnection _connection; + private NpgsqlTransaction _transaction = null!; + private readonly OpenDddOptions _openDddOptions; + private readonly ILoggerFactory _loggerFactory; + private readonly ILogger _dbContextLogger; + + public EfCorePostgresRepositoryTests(ITestOutputHelper testOutputHelper) + : base(testOutputHelper, enableLogging: true) + { + _connectionString = Environment.GetEnvironmentVariable("POSTGRES_TEST_CONNECTION_STRING") + ?? "Host=localhost;Port=5432;Database=testdb;Username=testuser;Password=testpassword"; + + _connection = new NpgsqlConnection(_connectionString); + + var options = new DbContextOptionsBuilder() + .UseNpgsql(_connection, x => x.MigrationsHistoryTable("__EFMigrationsHistory", "public")) + .EnableSensitiveDataLogging() + .Options; + + _openDddOptions = new OpenDddOptions + { + AutoRegister = { EfCoreConfigurations = true } + }; + + var services = new ServiceCollection() + .AddLogging(builder => builder.AddSimpleConsole()) + .BuildServiceProvider(); + _dbContextLogger = services.GetRequiredService>(); + + _dbContext = new PostgresTestDbContext(options, _openDddOptions, _dbContextLogger); + + _session = new EfCoreDatabaseSession(_dbContext); + + _mockDomainPublisher = new Mock(); + _mockIntegrationPublisher = new Mock(); + _mockOutboxRepository = new Mock(); + + _loggerFactory = services.GetRequiredService(); + + _unitOfWork = new EfCoreUnitOfWork( + _session, + _mockDomainPublisher.Object, + _mockIntegrationPublisher.Object, + _mockOutboxRepository.Object, + _loggerFactory.CreateLogger() + ); + _repository = new EfCoreRepository(_unitOfWork); + } + + public async Task InitializeAsync() + { + // Re-create database + await _dbContext.Database.EnsureDeletedAsync(); + EnsureDatabaseExists(); + + // Run migrations + _dbContext.Database.SetCommandTimeout(120); + await _dbContext.Database.MigrateAsync(); + + // Initialize connection and transaction + _connection.Open(); + _transaction = await _connection.BeginTransactionAsync(); + _dbContext.Database.UseTransaction(_transaction); + } + + public async Task DisposeAsync() + { + await _transaction.RollbackAsync(); + await _connection.CloseAsync(); + } + + private void EnsureDatabaseExists() + { + using var adminConnection = new NpgsqlConnection($"{_connectionString};Database=postgres"); + adminConnection.Open(); + + var databaseName = "testdb"; + + using var command = new NpgsqlCommand($"SELECT 1 FROM pg_database WHERE datname = '{databaseName}'", adminConnection); + var databaseExists = command.ExecuteScalar() != null; + + if (!databaseExists) + { + using var createCommand = new NpgsqlCommand($"CREATE DATABASE {databaseName}", adminConnection); + createCommand.ExecuteNonQuery(); + } + } + + [Fact] + public async Task SaveAsync_ShouldInsertOrUpdateEntity() + { + // Arrange + var aggregate = TestAggregateRoot.Create( + "Initial Root", + new List { TestEntity.Create("Entity 1"), TestEntity.Create("Entity 2") }, + new TestValueObject(100, "Value Object Data") + ); + + // Act + await _repository.SaveAsync(aggregate, CancellationToken.None); + var retrieved = await _repository.GetAsync(aggregate.Id, CancellationToken.None); + + // Assert + retrieved.Should().NotBeNull(); + retrieved.Id.Should().Be(aggregate.Id); + retrieved.Name.Should().Be("Initial Root"); + retrieved.Entities.Should().HaveCount(2); + retrieved.Value.Number.Should().Be(100); + + // Act (update) + aggregate = TestAggregateRoot.Create( + "Updated Root", + new List { TestEntity.Create("Updated Entity") }, + new TestValueObject(200, "Updated Value") + ); + + await _repository.SaveAsync(aggregate, CancellationToken.None); + var updated = await _repository.GetAsync(aggregate.Id, CancellationToken.None); + + // Assert (update) + updated.Name.Should().Be("Updated Root"); + updated.Entities.Should().HaveCount(1); + updated.Entities.First().Description.Should().Be("Updated Entity"); + updated.Value.Number.Should().Be(200); + } + + [Fact] + public async Task FindAsync_ShouldReturnEntityIfExists() + { + // Arrange + var aggregate = TestAggregateRoot.Create("Find Test", new List(), new TestValueObject(50, "Find Test Value")); + await _repository.SaveAsync(aggregate, CancellationToken.None); + + // Act + var result = await _repository.FindAsync(aggregate.Id, CancellationToken.None); + + // Assert + result.Should().NotBeNull(); + result!.Id.Should().Be(aggregate.Id); + } + + [Fact] + public async Task FindAsync_ShouldReturnNullIfNotExists() + { + // Act + var result = await _repository.FindAsync(Guid.NewGuid(), CancellationToken.None); + + // Assert + result.Should().BeNull(); + } + + [Fact] + public async Task FindWithAsync_ShouldReturnFilteredResults() + { + // Arrange + var aggregate1 = TestAggregateRoot.Create("Filter Match", new List(), new TestValueObject(300, "Match")); + var aggregate2 = TestAggregateRoot.Create("No Match", new List(), new TestValueObject(400, "Different")); + await _repository.SaveAsync(aggregate1, CancellationToken.None); + await _repository.SaveAsync(aggregate2, CancellationToken.None); + + // Act + Expression> filter = a => a.Value.Number == 300; + var results = (await _repository.FindWithAsync(filter, CancellationToken.None)).ToList(); + + // Assert + results.Should().HaveCount(1); + results[0].Name.Should().Be("Filter Match"); + } + + [Fact] + public async Task FindAllAsync_ShouldReturnAllEntities() + { + // Arrange + var aggregate1 = TestAggregateRoot.Create("Entity 1", new List(), new TestValueObject(10, "VO 1")); + var aggregate2 = TestAggregateRoot.Create("Entity 2", new List(), new TestValueObject(20, "VO 2")); + await _repository.SaveAsync(aggregate1, CancellationToken.None); + await _repository.SaveAsync(aggregate2, CancellationToken.None); + + // Act + var results = (await _repository.FindAllAsync(CancellationToken.None)).ToList(); + + // Assert + results.Should().HaveCount(2); + } + + [Fact] + public async Task DeleteAsync_ShouldRemoveEntity() + { + // Arrange + var aggregate = TestAggregateRoot.Create("To be deleted", new List(), new TestValueObject(99, "Delete")); + await _repository.SaveAsync(aggregate, CancellationToken.None); + + // Act + await _repository.DeleteAsync(aggregate, CancellationToken.None); + var result = await _repository.FindAsync(aggregate.Id, CancellationToken.None); + + // Assert + result.Should().BeNull(); + } + } +} diff --git a/src/OpenDDD.Tests/Integration/Infrastructure/Repository/EfCore/Sqlite/EfCoreSqliteRepositoryTests.cs b/src/OpenDDD.Tests/Integration/Infrastructure/Repository/EfCore/Sqlite/EfCoreSqliteRepositoryTests.cs new file mode 100644 index 0000000..ca2b550 --- /dev/null +++ b/src/OpenDDD.Tests/Integration/Infrastructure/Repository/EfCore/Sqlite/EfCoreSqliteRepositoryTests.cs @@ -0,0 +1,198 @@ +using System.Linq.Expressions; +using FluentAssertions; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Xunit.Abstractions; +using Moq; +using OpenDDD.API.Options; +using OpenDDD.Domain.Model; +using OpenDDD.Infrastructure.Persistence.EfCore.Base; +using OpenDDD.Infrastructure.Persistence.EfCore.DatabaseSession; +using OpenDDD.Infrastructure.Persistence.EfCore.UoW; +using OpenDDD.Infrastructure.Repository.EfCore; +using OpenDDD.Infrastructure.TransactionalOutbox; +using OpenDDD.Tests.Base; +using OpenDDD.Tests.Domain.Model; +using OpenDDD.Tests.Integration.Infrastructure.Persistence.EfCore.DbContext.Sqlite; + +namespace OpenDDD.Tests.Integration.Infrastructure.Repository.EfCore.Sqlite +{ + [Collection("EfCoreTests")] + public class EfCoreSqliteRepositoryTests : IntegrationTests, IAsyncLifetime + { + private readonly EfCoreDatabaseSession _session; + private readonly Mock _mockDomainPublisher; + private readonly Mock _mockIntegrationPublisher; + private readonly Mock _mockOutboxRepository; + private readonly EfCoreUnitOfWork _unitOfWork; + private readonly EfCoreRepository _repository; + private readonly SqliteTestDbContext _dbContext; + private readonly OpenDddOptions _openDddOptions; + private readonly ILoggerFactory _loggerFactory; + private readonly ILogger _dbContextLogger; + private readonly string _connectionString = "DataSource=:memory:"; + + public EfCoreSqliteRepositoryTests(ITestOutputHelper testOutputHelper) + : base(testOutputHelper, enableLogging: true) + { + var options = new DbContextOptionsBuilder() + .UseSqlite(_connectionString) + .EnableSensitiveDataLogging() + .Options; + + _openDddOptions = new OpenDddOptions + { + AutoRegister = { EfCoreConfigurations = true } + }; + + var services = new ServiceCollection() + .AddLogging(builder => builder.AddSimpleConsole()) + .BuildServiceProvider(); + _dbContextLogger = services.GetRequiredService>(); + + _dbContext = new SqliteTestDbContext(options, _openDddOptions, _dbContextLogger); + + _session = new EfCoreDatabaseSession(_dbContext); + + _mockDomainPublisher = new Mock(); + _mockIntegrationPublisher = new Mock(); + _mockOutboxRepository = new Mock(); + + _loggerFactory = services.GetRequiredService(); + + _unitOfWork = new EfCoreUnitOfWork( + _session, + _mockDomainPublisher.Object, + _mockIntegrationPublisher.Object, + _mockOutboxRepository.Object, + _loggerFactory.CreateLogger() + ); + _repository = new EfCoreRepository(_unitOfWork); + } + + public async Task InitializeAsync() + { + await _dbContext.Database.OpenConnectionAsync(); + await _dbContext.Database.EnsureCreatedAsync(); + } + + public async Task DisposeAsync() + { + await _dbContext.Database.EnsureDeletedAsync(); + await _dbContext.DisposeAsync(); + } + + [Fact] + public async Task SaveAsync_ShouldInsertOrUpdateEntity() + { + // Arrange + var aggregate = TestAggregateRoot.Create( + "Initial Root", + new List { TestEntity.Create("Entity 1"), TestEntity.Create("Entity 2") }, + new TestValueObject(100, "Value Object Data") + ); + + // Act + await _repository.SaveAsync(aggregate, CancellationToken.None); + var retrieved = await _repository.GetAsync(aggregate.Id, CancellationToken.None); + + // Assert + retrieved.Should().NotBeNull(); + retrieved.Id.Should().Be(aggregate.Id); + retrieved.Name.Should().Be("Initial Root"); + retrieved.Entities.Should().HaveCount(2); + retrieved.Value.Number.Should().Be(100); + + // Act (update) + aggregate = TestAggregateRoot.Create( + "Updated Root", + new List { TestEntity.Create("Updated Entity") }, + new TestValueObject(200, "Updated Value") + ); + + await _repository.SaveAsync(aggregate, CancellationToken.None); + var updated = await _repository.GetAsync(aggregate.Id, CancellationToken.None); + + // Assert (update) + updated.Name.Should().Be("Updated Root"); + updated.Entities.Should().HaveCount(1); + updated.Entities.First().Description.Should().Be("Updated Entity"); + updated.Value.Number.Should().Be(200); + } + + [Fact] + public async Task FindAsync_ShouldReturnEntityIfExists() + { + // Arrange + var aggregate = TestAggregateRoot.Create("Find Test", new List(), new TestValueObject(50, "Find Test Value")); + await _repository.SaveAsync(aggregate, CancellationToken.None); + + // Act + var result = await _repository.FindAsync(aggregate.Id, CancellationToken.None); + + // Assert + result.Should().NotBeNull(); + result!.Id.Should().Be(aggregate.Id); + } + + [Fact] + public async Task FindAsync_ShouldReturnNullIfNotExists() + { + // Act + var result = await _repository.FindAsync(Guid.NewGuid(), CancellationToken.None); + + // Assert + result.Should().BeNull(); + } + + [Fact] + public async Task FindWithAsync_ShouldReturnFilteredResults() + { + // Arrange + var aggregate1 = TestAggregateRoot.Create("Filter Match", new List(), new TestValueObject(300, "Match")); + var aggregate2 = TestAggregateRoot.Create("No Match", new List(), new TestValueObject(400, "Different")); + await _repository.SaveAsync(aggregate1, CancellationToken.None); + await _repository.SaveAsync(aggregate2, CancellationToken.None); + + // Act + Expression> filter = a => a.Value.Number == 300; + var results = (await _repository.FindWithAsync(filter, CancellationToken.None)).ToList(); + + // Assert + results.Should().HaveCount(1); + results[0].Name.Should().Be("Filter Match"); + } + + [Fact] + public async Task FindAllAsync_ShouldReturnAllEntities() + { + // Arrange + var aggregate1 = TestAggregateRoot.Create("Entity 1", new List(), new TestValueObject(10, "VO 1")); + var aggregate2 = TestAggregateRoot.Create("Entity 2", new List(), new TestValueObject(20, "VO 2")); + await _repository.SaveAsync(aggregate1, CancellationToken.None); + await _repository.SaveAsync(aggregate2, CancellationToken.None); + + // Act + var results = (await _repository.FindAllAsync(CancellationToken.None)).ToList(); + + // Assert + results.Should().HaveCount(2); + } + + [Fact] + public async Task DeleteAsync_ShouldRemoveEntity() + { + // Arrange + var aggregate = TestAggregateRoot.Create("To be deleted", new List(), new TestValueObject(99, "Delete")); + await _repository.SaveAsync(aggregate, CancellationToken.None); + + // Act + await _repository.DeleteAsync(aggregate, CancellationToken.None); + var result = await _repository.FindAsync(aggregate.Id, CancellationToken.None); + + // Assert + result.Should().BeNull(); + } + } +} diff --git a/src/OpenDDD.Tests/OpenDDD.Tests.csproj b/src/OpenDDD.Tests/OpenDDD.Tests.csproj index 2070590..73390bb 100644 --- a/src/OpenDDD.Tests/OpenDDD.Tests.csproj +++ b/src/OpenDDD.Tests/OpenDDD.Tests.csproj @@ -15,6 +15,11 @@ + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + diff --git a/src/OpenDDD/Infrastructure/Persistence/EfCore/Base/OpenDddDbContextBase.cs b/src/OpenDDD/Infrastructure/Persistence/EfCore/Base/OpenDddDbContextBase.cs index d4f9061..5c0be94 100644 --- a/src/OpenDDD/Infrastructure/Persistence/EfCore/Base/OpenDddDbContextBase.cs +++ b/src/OpenDDD/Infrastructure/Persistence/EfCore/Base/OpenDddDbContextBase.cs @@ -46,7 +46,7 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) public void ApplyConfigurations(ModelBuilder modelBuilder) { - var configurationTypes = TypeScanner.GetRelevantTypes() + var configurationTypes = TypeScanner.GetRelevantTypes(excludeTestNamespaces: false) .Where(t => t.IsClass && !t.IsAbstract && t.BaseType != null && (t.BaseType.IsGenericType && diff --git a/src/OpenDDD/Infrastructure/Utils/TypeScanner.cs b/src/OpenDDD/Infrastructure/Utils/TypeScanner.cs index b326f67..231bc70 100644 --- a/src/OpenDDD/Infrastructure/Utils/TypeScanner.cs +++ b/src/OpenDDD/Infrastructure/Utils/TypeScanner.cs @@ -7,7 +7,7 @@ public static class TypeScanner private static readonly object _assemblyCacheLock = new(); private static IEnumerable? _cachedAssemblies; - public static IEnumerable GetRelevantAssemblies(bool includeDynamic = false) + public static IEnumerable GetRelevantAssemblies(bool includeDynamic = false, bool excludeTestNamespaces = true) { if (_cachedAssemblies == null) { @@ -20,7 +20,7 @@ public static IEnumerable GetRelevantAssemblies(bool includeDynamic = .Where(a => !a.FullName.StartsWith("System", StringComparison.OrdinalIgnoreCase) && !a.FullName.StartsWith("Microsoft", StringComparison.OrdinalIgnoreCase) && - !a.FullName.Contains("Tests", StringComparison.OrdinalIgnoreCase)) // Exclude test assemblies + (excludeTestNamespaces ? !a.FullName.Contains("Tests", StringComparison.OrdinalIgnoreCase) : true)) .ToList(); } } @@ -34,7 +34,7 @@ public static IEnumerable GetRelevantTypes( bool onlyConcreteClasses = false, bool excludeTestNamespaces = true) { - var types = GetRelevantAssemblies(includeDynamic) + var types = GetRelevantAssemblies(includeDynamic, excludeTestNamespaces) .SelectMany(assembly => assembly.GetTypes()); if (excludeTestNamespaces) From ea843f18a7ad14e1e63d5ddf1d8dfea2eaf1a239 Mon Sep 17 00:00:00 2001 From: David Runemalm Date: Tue, 4 Mar 2025 09:45:51 +0700 Subject: [PATCH 078/109] Rename test classes. --- .../EfCore/DbContext/Sqlite/SqliteTestDbContextFactory.cs | 1 - ...resRepositoryTests.cs => PostgresEfCoreRepositoryTests.cs} | 4 ++-- ...qliteRepositoryTests.cs => SqliteEfCoreRepositoryTests.cs} | 4 ++-- .../Repository/OpenDdd/Postgres/PostgresOpenDddRepository.cs | 2 +- 4 files changed, 5 insertions(+), 6 deletions(-) rename src/OpenDDD.Tests/Integration/Infrastructure/Repository/EfCore/Postgres/{EfCorePostgresRepositoryTests.cs => PostgresEfCoreRepositoryTests.cs} (98%) rename src/OpenDDD.Tests/Integration/Infrastructure/Repository/EfCore/Sqlite/{EfCoreSqliteRepositoryTests.cs => SqliteEfCoreRepositoryTests.cs} (98%) diff --git a/src/OpenDDD.Tests/Integration/Infrastructure/Persistence/EfCore/DbContext/Sqlite/SqliteTestDbContextFactory.cs b/src/OpenDDD.Tests/Integration/Infrastructure/Persistence/EfCore/DbContext/Sqlite/SqliteTestDbContextFactory.cs index 4f6c2e3..1f1e124 100644 --- a/src/OpenDDD.Tests/Integration/Infrastructure/Persistence/EfCore/DbContext/Sqlite/SqliteTestDbContextFactory.cs +++ b/src/OpenDDD.Tests/Integration/Infrastructure/Persistence/EfCore/DbContext/Sqlite/SqliteTestDbContextFactory.cs @@ -19,6 +19,5 @@ public SqliteTestDbContext CreateDbContext(string[] args) return new SqliteTestDbContext(optionsBuilder.Options, openDddOptions, logger); } - } } diff --git a/src/OpenDDD.Tests/Integration/Infrastructure/Repository/EfCore/Postgres/EfCorePostgresRepositoryTests.cs b/src/OpenDDD.Tests/Integration/Infrastructure/Repository/EfCore/Postgres/PostgresEfCoreRepositoryTests.cs similarity index 98% rename from src/OpenDDD.Tests/Integration/Infrastructure/Repository/EfCore/Postgres/EfCorePostgresRepositoryTests.cs rename to src/OpenDDD.Tests/Integration/Infrastructure/Repository/EfCore/Postgres/PostgresEfCoreRepositoryTests.cs index 7e1eb46..1ab4bf9 100644 --- a/src/OpenDDD.Tests/Integration/Infrastructure/Repository/EfCore/Postgres/EfCorePostgresRepositoryTests.cs +++ b/src/OpenDDD.Tests/Integration/Infrastructure/Repository/EfCore/Postgres/PostgresEfCoreRepositoryTests.cs @@ -20,7 +20,7 @@ namespace OpenDDD.Tests.Integration.Infrastructure.Repository.EfCore.Postgres { [Collection("EfCoreTests")] - public class EfCorePostgresRepositoryTests : IntegrationTests, IAsyncLifetime + public class PostgresEfCoreRepositoryTests : IntegrationTests, IAsyncLifetime { private readonly string _connectionString; private readonly EfCoreDatabaseSession _session; @@ -36,7 +36,7 @@ public class EfCorePostgresRepositoryTests : IntegrationTests, IAsyncLifetime private readonly ILoggerFactory _loggerFactory; private readonly ILogger _dbContextLogger; - public EfCorePostgresRepositoryTests(ITestOutputHelper testOutputHelper) + public PostgresEfCoreRepositoryTests(ITestOutputHelper testOutputHelper) : base(testOutputHelper, enableLogging: true) { _connectionString = Environment.GetEnvironmentVariable("POSTGRES_TEST_CONNECTION_STRING") diff --git a/src/OpenDDD.Tests/Integration/Infrastructure/Repository/EfCore/Sqlite/EfCoreSqliteRepositoryTests.cs b/src/OpenDDD.Tests/Integration/Infrastructure/Repository/EfCore/Sqlite/SqliteEfCoreRepositoryTests.cs similarity index 98% rename from src/OpenDDD.Tests/Integration/Infrastructure/Repository/EfCore/Sqlite/EfCoreSqliteRepositoryTests.cs rename to src/OpenDDD.Tests/Integration/Infrastructure/Repository/EfCore/Sqlite/SqliteEfCoreRepositoryTests.cs index ca2b550..e5dbe45 100644 --- a/src/OpenDDD.Tests/Integration/Infrastructure/Repository/EfCore/Sqlite/EfCoreSqliteRepositoryTests.cs +++ b/src/OpenDDD.Tests/Integration/Infrastructure/Repository/EfCore/Sqlite/SqliteEfCoreRepositoryTests.cs @@ -19,7 +19,7 @@ namespace OpenDDD.Tests.Integration.Infrastructure.Repository.EfCore.Sqlite { [Collection("EfCoreTests")] - public class EfCoreSqliteRepositoryTests : IntegrationTests, IAsyncLifetime + public class SqliteEfCoreRepositoryTests : IntegrationTests, IAsyncLifetime { private readonly EfCoreDatabaseSession _session; private readonly Mock _mockDomainPublisher; @@ -33,7 +33,7 @@ public class EfCoreSqliteRepositoryTests : IntegrationTests, IAsyncLifetime private readonly ILogger _dbContextLogger; private readonly string _connectionString = "DataSource=:memory:"; - public EfCoreSqliteRepositoryTests(ITestOutputHelper testOutputHelper) + public SqliteEfCoreRepositoryTests(ITestOutputHelper testOutputHelper) : base(testOutputHelper, enableLogging: true) { var options = new DbContextOptionsBuilder() diff --git a/src/OpenDDD/Infrastructure/Repository/OpenDdd/Postgres/PostgresOpenDddRepository.cs b/src/OpenDDD/Infrastructure/Repository/OpenDdd/Postgres/PostgresOpenDddRepository.cs index d152c85..aae13a7 100644 --- a/src/OpenDDD/Infrastructure/Repository/OpenDdd/Postgres/PostgresOpenDddRepository.cs +++ b/src/OpenDDD/Infrastructure/Repository/OpenDdd/Postgres/PostgresOpenDddRepository.cs @@ -22,7 +22,7 @@ public PostgresOpenDddRepository(PostgresDatabaseSession session, IAggregateSeri Session = session ?? throw new ArgumentNullException(nameof(session)); _tableName = typeof(TAggregateRoot).Name.ToLower().Pluralize(); } - + public override async Task GetAsync(TId id, CancellationToken ct) { var entity = await FindAsync(id, ct); From 566476015ca860dad2295763813364a3dda22df7 Mon Sep 17 00:00:00 2001 From: David Runemalm Date: Tue, 4 Mar 2025 12:28:07 +0700 Subject: [PATCH 079/109] Add consumer offset and re-delivery support to in-memory messaging provider. --- .../InMemory/InMemoryMessagingProvider.cs | 128 +++++++++++++++++- 1 file changed, 122 insertions(+), 6 deletions(-) diff --git a/src/OpenDDD/Infrastructure/Events/InMemory/InMemoryMessagingProvider.cs b/src/OpenDDD/Infrastructure/Events/InMemory/InMemoryMessagingProvider.cs index b2bda45..7552e97 100644 --- a/src/OpenDDD/Infrastructure/Events/InMemory/InMemoryMessagingProvider.cs +++ b/src/OpenDDD/Infrastructure/Events/InMemory/InMemoryMessagingProvider.cs @@ -3,42 +3,99 @@ namespace OpenDDD.Infrastructure.Events.InMemory { - public class InMemoryMessagingProvider : IMessagingProvider + public class InMemoryMessagingProvider : IMessagingProvider, IAsyncDisposable { + private readonly ConcurrentDictionary> _messageLog = new(); + private readonly ConcurrentDictionary _consumerOffsets = new(); private readonly ConcurrentDictionary>> _subscribers = new(); + private readonly ConcurrentQueue<(string Topic, string Message, string ConsumerGroup, int RetryCount)> _retryQueue = new(); private readonly ILogger _logger; + private readonly TimeSpan _initialRetryDelay = TimeSpan.FromSeconds(1); + private readonly Task _retryTask; + private readonly int _maxRetries = 5; + private readonly CancellationTokenSource _cts = new(); public InMemoryMessagingProvider(ILogger logger) { _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _retryTask = Task.Run(ProcessRetries, _cts.Token); // Start retry processing loop + } + + private static string GetGroupKey(string topic, string consumerGroup) + { + if (string.IsNullOrWhiteSpace(topic)) + throw new ArgumentException("Topic cannot be null or empty.", nameof(topic)); + + if (string.IsNullOrWhiteSpace(consumerGroup)) + throw new ArgumentException("Consumer group cannot be null or empty.", nameof(consumerGroup)); + + return $"{topic}:{consumerGroup}"; } public Task SubscribeAsync(string topic, string consumerGroup, Func messageHandler, CancellationToken ct) { - var groupKey = $"{topic}:{consumerGroup}"; + if (messageHandler is null) + throw new ArgumentNullException(nameof(messageHandler)); + var groupKey = GetGroupKey(topic, consumerGroup); var handlers = _subscribers.GetOrAdd(groupKey, _ => new ConcurrentBag>()); handlers.Add(messageHandler); + _logger.LogDebug("Subscribed to topic: {Topic} in listener group: {ConsumerGroup}", topic, consumerGroup); + + if (!_consumerOffsets.ContainsKey(groupKey)) + { + var messageCount = _messageLog.TryGetValue(topic, out var messages) ? messages.Count : 0; + _consumerOffsets[groupKey] = messageCount; + _logger.LogDebug("Consumer group '{ConsumerGroup}' is subscribing for the first time, starting at offset {Offset}.", consumerGroup, messageCount); + return Task.CompletedTask; + } + + if (_messageLog.TryGetValue(topic, out var storedMessages)) + { + var offset = _consumerOffsets[groupKey]; + var unseenMessages = storedMessages.Skip(offset).ToList(); + + foreach (var msg in unseenMessages) + { + _ = Task.Run(async () => await messageHandler(msg, ct), ct); + _consumerOffsets[groupKey]++; + } + } - _logger.LogInformation("Subscribed to topic: {Topic} in listener group: {ConsumerGroup}", topic, consumerGroup); return Task.CompletedTask; } public Task UnsubscribeAsync(string topic, string consumerGroup, CancellationToken cancellationToken = default) { - throw new NotImplementedException(); + var groupKey = GetGroupKey(topic, consumerGroup); + + if (_subscribers.TryRemove(groupKey, out _)) + { + _logger.LogDebug("Unsubscribed from topic: {Topic} in listener group: {ConsumerGroup}", topic, consumerGroup); + } + else + { + _logger.LogWarning("No active subscriptions found for topic: {Topic} in listener group: {ConsumerGroup}", topic, consumerGroup); + } + + return Task.CompletedTask; } public Task PublishAsync(string topic, string message, CancellationToken ct) { - var matchingGroups = _subscribers.Keys.Where(key => key.StartsWith($"{topic}:")); + var messages = _messageLog.GetOrAdd(topic, _ => new List()); + lock (messages) + { + messages.Add(message); + } + + var matchingGroups = _subscribers.Keys.Where(key => key.StartsWith($"{topic}:")).ToList(); foreach (var groupKey in matchingGroups) { if (_subscribers.TryGetValue(groupKey, out var handlers) && handlers.Any()) { - // Select one handler at random to simulate competing consumers var handler = handlers.OrderBy(_ => Guid.NewGuid()).First(); _ = Task.Run(async () => @@ -46,10 +103,12 @@ public Task PublishAsync(string topic, string message, CancellationToken ct) try { await handler(message, ct); + _consumerOffsets.AddOrUpdate(groupKey, 0, (_, currentOffset) => currentOffset + 1); // Update offset } catch (Exception ex) { _logger.LogError(ex, $"Error in handler for topic '{topic}': {ex.Message}"); + _retryQueue.Enqueue((topic, message, groupKey.Split(':')[1], 1)); } }, ct); } @@ -57,5 +116,62 @@ public Task PublishAsync(string topic, string message, CancellationToken ct) return Task.CompletedTask; } + + private async Task ProcessRetries() + { + while (!_cts.Token.IsCancellationRequested) + { + if (_retryQueue.TryDequeue(out var retryMessage)) + { + var (topic, message, consumerGroup, retryCount) = retryMessage; + + if (retryCount > _maxRetries) + { + _logger.LogError("Message dropped after exceeding max retries: {Message}", message); + continue; + } + + var groupKey = GetGroupKey(topic, consumerGroup); + if (_subscribers.TryGetValue(groupKey, out var handlers) && handlers.Any()) + { + var handler = handlers.OrderBy(_ => Guid.NewGuid()).First(); + + await Task.Delay(ComputeBackoff(retryCount), _cts.Token); // Exponential backoff + + try + { + await handler(message, _cts.Token); + } + catch (Exception ex) + { + _logger.LogError(ex, $"Retry failed for message: {message}"); + _retryQueue.Enqueue((topic, message, consumerGroup, retryCount + 1)); + } + } + } + else + { + await Task.Delay(500, _cts.Token); + } + } + } + + private TimeSpan ComputeBackoff(int retryCount) + { + return TimeSpan.FromMilliseconds(_initialRetryDelay.TotalMilliseconds * Math.Pow(2, retryCount)); + } + + public async ValueTask DisposeAsync() + { + _cts.Cancel(); + try + { + await _retryTask; + } + catch (OperationCanceledException) + { + _logger.LogDebug("Retry processing task canceled."); + } + } } } From ed55ee41a9b0dc558366bcce3b27c5c81bdc4850 Mon Sep 17 00:00:00 2001 From: David Runemalm Date: Tue, 4 Mar 2025 12:28:20 +0700 Subject: [PATCH 080/109] Add integration tests for in-memory messaging provider. --- .../InMemoryMessagingProviderTests.cs | 240 ++++++++++++++++++ .../InMemory/InMemoryTestsCollection.cs | 5 + 2 files changed, 245 insertions(+) create mode 100644 src/OpenDDD.Tests/Integration/Infrastructure/Events/InMemory/InMemoryMessagingProviderTests.cs create mode 100644 src/OpenDDD.Tests/Integration/Infrastructure/Events/InMemory/InMemoryTestsCollection.cs diff --git a/src/OpenDDD.Tests/Integration/Infrastructure/Events/InMemory/InMemoryMessagingProviderTests.cs b/src/OpenDDD.Tests/Integration/Infrastructure/Events/InMemory/InMemoryMessagingProviderTests.cs new file mode 100644 index 0000000..d47275b --- /dev/null +++ b/src/OpenDDD.Tests/Integration/Infrastructure/Events/InMemory/InMemoryMessagingProviderTests.cs @@ -0,0 +1,240 @@ +using System.Collections.Concurrent; +using Microsoft.Extensions.Logging; +using Xunit.Abstractions; +using FluentAssertions; +using OpenDDD.Infrastructure.Events.InMemory; +using OpenDDD.Tests.Base; + +namespace OpenDDD.Tests.Integration.Infrastructure.Events.InMemory +{ + [Collection("InMemoryTests")] + public class InMemoryMessagingProviderTests : IntegrationTests, IAsyncLifetime + { + private readonly ILogger _logger; + private readonly InMemoryMessagingProvider _messagingProvider; + private readonly CancellationTokenSource _cts = new(TimeSpan.FromSeconds(60)); + + public InMemoryMessagingProviderTests(ITestOutputHelper testOutputHelper) + : base(testOutputHelper, enableLogging: true) + { + _logger = LoggerFactory.CreateLogger(); + _messagingProvider = new InMemoryMessagingProvider(_logger); + } + + public Task InitializeAsync() => Task.CompletedTask; + + public async Task DisposeAsync() + { + _cts.Cancel(); + await _messagingProvider.DisposeAsync(); + } + + [Fact] + public async Task AtLeastOnceGuarantee_ShouldDeliverToLateSubscriber_WhenSubscribedBefore() + { + // Arrange + var topicName = $"test-topic-{Guid.NewGuid()}"; + var groupName = "test-subscription"; + var receivedMessages = new ConcurrentBag(); + var messageToSend = "Persistent Message Test"; + var cts = new CancellationTokenSource(TimeSpan.FromSeconds(30)); + + var lateSubscriberReceived = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + + await _messagingProvider.SubscribeAsync(topicName, groupName, async (msg, token) => + { + Assert.Fail("First subscription should not receive the message."); + }, cts.Token); + + await Task.Delay(500); + + await _messagingProvider.UnsubscribeAsync(topicName, groupName, cts.Token); + + // Act + await _messagingProvider.PublishAsync(topicName, messageToSend, cts.Token); + + await _messagingProvider.SubscribeAsync(topicName, groupName, async (msg, token) => + { + receivedMessages.Add(msg); + lateSubscriberReceived.TrySetResult(true); + }, cts.Token); + + // Assert + try + { + await lateSubscriberReceived.Task.WaitAsync(TimeSpan.FromSeconds(5)); + } + catch (TimeoutException) + { + Assert.Fail($"Late subscriber did not receive the expected message '{messageToSend}' within 5 seconds."); + } + + receivedMessages.Should().Contain(messageToSend, "The subscriber should receive messages published while it was previously subscribed."); + } + + [Fact] + public async Task AtLeastOnceGuarantee_ShouldNotDeliverToLateSubscriber_WhenNotSubscribedBefore() + { + // Arrange + var topicName = "test-topic-no-late-subscriber"; + var consumerGroup = "test-consumer-group"; + var messageToSend = "Non-Persistent Message Test"; + ConcurrentBag _receivedMessages = new(); + + // Act + await _messagingProvider.PublishAsync(topicName, messageToSend, _cts.Token); + await Task.Delay(500); + + // Late subscriber + await _messagingProvider.SubscribeAsync(topicName, consumerGroup, async (msg, token) => + { + _receivedMessages.Add(msg); + }, _cts.Token); + + await Task.Delay(2000); + + // Assert + _receivedMessages.Should().NotContain(messageToSend); + } + + [Fact] + public async Task AtLeastOnceGuarantee_ShouldRedeliverLater_WhenMessageNotAcked() + { + // Arrange + var topicName = "test-topic-redelivery"; + var consumerGroup = "test-consumer-group"; + var messageToSend = "Redelivery Test"; + ConcurrentBag _receivedMessages = new(); + + async Task FaultyHandler(string msg, CancellationToken token) + { + _receivedMessages.Add(msg); + throw new Exception("Simulated consumer crash before acknowledgment."); + } + + await _messagingProvider.SubscribeAsync(topicName, consumerGroup, FaultyHandler, _cts.Token); + + // Act + await _messagingProvider.PublishAsync(topicName, messageToSend, _cts.Token); + + for (int i = 0; i < 20; i++) + { + if (_receivedMessages.Count > 1) break; + await Task.Delay(500); + } + + // Assert + _receivedMessages.Count.Should().BeGreaterThan(1); + } + + [Fact] + public async Task CompetingConsumers_ShouldDeliverOnlyOnce_WhenMultipleConsumersInGroup() + { + // Arrange + var topicName = "test-topic-competing-consumers"; + var consumerGroup = "test-consumer-group"; + var receivedMessages = new ConcurrentDictionary(); + var messageToSend = "Competing Consumer Test"; + var allSubscribersProcessed = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + + async Task MessageHandler(string msg, CancellationToken token) + { + receivedMessages.AddOrUpdate("received", 1, (key, value) => value + 1); + + // If any consumer has received more than 1 message, fail immediately + if (receivedMessages["received"] > 1) + { + allSubscribersProcessed.TrySetException(new Exception("More than one consumer in the group received the message!")); + } + else + { + allSubscribersProcessed.TrySetResult(true); + } + } + + // Subscribe multiple consumers in the same group + await _messagingProvider.SubscribeAsync(topicName, consumerGroup, MessageHandler, _cts.Token); + await _messagingProvider.SubscribeAsync(topicName, consumerGroup, MessageHandler, _cts.Token); + + // Act + await _messagingProvider.PublishAsync(topicName, messageToSend, _cts.Token); + + try + { + await allSubscribersProcessed.Task.WaitAsync(TimeSpan.FromSeconds(5)); + } + catch (TimeoutException) + { + Assert.Fail("Timed out waiting for message processing."); + } + + // Assert: Only one consumer in the group should receive the message + receivedMessages.GetValueOrDefault("received", 0).Should().Be(1); + } + + [Fact] + public async Task CompetingConsumers_ShouldDistributeEvenly_WhenMultipleConsumersInGroup() + { + // Arrange + var topicName = "test-topic-even-distribution"; + var consumerGroup = "test-consumer-group"; + var totalMessages = 100; + var numConsumers = 2; + var variancePercentage = 0.1; // Allow 10% variance + var perConsumerMessageCount = new ConcurrentDictionary(); + var allMessagesProcessed = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + + async Task CreateConsumer() + { + var consumerId = Guid.NewGuid(); + + async Task MessageHandler(string msg, CancellationToken token) + { + perConsumerMessageCount.AddOrUpdate(consumerId, 1, (_, count) => count + 1); + + _logger.LogDebug($"Consumer {consumerId} received a message."); + + if (perConsumerMessageCount.Values.Sum() >= totalMessages) + { + allMessagesProcessed.TrySetResult(true); + } + } + + await _messagingProvider.SubscribeAsync(topicName, consumerGroup, MessageHandler, _cts.Token); + } + + for (int i = 0; i < numConsumers; i++) + { + await CreateConsumer(); + } + + // Act + for (int i = 0; i < totalMessages; i++) + { + await _messagingProvider.PublishAsync(topicName, "Test Message", _cts.Token); + } + + try + { + await allMessagesProcessed.Task.WaitAsync(TimeSpan.FromSeconds(10)); + } + catch (TimeoutException) + { + _logger.LogDebug("Timed out waiting for consumers to receive all messages."); + Assert.Fail($"Consumers only processed {perConsumerMessageCount.Values.Sum()} of {totalMessages} messages."); + } + + // Assert + var messageCounts = perConsumerMessageCount.Values.ToList(); + var expectedPerConsumer = totalMessages / numConsumers; + var variance = (int)(expectedPerConsumer * variancePercentage); + var minAllowed = expectedPerConsumer - variance; + var maxAllowed = expectedPerConsumer + variance; + + foreach (var count in messageCounts) + { + Assert.InRange(count, minAllowed, maxAllowed); + } + } + } +} diff --git a/src/OpenDDD.Tests/Integration/Infrastructure/Events/InMemory/InMemoryTestsCollection.cs b/src/OpenDDD.Tests/Integration/Infrastructure/Events/InMemory/InMemoryTestsCollection.cs new file mode 100644 index 0000000..4f0e579 --- /dev/null +++ b/src/OpenDDD.Tests/Integration/Infrastructure/Events/InMemory/InMemoryTestsCollection.cs @@ -0,0 +1,5 @@ +namespace OpenDDD.Tests.Integration.Infrastructure.Events.InMemory +{ + [CollectionDefinition("InMemoryTests", DisableParallelization = true)] + public class InMemoryTestsCollection { } +} From bcb49c41f475a2035ccfd08e29fde40ff5ccc96b Mon Sep 17 00:00:00 2001 From: David Runemalm Date: Tue, 4 Mar 2025 16:58:47 +0700 Subject: [PATCH 081/109] Fix issue in kafka messaging provider test. --- .../Kafka/KafkaMessagingProviderTests.cs | 126 +++++++++++------- .../Events/Kafka/KafkaMessagingProvider.cs | 40 +++--- 2 files changed, 104 insertions(+), 62 deletions(-) diff --git a/src/OpenDDD.Tests/Integration/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs b/src/OpenDDD.Tests/Integration/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs index 38f5d79..8f7b6d1 100644 --- a/src/OpenDDD.Tests/Integration/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs +++ b/src/OpenDDD.Tests/Integration/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs @@ -172,17 +172,23 @@ await _messagingProvider.SubscribeAsync(topicName, consumerGroup, async (msg, to await firstSubscriberReceived.Task.WaitAsync(TimeSpan.FromSeconds(10)); await _messagingProvider.UnsubscribeAsync(topicName, consumerGroup, _cts.Token); + + await Task.Delay(5000); // Late subscriber + await _messagingProvider.PublishAsync(topicName, messageToSend, _cts.Token); + + await Task.Delay(5000); + await _messagingProvider.SubscribeAsync(topicName, consumerGroup, async (msg, token) => { _receivedMessages.Add(msg); lateSubscriberReceived.TrySetResult(true); }, _cts.Token); - await _messagingProvider.PublishAsync(topicName, messageToSend, _cts.Token); + await WaitForKafkaConsumerGroupStable(consumerGroup, _cts.Token); - await lateSubscriberReceived.Task.WaitAsync(TimeSpan.FromSeconds(10)); + await lateSubscriberReceived.Task.WaitAsync(TimeSpan.FromSeconds(30)); // Assert _receivedMessages.Should().Contain(messageToSend); @@ -209,7 +215,7 @@ await _messagingProvider.SubscribeAsync(topicName, consumerGroup, async (msg, to await WaitForKafkaConsumerGroupStable(consumerGroup, _cts.Token); - await Task.Delay(1000); + await Task.Delay(5000); // Assert _receivedMessages.Should().NotContain(messageToSend); @@ -231,7 +237,7 @@ async Task FaultyHandler(string msg, CancellationToken token) } await _messagingProvider.SubscribeAsync(topicName, consumerGroup, FaultyHandler, _cts.Token); - + await WaitForKafkaConsumerGroupStable(consumerGroup, _cts.Token); // Act @@ -253,87 +259,117 @@ public async Task CompetingConsumers_ShouldDeliverOnlyOnce_WhenMultipleConsumers // Arrange var topicName = $"test-topic-{Guid.NewGuid()}"; var consumerGroup = "test-consumer-group"; - var receivedMessages = new ConcurrentDictionary(); + var receivedMessages = new ConcurrentDictionary(); // Track messages per consumer var messageToSend = "Competing Consumer Test"; - var subscriberReceived = new TaskCompletionSource(); - + var messageProcessed = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var consumerIds = new ConcurrentBag(); + async Task MessageHandler(string msg, CancellationToken token) { - receivedMessages.AddOrUpdate("received", 1, (key, value) => value + 1); - subscriberReceived.SetResult(true); + var consumerId = Guid.NewGuid(); + + receivedMessages.AddOrUpdate(consumerId, 1, (_, count) => count + 1); + consumerIds.Add(consumerId); + + _logger.LogDebug($"Consumer {consumerId} received a message."); + + messageProcessed.TrySetResult(true); } - + + // Subscribe multiple consumers in the same group await _messagingProvider.SubscribeAsync(topicName, consumerGroup, MessageHandler, _cts.Token); await _messagingProvider.SubscribeAsync(topicName, consumerGroup, MessageHandler, _cts.Token); - + await WaitForKafkaConsumerGroupStable(consumerGroup, _cts.Token); - + // Act await _messagingProvider.PublishAsync(topicName, messageToSend, _cts.Token); - + try { - await subscriberReceived.Task.WaitAsync(TimeSpan.FromSeconds(20)); + await messageProcessed.Task.WaitAsync(TimeSpan.FromSeconds(5)); } catch (TimeoutException) { - _logger.LogDebug("Timed out waiting for subscriber to receive message."); + _logger.LogDebug("Timed out waiting for consumer to receive the message."); + Assert.Fail("No consumer received the message."); } - - // Assert - receivedMessages.GetValueOrDefault("received", 0).Should().Be(1); + + // Wait a little longer to ensure no second consumer processes the same message + await Task.Delay(5000); + + // Assert: Exactly one consumer should have received the message + receivedMessages.Count.Should().Be(1, + $"Expected only one consumer to receive the message, but {receivedMessages.Count} consumers received it."); } - [Fact] + [Fact(Skip = "Skipping this test temporarily due to not working with > 1 partitions.")] public async Task CompetingConsumers_ShouldDistributeEvenly_WhenMultipleConsumersInGroup() { // Arrange var topicName = $"test-topic-{Guid.NewGuid()}"; var consumerGroup = "test-consumer-group"; - var receivedMessages = new ConcurrentDictionary(); var totalMessages = 10; - var allMessagesReceived = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - - async Task MessageHandler(string msg, CancellationToken token) + var numConsumers = 10; + var variancePercentage = 0.1; // Allow 10% variance + var perConsumerMessageCount = new ConcurrentDictionary(); // Track messages per consumer + var allMessagesProcessed = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + + async Task CreateConsumer() { - _logger.LogDebug("Received a message."); + var consumerId = Guid.NewGuid(); - receivedMessages.AddOrUpdate(msg, 1, (key, value) => value + 1); - - if (receivedMessages.Count >= totalMessages) + async Task MessageHandler(string msg, CancellationToken token) { - allMessagesReceived.TrySetResult(true); + perConsumerMessageCount.AddOrUpdate(consumerId, 1, (_, count) => count + 1); + _logger.LogDebug($"Consumer {consumerId} received a message."); + + if (perConsumerMessageCount.Values.Sum() >= totalMessages) + { + allMessagesProcessed.TrySetResult(true); + } } + + await _messagingProvider.SubscribeAsync(topicName, consumerGroup, MessageHandler, _cts.Token); } - - await _messagingProvider.SubscribeAsync(topicName, consumerGroup, MessageHandler, _cts.Token); - await _messagingProvider.SubscribeAsync(topicName, consumerGroup, MessageHandler, _cts.Token); - + + // Subscribe multiple consumers + for (int i = 0; i < numConsumers; i++) + { + await CreateConsumer(); + } + await WaitForKafkaConsumerGroupStable(consumerGroup, _cts.Token); + + await Task.Delay(2000); // Act for (int i = 0; i < totalMessages; i++) { - await _messagingProvider.PublishAsync(topicName, $"Message {i}", _cts.Token); + await _messagingProvider.PublishAsync(topicName, "Test Message", _cts.Token); } - + try { - await allMessagesReceived.Task.WaitAsync(TimeSpan.FromSeconds(10)); + await allMessagesProcessed.Task.WaitAsync(TimeSpan.FromSeconds(10)); } catch (TimeoutException) { - _logger.LogDebug("Timed out waiting for subscriber to receive all messages."); - Assert.Fail($"Consumers only received {receivedMessages.Count} of {totalMessages} messages."); + _logger.LogDebug("Timed out waiting for consumers to receive all messages."); + Assert.Fail($"Consumers only processed {perConsumerMessageCount.Values.Sum()} of {totalMessages} messages."); + } + + // Assert + var messageCounts = perConsumerMessageCount.Values.ToList(); + var expectedPerConsumer = totalMessages / numConsumers; + var variance = (int)(expectedPerConsumer * variancePercentage); + var minAllowed = expectedPerConsumer - variance; + var maxAllowed = expectedPerConsumer + variance; + + foreach (var count in messageCounts) + { + Assert.InRange(count, minAllowed, maxAllowed); } - - // Assert: Messages should be evenly distributed across consumers - var messageCounts = receivedMessages.Values; - var minReceived = messageCounts.Any() ? messageCounts.Min() : 0; - var maxReceived = messageCounts.Any() ? messageCounts.Max() : 0; - - Assert.True(maxReceived - minReceived <= 1, - "Messages should be evenly distributed among competing consumers."); } private async Task WaitForKafkaConsumerGroupStable(string consumerGroup, CancellationToken cancellationToken) diff --git a/src/OpenDDD/Infrastructure/Events/Kafka/KafkaMessagingProvider.cs b/src/OpenDDD/Infrastructure/Events/Kafka/KafkaMessagingProvider.cs index dbac38b..aced766 100644 --- a/src/OpenDDD/Infrastructure/Events/Kafka/KafkaMessagingProvider.cs +++ b/src/OpenDDD/Infrastructure/Events/Kafka/KafkaMessagingProvider.cs @@ -37,11 +37,7 @@ public KafkaMessagingProvider( _logger = logger ?? throw new ArgumentNullException(nameof(logger)); } - public async Task SubscribeAsync( - string topic, - string consumerGroup, - Func messageHandler, - CancellationToken cancellationToken) + private static string GetGroupKey(string topic, string consumerGroup) { if (string.IsNullOrWhiteSpace(topic)) throw new ArgumentException("Topic cannot be null or empty.", nameof(topic)); @@ -49,9 +45,20 @@ public async Task SubscribeAsync( if (string.IsNullOrWhiteSpace(consumerGroup)) throw new ArgumentException("Consumer group cannot be null or empty.", nameof(consumerGroup)); + return $"{topic}:{consumerGroup}"; + } + + public async Task SubscribeAsync( + string topic, + string consumerGroup, + Func messageHandler, + CancellationToken cancellationToken) + { if (messageHandler is null) throw new ArgumentNullException(nameof(messageHandler)); - + + var groupKey = GetGroupKey(topic, consumerGroup); + if (_autoCreateTopics) { await CreateTopicIfNotExistsAsync(topic, cancellationToken); @@ -66,9 +73,12 @@ public async Task SubscribeAsync( } } - var kafkaConsumer = _consumers.GetOrAdd(consumerGroup, _ => _consumerFactory.Create(consumerGroup)); - - kafkaConsumer.Subscribe(topic); + var kafkaConsumer = _consumers.GetOrAdd(groupKey, _ => + { + var newConsumer = _consumerFactory.Create(consumerGroup); + newConsumer.Subscribe(topic); + return newConsumer; + }); _logger.LogDebug("Subscribed to Kafka topic '{Topic}' with consumer group '{ConsumerGroup}'", topic, consumerGroup); @@ -77,19 +87,15 @@ public async Task SubscribeAsync( public async Task UnsubscribeAsync(string topic, string consumerGroup, CancellationToken cancellationToken) { - if (string.IsNullOrWhiteSpace(topic)) - throw new ArgumentException("Topic cannot be null or empty.", nameof(topic)); + var groupKey = GetGroupKey(topic, consumerGroup); - if (string.IsNullOrWhiteSpace(consumerGroup)) - throw new ArgumentException("Consumer group cannot be null or empty.", nameof(consumerGroup)); - - if (!_consumers.TryRemove(consumerGroup, out var kafkaConsumer)) + if (!_consumers.TryRemove(groupKey, out var kafkaConsumer)) { - _logger.LogWarning("No active consumer found for consumer group '{ConsumerGroup}'. It may have already stopped.", consumerGroup); + _logger.LogWarning("No active consumer found for consumer group '{ConsumerGroup}' on topic '{Topic}'. It may have already stopped.", consumerGroup, topic); return; } - _logger.LogInformation("Stopping consumer group '{ConsumerGroup}'...", consumerGroup); + _logger.LogDebug("Stopping consumer for topic '{Topic}' and consumer group '{ConsumerGroup}'...", topic, consumerGroup); await kafkaConsumer.StopProcessingAsync(); kafkaConsumer.Dispose(); } From 790e544a96baa461eae4f041c13700b8a6fe8461 Mon Sep 17 00:00:00 2001 From: David Runemalm Date: Tue, 4 Mar 2025 17:05:07 +0700 Subject: [PATCH 082/109] Refactor in-memory messaging provider unit tests. --- .../InMemoryMessagingProviderTests.cs | 197 ++++-------------- .../InMemory/InMemoryMessagingProvider.cs | 6 + 2 files changed, 49 insertions(+), 154 deletions(-) diff --git a/src/OpenDDD.Tests/Infrastructure/Events/InMemory/InMemoryMessagingProviderTests.cs b/src/OpenDDD.Tests/Infrastructure/Events/InMemory/InMemoryMessagingProviderTests.cs index 4619954..5d267b5 100644 --- a/src/OpenDDD.Tests/Infrastructure/Events/InMemory/InMemoryMessagingProviderTests.cs +++ b/src/OpenDDD.Tests/Infrastructure/Events/InMemory/InMemoryMessagingProviderTests.cs @@ -1,7 +1,5 @@ using Microsoft.Extensions.Logging; -using FluentAssertions; using Moq; -using Xunit; using OpenDDD.Infrastructure.Events.InMemory; using OpenDDD.Tests.Base; @@ -11,6 +9,9 @@ public class InMemoryMessagingProviderTests : UnitTests { private readonly Mock> _mockLogger; private readonly InMemoryMessagingProvider _messagingProvider; + private const string Topic = "TestTopic"; + private const string ConsumerGroup = "TestGroup"; + private const string Message = "Hello, InMemory!"; public InMemoryMessagingProviderTests() { @@ -18,174 +19,62 @@ public InMemoryMessagingProviderTests() _messagingProvider = new InMemoryMessagingProvider(_mockLogger.Object); } + // Constructor validation tests [Fact] - public async Task SubscribeAsync_ShouldStoreMessageHandler_ForGivenTopicAndConsumerGroup() + public void Constructor_ShouldThrowException_WhenLoggerIsNull() { - // Arrange - var topic = "TestTopic"; - var consumerGroup = "TestGroup"; - Func handler = async (message, ct) => await Task.CompletedTask; - - // Act - await _messagingProvider.SubscribeAsync(topic, consumerGroup, handler, CancellationToken.None); - - // Assert - _mockLogger.Verify( - log => log.Log( - LogLevel.Information, - It.IsAny(), - It.Is((v, t) => v.ToString().Contains($"Subscribed to topic: {topic} in listener group: {consumerGroup}")), - It.IsAny(), - It.IsAny>() - ), - Times.Once - ); + Assert.Throws(() => new InMemoryMessagingProvider(null!)); } - [Fact] - public async Task PublishAsync_ShouldInvokeSubscribedHandler() + // SubscribeAsync validation tests + [Theory] + [InlineData(null)] + [InlineData("")] + [InlineData(" ")] + public async Task SubscribeAsync_ShouldThrowException_WhenTopicIsInvalid(string invalidTopic) { - // Arrange - var topic = "TestTopic"; - var consumerGroup = "TestGroup"; - var receivedMessages = new List(); - - Func handler = async (message, ct) => - { - receivedMessages.Add(message); - await Task.CompletedTask; - }; - - await _messagingProvider.SubscribeAsync(topic, consumerGroup, handler, CancellationToken.None); - - // Act - await _messagingProvider.PublishAsync(topic, "Hello, World!", CancellationToken.None); - - // Allow some time for async handlers to execute - await Task.Delay(100); - - // Assert - receivedMessages.Should().ContainSingle() - .Which.Should().Be("Hello, World!"); + await Assert.ThrowsAsync(() => + _messagingProvider.SubscribeAsync(invalidTopic, ConsumerGroup, (msg, token) => Task.CompletedTask, + CancellationToken.None)); } - [Fact] - public async Task PublishAsync_ShouldNotThrow_WhenNoSubscribersExist() + [Theory] + [InlineData(null)] + [InlineData("")] + [InlineData(" ")] + public async Task SubscribeAsync_ShouldThrowException_WhenConsumerGroupIsInvalid(string invalidConsumerGroup) { - // Arrange - var topic = "NonExistentTopic"; - - // Act - Func act = async () => await _messagingProvider.PublishAsync(topic, "Message", CancellationToken.None); - - // Assert - await act.Should().NotThrowAsync(); + await Assert.ThrowsAsync(() => + _messagingProvider.SubscribeAsync(Topic, invalidConsumerGroup, (msg, token) => Task.CompletedTask, + CancellationToken.None)); } [Fact] - public async Task PublishAsync_ShouldInvokeAllHandlers_ForMultipleSubscriptions() + public async Task SubscribeAsync_ShouldThrowException_WhenHandlerIsNull() { - // Arrange - var topic = "TestTopic"; - var consumerGroup1 = "Group1"; - var consumerGroup2 = "Group2"; - var receivedMessages1 = new List(); - var receivedMessages2 = new List(); - - Func handler1 = async (message, ct) => - { - receivedMessages1.Add(message); - await Task.CompletedTask; - }; - - Func handler2 = async (message, ct) => - { - receivedMessages2.Add(message); - await Task.CompletedTask; - }; - - await _messagingProvider.SubscribeAsync(topic, consumerGroup1, handler1, CancellationToken.None); - await _messagingProvider.SubscribeAsync(topic, consumerGroup2, handler2, CancellationToken.None); - - // Act - await _messagingProvider.PublishAsync(topic, "Event 1", CancellationToken.None); - - // Allow time for async handlers - await Task.Delay(100); - - // Assert - receivedMessages1.Should().ContainSingle().Which.Should().Be("Event 1"); - receivedMessages2.Should().ContainSingle().Which.Should().Be("Event 1"); + await Assert.ThrowsAsync(() => + _messagingProvider.SubscribeAsync(Topic, ConsumerGroup, null!, CancellationToken.None)); } - - [Fact] - public async Task PublishAsync_ShouldDeliverMessageToOnlyOneConsumer_InCompetingConsumerGroup() - { - // Arrange - var topic = "TestTopic"; - var consumerGroup = "CompetingGroup"; - var receivedMessages1 = new List(); - var receivedMessages2 = new List(); - - Func handler1 = async (message, ct) => - { - receivedMessages1.Add(message); - await Task.CompletedTask; - }; - - Func handler2 = async (message, ct) => - { - receivedMessages2.Add(message); - await Task.CompletedTask; - }; - - // Two consumers in the same group - await _messagingProvider.SubscribeAsync(topic, consumerGroup, handler1, CancellationToken.None); - await _messagingProvider.SubscribeAsync(topic, consumerGroup, handler2, CancellationToken.None); - - // Act - await _messagingProvider.PublishAsync(topic, "Competing Message", CancellationToken.None); - - // Allow time for async handlers - await Task.Delay(100); - // Assert: Only one of the consumers should receive the message - var totalMessagesReceived = receivedMessages1.Count + receivedMessages2.Count; - totalMessagesReceived.Should().Be(1); + // PublishAsync validation tests + [Theory] + [InlineData(null)] + [InlineData("")] + [InlineData(" ")] + public async Task PublishAsync_ShouldThrowException_WhenTopicIsInvalid(string invalidTopic) + { + await Assert.ThrowsAsync(() => + _messagingProvider.PublishAsync(invalidTopic, Message, CancellationToken.None)); } - [Fact] - public async Task PublishAsync_ShouldHandleException_WhenHandlerThrows() + [Theory] + [InlineData(null)] + [InlineData("")] + [InlineData(" ")] + public async Task PublishAsync_ShouldThrowException_WhenMessageIsInvalid(string invalidMessage) { - // Arrange - var topic = "TestTopic"; - var consumerGroup = "TestGroup"; - - Func failingHandler = async (message, ct) => - { - await Task.CompletedTask; - throw new InvalidOperationException("Handler error"); - }; - - await _messagingProvider.SubscribeAsync(topic, consumerGroup, failingHandler, CancellationToken.None); - - // Act - await _messagingProvider.PublishAsync(topic, "Test Message", CancellationToken.None); - - // Allow some time for async handlers to execute - await Task.Delay(100); - - // Assert - _mockLogger.Verify( - log => log.Log( - LogLevel.Error, - It.IsAny(), - It.Is((v, t) => v.ToString().Contains($"Error in handler for topic '{topic}': Handler error")), - It.IsAny(), - It.IsAny>() - ), - Times.Once - ); + await Assert.ThrowsAsync(() => + _messagingProvider.PublishAsync(Topic, invalidMessage, CancellationToken.None)); } } -} +} \ No newline at end of file diff --git a/src/OpenDDD/Infrastructure/Events/InMemory/InMemoryMessagingProvider.cs b/src/OpenDDD/Infrastructure/Events/InMemory/InMemoryMessagingProvider.cs index 7552e97..a912304 100644 --- a/src/OpenDDD/Infrastructure/Events/InMemory/InMemoryMessagingProvider.cs +++ b/src/OpenDDD/Infrastructure/Events/InMemory/InMemoryMessagingProvider.cs @@ -84,6 +84,12 @@ public Task UnsubscribeAsync(string topic, string consumerGroup, CancellationTok public Task PublishAsync(string topic, string message, CancellationToken ct) { + if (string.IsNullOrWhiteSpace(topic)) + throw new ArgumentException("Topic cannot be null or empty.", nameof(topic)); + + if (string.IsNullOrWhiteSpace(message)) + throw new ArgumentException("Message cannot be null or empty.", nameof(message)); + var messages = _messageLog.GetOrAdd(topic, _ => new List()); lock (messages) { From bf4de0188ec051f3e7ca2743a9f0888ab0d5f71c Mon Sep 17 00:00:00 2001 From: David Runemalm Date: Tue, 4 Mar 2025 17:37:52 +0700 Subject: [PATCH 083/109] Fix issue in in-memory test. --- .../Events/InMemory/InMemoryMessagingProviderTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/OpenDDD.Tests/Integration/Infrastructure/Events/InMemory/InMemoryMessagingProviderTests.cs b/src/OpenDDD.Tests/Integration/Infrastructure/Events/InMemory/InMemoryMessagingProviderTests.cs index d47275b..5edf089 100644 --- a/src/OpenDDD.Tests/Integration/Infrastructure/Events/InMemory/InMemoryMessagingProviderTests.cs +++ b/src/OpenDDD.Tests/Integration/Infrastructure/Events/InMemory/InMemoryMessagingProviderTests.cs @@ -180,7 +180,7 @@ public async Task CompetingConsumers_ShouldDistributeEvenly_WhenMultipleConsumer var consumerGroup = "test-consumer-group"; var totalMessages = 100; var numConsumers = 2; - var variancePercentage = 0.1; // Allow 10% variance + var variancePercentage = 0.2; var perConsumerMessageCount = new ConcurrentDictionary(); var allMessagesProcessed = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); From 913caef70c0d5cfd640820eec0332eb00b45d09e Mon Sep 17 00:00:00 2001 From: David Runemalm Date: Tue, 4 Mar 2025 17:41:40 +0700 Subject: [PATCH 084/109] Refactor azure service bus tests a bit. --- .../AzureServiceBusMessagingProviderTests.cs | 160 ++++++++++-------- 1 file changed, 92 insertions(+), 68 deletions(-) diff --git a/src/OpenDDD.Tests/Integration/Infrastructure/Events/Azure/AzureServiceBusMessagingProviderTests.cs b/src/OpenDDD.Tests/Integration/Infrastructure/Events/Azure/AzureServiceBusMessagingProviderTests.cs index 3e17074..defb370 100644 --- a/src/OpenDDD.Tests/Integration/Infrastructure/Events/Azure/AzureServiceBusMessagingProviderTests.cs +++ b/src/OpenDDD.Tests/Integration/Infrastructure/Events/Azure/AzureServiceBusMessagingProviderTests.cs @@ -1,37 +1,38 @@ using System.Collections.Concurrent; using Microsoft.Extensions.Logging; -using Moq; +using Xunit.Abstractions; using OpenDDD.Infrastructure.Events.Azure; using OpenDDD.Tests.Base; using Azure.Messaging.ServiceBus; using Azure.Messaging.ServiceBus.Administration; -using Xunit.Abstractions; namespace OpenDDD.Tests.Integration.Infrastructure.Events.Azure { - [Collection("AzureServiceBusTests")] // Ensure tests run sequentially + [Collection("AzureServiceBusTests")] public class AzureServiceBusMessagingProviderTests : IntegrationTests, IAsyncLifetime { private readonly string _connectionString; private readonly ServiceBusAdministrationClient _adminClient; - private readonly Mock> _loggerMock; + private readonly ILogger _logger; private readonly ServiceBusClient _serviceBusClient; private readonly AzureServiceBusMessagingProvider _messagingProvider; + private readonly CancellationTokenSource _cts = new(TimeSpan.FromSeconds(60)); - public AzureServiceBusMessagingProviderTests(ITestOutputHelper testOutputHelper) : base(testOutputHelper) + public AzureServiceBusMessagingProviderTests(ITestOutputHelper testOutputHelper) + : base(testOutputHelper, enableLogging: true) { _connectionString = Environment.GetEnvironmentVariable("AZURE_SERVICE_BUS_CONNECTION_STRING") ?? throw new InvalidOperationException("AZURE_SERVICE_BUS_CONNECTION_STRING is not set."); _adminClient = new ServiceBusAdministrationClient(_connectionString); _serviceBusClient = new ServiceBusClient(_connectionString); - _loggerMock = new Mock>(); - + _logger = LoggerFactory.CreateLogger(); + _messagingProvider = new AzureServiceBusMessagingProvider( _serviceBusClient, _adminClient, autoCreateTopics: true, - _loggerMock.Object); + _logger); } public async Task InitializeAsync() @@ -41,7 +42,8 @@ public async Task InitializeAsync() public async Task DisposeAsync() { - + await _cts.CancelAsync(); + await _messagingProvider.DisposeAsync(); } private async Task CleanupTopicsAndSubscriptionsAsync() @@ -86,8 +88,7 @@ public async Task AutoCreateTopic_ShouldNotCreateTopicOnSubscribe_WhenSettingDis { // Arrange var topicName = $"test-topic-{Guid.NewGuid()}"; - - // Ensure topic does not exist before the test + if (await _adminClient.TopicExistsAsync(topicName)) { await _adminClient.DeleteTopicAsync(topicName); @@ -97,7 +98,7 @@ public async Task AutoCreateTopic_ShouldNotCreateTopicOnSubscribe_WhenSettingDis _serviceBusClient, _adminClient, autoCreateTopics: false, - _loggerMock.Object); + _logger); // Act & Assert var exception = await Assert.ThrowsAsync(async () => @@ -118,31 +119,31 @@ public async Task AtLeastOnceGurantee_ShouldDeliverToLateSubscriber_WhenSubscrib var subscriptionName = "test-subscription"; var receivedMessages = new ConcurrentBag(); var messageToSend = "Persistent Message Test"; - var cts = new CancellationTokenSource(TimeSpan.FromSeconds(30)); + var messageReceivedTcs = new TaskCompletionSource(); await _messagingProvider.SubscribeAsync(topicName, subscriptionName, async (msg, token) => { Assert.Fail("First subscription should not receive the message."); - }, cts.Token); + }, _cts.Token); + await Task.Delay(500); - await _messagingProvider.UnsubscribeAsync(topicName, subscriptionName, cts.Token); + await _messagingProvider.UnsubscribeAsync(topicName, subscriptionName, _cts.Token); + await Task.Delay(500); - // Act: Publish message - await _messagingProvider.PublishAsync(topicName, messageToSend, cts.Token); - - // Delay to simulate late subscriber - await Task.Delay(2000); + // Act + await _messagingProvider.PublishAsync(topicName, messageToSend, _cts.Token); - // Late subscriber await _messagingProvider.SubscribeAsync(topicName, subscriptionName, async (msg, token) => { receivedMessages.Add(msg); - }, cts.Token); + messageReceivedTcs.TrySetResult(true); + }, _cts.Token); + + // Wait for message with timeout + await messageReceivedTcs.Task.WaitAsync(TimeSpan.FromSeconds(10)); - // Wait for message processing - await Task.Delay(1000); // Assert Assert.Contains(messageToSend, receivedMessages); @@ -156,21 +157,18 @@ public async Task AtLeastOnceGurantee_ShouldNotDeliverToLateSubscriber_WhenNotSu var subscriptionName = "test-subscription"; var receivedMessages = new ConcurrentBag(); var messageToSend = "Non-Persistent Message Test"; - var cts = new CancellationTokenSource(TimeSpan.FromSeconds(10)); - // Act: Publish message before any subscription - await _messagingProvider.PublishAsync(topicName, messageToSend, cts.Token); + // Act + await _messagingProvider.PublishAsync(topicName, messageToSend, _cts.Token); await Task.Delay(500); - // Late subscriber await _messagingProvider.SubscribeAsync(topicName, subscriptionName, async (msg, token) => { receivedMessages.Add(msg); - }, cts.Token); + }, _cts.Token); - // Wait for message processing - await Task.Delay(1000); + await Task.Delay(5000); // Assert Assert.DoesNotContain(messageToSend, receivedMessages); @@ -184,7 +182,6 @@ public async Task AtLeastOnceGurantee_ShouldRedeliverLater_WhenMessageNotAcked() var subscriptionName = "test-subscription"; var receivedMessages = new ConcurrentBag(); var messageToSend = "Redelivery Test"; - var cts = new CancellationTokenSource(TimeSpan.FromSeconds(10)); async Task FaultyHandler(string msg, CancellationToken token) { @@ -192,16 +189,19 @@ async Task FaultyHandler(string msg, CancellationToken token) throw new Exception("Simulated consumer crash before acknowledgment."); } - await _messagingProvider.SubscribeAsync(topicName, subscriptionName, FaultyHandler, cts.Token); + await _messagingProvider.SubscribeAsync(topicName, subscriptionName, FaultyHandler, _cts.Token); await Task.Delay(500); - // Act: Publish message - await _messagingProvider.PublishAsync(topicName, messageToSend, cts.Token); + // Act + await _messagingProvider.PublishAsync(topicName, messageToSend, _cts.Token); - // Wait for redelivery - await Task.Delay(3000); + for (int i = 0; i < 300; i++) + { + if (receivedMessages.Count > 1) break; + await Task.Delay(1000); + } - // Assert: The message should be received multiple times due to reattempts + // Assert Assert.True(receivedMessages.Count > 1, "Message should be redelivered at least once."); } @@ -213,26 +213,23 @@ public async Task CompetingConsumers_ShouldDeliverOnlyOnce_WhenMultipleConsumers var subscriptionName = "test-consumer-group"; var receivedMessages = new ConcurrentDictionary(); var messageToSend = "Competing Consumer Test"; - var cts = new CancellationTokenSource(TimeSpan.FromSeconds(10)); async Task MessageHandler(string msg, CancellationToken token) { receivedMessages.AddOrUpdate("received", 1, (key, value) => value + 1); } - // Multiple competing consumers in the same group - await _messagingProvider.SubscribeAsync(topicName, subscriptionName, MessageHandler, cts.Token); - await _messagingProvider.SubscribeAsync(topicName, subscriptionName, MessageHandler, cts.Token); - await _messagingProvider.SubscribeAsync(topicName, subscriptionName, MessageHandler, cts.Token); + await _messagingProvider.SubscribeAsync(topicName, subscriptionName, MessageHandler, _cts.Token); + await _messagingProvider.SubscribeAsync(topicName, subscriptionName, MessageHandler, _cts.Token); + await _messagingProvider.SubscribeAsync(topicName, subscriptionName, MessageHandler, _cts.Token); await Task.Delay(500); // Act - await _messagingProvider.PublishAsync(topicName, messageToSend, cts.Token); + await _messagingProvider.PublishAsync(topicName, messageToSend, _cts.Token); - // Wait for processing - await Task.Delay(1000); + await Task.Delay(3000); - // Assert: Only one of the competing consumers should receive the message + // Assert Assert.Equal(1, receivedMessages.GetValueOrDefault("received", 0)); } @@ -241,38 +238,65 @@ public async Task CompetingConsumers_ShouldDistributeEvenly_WhenMultipleConsumer { // Arrange var topicName = $"test-topic-{Guid.NewGuid()}"; - var subscriptionName = "test-consumer-group"; - var receivedMessages = new ConcurrentDictionary(); - var totalMessages = 10; - var cts = new CancellationTokenSource(TimeSpan.FromSeconds(30)); + var consumerGroup = "test-consumer-group"; + var totalMessages = 50; + var numConsumers = 10; + var variancePercentage = 0.1; + var perConsumerMessageCount = new ConcurrentDictionary(); // Track messages per consumer + var allMessagesProcessed = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + + async Task Subscribe() + { + var consumerId = Guid.NewGuid(); - async Task MessageHandler(string msg, CancellationToken token) + async Task MessageHandler(string msg, CancellationToken token) + { + perConsumerMessageCount.AddOrUpdate(consumerId, 1, (_, count) => count + 1); + _logger.LogDebug($"Subscriber {consumerId} received a message."); + + if (perConsumerMessageCount.Values.Sum() >= totalMessages) + { + allMessagesProcessed.TrySetResult(true); + } + } + + await _messagingProvider.SubscribeAsync(topicName, consumerGroup, MessageHandler, _cts.Token); + } + + for (int i = 0; i < numConsumers; i++) { - receivedMessages.AddOrUpdate(msg, 1, (key, value) => value + 1); + await Subscribe(); } - // Multiple competing consumers in the same group - await _messagingProvider.SubscribeAsync(topicName, subscriptionName, MessageHandler, cts.Token); - await _messagingProvider.SubscribeAsync(topicName, subscriptionName, MessageHandler, cts.Token); - await _messagingProvider.SubscribeAsync(topicName, subscriptionName, MessageHandler, cts.Token); await Task.Delay(500); - - // Act: Publish multiple messages + + // Act for (int i = 0; i < totalMessages; i++) { - await _messagingProvider.PublishAsync(topicName, $"Message {i}", cts.Token); + await _messagingProvider.PublishAsync(topicName, "Test Message", _cts.Token); } - // Wait for processing - await Task.Delay(2000); + try + { + await allMessagesProcessed.Task.WaitAsync(TimeSpan.FromSeconds(10)); + } + catch (TimeoutException) + { + _logger.LogDebug("Timed out waiting for consumers to receive all messages."); + Assert.Fail($"Consumers only processed {perConsumerMessageCount.Values.Sum()} of {totalMessages} messages."); + } - // Assert: Messages should be evenly distributed across consumers - var messageCounts = receivedMessages.Values; - var minReceived = messageCounts.Min(); - var maxReceived = messageCounts.Max(); + // Assert + var messageCounts = perConsumerMessageCount.Values.ToList(); + var expectedPerConsumer = totalMessages / numConsumers; + var variance = (int)(expectedPerConsumer * variancePercentage); + var minAllowed = expectedPerConsumer - variance; + var maxAllowed = expectedPerConsumer + variance; - Assert.True(maxReceived - minReceived <= 1, - "Messages should be evenly distributed among competing consumers."); + foreach (var count in messageCounts) + { + Assert.InRange(count, minAllowed, maxAllowed); + } } } } From da1964d1e1fc60490cc6562aae344444f7537b21 Mon Sep 17 00:00:00 2001 From: David Runemalm Date: Tue, 4 Mar 2025 18:02:21 +0700 Subject: [PATCH 085/109] Refactor a bit. --- .../InMemoryMessagingProviderTests.cs | 24 +++++++++---------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/src/OpenDDD.Tests/Integration/Infrastructure/Events/InMemory/InMemoryMessagingProviderTests.cs b/src/OpenDDD.Tests/Integration/Infrastructure/Events/InMemory/InMemoryMessagingProviderTests.cs index 5edf089..2380d2a 100644 --- a/src/OpenDDD.Tests/Integration/Infrastructure/Events/InMemory/InMemoryMessagingProviderTests.cs +++ b/src/OpenDDD.Tests/Integration/Infrastructure/Events/InMemory/InMemoryMessagingProviderTests.cs @@ -37,27 +37,26 @@ public async Task AtLeastOnceGuarantee_ShouldDeliverToLateSubscriber_WhenSubscri var groupName = "test-subscription"; var receivedMessages = new ConcurrentBag(); var messageToSend = "Persistent Message Test"; - var cts = new CancellationTokenSource(TimeSpan.FromSeconds(30)); var lateSubscriberReceived = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); await _messagingProvider.SubscribeAsync(topicName, groupName, async (msg, token) => { Assert.Fail("First subscription should not receive the message."); - }, cts.Token); + }, _cts.Token); await Task.Delay(500); - await _messagingProvider.UnsubscribeAsync(topicName, groupName, cts.Token); + await _messagingProvider.UnsubscribeAsync(topicName, groupName, _cts.Token); // Act - await _messagingProvider.PublishAsync(topicName, messageToSend, cts.Token); + await _messagingProvider.PublishAsync(topicName, messageToSend, _cts.Token); await _messagingProvider.SubscribeAsync(topicName, groupName, async (msg, token) => { receivedMessages.Add(msg); lateSubscriberReceived.TrySetResult(true); - }, cts.Token); + }, _cts.Token); // Assert try @@ -79,7 +78,7 @@ public async Task AtLeastOnceGuarantee_ShouldNotDeliverToLateSubscriber_WhenNotS var topicName = "test-topic-no-late-subscriber"; var consumerGroup = "test-consumer-group"; var messageToSend = "Non-Persistent Message Test"; - ConcurrentBag _receivedMessages = new(); + ConcurrentBag receivedMessages = new(); // Act await _messagingProvider.PublishAsync(topicName, messageToSend, _cts.Token); @@ -88,13 +87,13 @@ public async Task AtLeastOnceGuarantee_ShouldNotDeliverToLateSubscriber_WhenNotS // Late subscriber await _messagingProvider.SubscribeAsync(topicName, consumerGroup, async (msg, token) => { - _receivedMessages.Add(msg); + receivedMessages.Add(msg); }, _cts.Token); await Task.Delay(2000); // Assert - _receivedMessages.Should().NotContain(messageToSend); + receivedMessages.Should().NotContain(messageToSend); } [Fact] @@ -104,11 +103,11 @@ public async Task AtLeastOnceGuarantee_ShouldRedeliverLater_WhenMessageNotAcked( var topicName = "test-topic-redelivery"; var consumerGroup = "test-consumer-group"; var messageToSend = "Redelivery Test"; - ConcurrentBag _receivedMessages = new(); + ConcurrentBag receivedMessages = new(); async Task FaultyHandler(string msg, CancellationToken token) { - _receivedMessages.Add(msg); + receivedMessages.Add(msg); throw new Exception("Simulated consumer crash before acknowledgment."); } @@ -119,12 +118,12 @@ async Task FaultyHandler(string msg, CancellationToken token) for (int i = 0; i < 20; i++) { - if (_receivedMessages.Count > 1) break; + if (receivedMessages.Count > 1) break; await Task.Delay(500); } // Assert - _receivedMessages.Count.Should().BeGreaterThan(1); + receivedMessages.Count.Should().BeGreaterThan(1); } [Fact] @@ -152,7 +151,6 @@ async Task MessageHandler(string msg, CancellationToken token) } } - // Subscribe multiple consumers in the same group await _messagingProvider.SubscribeAsync(topicName, consumerGroup, MessageHandler, _cts.Token); await _messagingProvider.SubscribeAsync(topicName, consumerGroup, MessageHandler, _cts.Token); From 8c285092a9db5b7c45174a0a6c875e2f0b14e6f5 Mon Sep 17 00:00:00 2001 From: David Runemalm Date: Tue, 4 Mar 2025 18:03:07 +0700 Subject: [PATCH 086/109] Refactor rabbitmq messaging provider integration tests. --- .../RabbitMqMessagingProviderTests.cs | 182 ++++++++++++------ 1 file changed, 118 insertions(+), 64 deletions(-) diff --git a/src/OpenDDD.Tests/Integration/Infrastructure/Events/RabbitMq/RabbitMqMessagingProviderTests.cs b/src/OpenDDD.Tests/Integration/Infrastructure/Events/RabbitMq/RabbitMqMessagingProviderTests.cs index eec960d..ea9d796 100644 --- a/src/OpenDDD.Tests/Integration/Infrastructure/Events/RabbitMq/RabbitMqMessagingProviderTests.cs +++ b/src/OpenDDD.Tests/Integration/Infrastructure/Events/RabbitMq/RabbitMqMessagingProviderTests.cs @@ -1,7 +1,6 @@ using System.Collections.Concurrent; using Microsoft.Extensions.Logging; using Xunit.Abstractions; -using Moq; using OpenDDD.Infrastructure.Events.RabbitMq; using OpenDDD.Infrastructure.Events.RabbitMq.Factories; using OpenDDD.Tests.Base; @@ -16,16 +15,18 @@ public class RabbitMqMessagingProviderTests : IntegrationTests, IAsyncLifetime private readonly RabbitMqMessagingProvider _messagingProvider; private readonly IConnectionFactory _connectionFactory; private readonly IRabbitMqConsumerFactory _consumerFactory; - private readonly Mock> _loggerMock; + private readonly ILogger _logger; private IConnection? _connection; private IChannel? _channel; + private readonly CancellationTokenSource _cts = new(TimeSpan.FromSeconds(60)); private readonly string _testTopic = "OpenDddTestTopic"; private readonly string _testConsumerGroup = "OpenDddTestGroup"; - public RabbitMqMessagingProviderTests(ITestOutputHelper testOutputHelper) : base(testOutputHelper) + public RabbitMqMessagingProviderTests(ITestOutputHelper testOutputHelper) + : base(testOutputHelper, enableLogging: true) { - _loggerMock = new Mock>(); + _logger = LoggerFactory.CreateLogger(); _connectionFactory = new ConnectionFactory { @@ -36,8 +37,8 @@ public RabbitMqMessagingProviderTests(ITestOutputHelper testOutputHelper) : base VirtualHost = Environment.GetEnvironmentVariable("RABBITMQ_VHOST") ?? "/" }; - _consumerFactory = new RabbitMqConsumerFactory(_loggerMock.Object); - _messagingProvider = new RabbitMqMessagingProvider(_connectionFactory, _consumerFactory, _loggerMock.Object); + _consumerFactory = new RabbitMqConsumerFactory(_logger); + _messagingProvider = new RabbitMqMessagingProvider(_connectionFactory, _consumerFactory, _logger); } public async Task InitializeAsync() @@ -61,6 +62,9 @@ public async Task DisposeAsync() await _connection.CloseAsync(); await _connection.DisposeAsync(); } + + _cts.Cancel(); + await _messagingProvider.DisposeAsync(); } private async Task VerifyExchangeAndQueueDoNotExist() @@ -153,35 +157,42 @@ public async Task AtLeastOnceGurantee_ShouldDeliverToLateSubscriber_WhenSubscrib // Arrange var receivedMessages = new ConcurrentBag(); var messageToSend = "Persistent Message Test"; - var cts = new CancellationTokenSource(TimeSpan.FromSeconds(10)); + + var lateSubscriberReceived = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - // First subscription to establish the listener group await _messagingProvider.SubscribeAsync(_testTopic, _testConsumerGroup, async (msg, token) => { Assert.Fail("First subscription should not receive the message."); - }, cts.Token); + }, _cts.Token); await Task.Delay(500); - - // Unsubscribe - await _messagingProvider.UnsubscribeAsync(_testTopic, _testConsumerGroup, cts.Token); + + await _messagingProvider.UnsubscribeAsync(_testTopic, _testConsumerGroup, _cts.Token); await Task.Delay(500); - // Act: Publish message - await _messagingProvider.PublishAsync(_testTopic, messageToSend, cts.Token); + // Act + await _messagingProvider.PublishAsync(_testTopic, messageToSend, _cts.Token); - // Delay to simulate late subscriber await Task.Delay(2000); // Late subscriber await _messagingProvider.SubscribeAsync(_testTopic, _testConsumerGroup, async (msg, token) => { receivedMessages.Add(msg); - }, cts.Token); + lateSubscriberReceived.TrySetResult(true); + }, _cts.Token); - // Wait for message processing await Task.Delay(1000); // Assert + try + { + await lateSubscriberReceived.Task.WaitAsync(TimeSpan.FromSeconds(5)); + } + catch (TimeoutException) + { + Assert.Fail($"Late subscriber did not receive the expected message '{messageToSend}' within 5 seconds."); + } + Assert.Contains(messageToSend, receivedMessages); } @@ -189,23 +200,19 @@ await _messagingProvider.SubscribeAsync(_testTopic, _testConsumerGroup, async (m public async Task AtLeastOnceGurantee_ShouldNotDeliverToLateSubscriber_WhenNotSubscribedBefore() { // Arrange - var receivedMessages = new ConcurrentBag(); var messageToSend = "Non-Persistent Message Test"; - var cts = new CancellationTokenSource(TimeSpan.FromSeconds(10)); - - // Act: Publish message before any subscription - await _messagingProvider.PublishAsync(_testTopic, messageToSend, cts.Token); + var receivedMessages = new ConcurrentBag(); - // Delay to simulate late subscriber + // Act + await _messagingProvider.PublishAsync(_testTopic, messageToSend, _cts.Token); await Task.Delay(2000); // Late subscriber await _messagingProvider.SubscribeAsync(_testTopic, _testConsumerGroup, async (msg, token) => { receivedMessages.Add(msg); - }, cts.Token); + }, _cts.Token); - // Wait for message processing await Task.Delay(1000); // Assert @@ -216,9 +223,8 @@ await _messagingProvider.SubscribeAsync(_testTopic, _testConsumerGroup, async (m public async Task AtLeastOnceGurantee_ShouldRedeliverLater_WhenMessageNotAcked() { // Arrange - var receivedMessages = new ConcurrentBag(); var messageToSend = "Redelivery Test"; - var cts = new CancellationTokenSource(TimeSpan.FromSeconds(10)); + var receivedMessages = new ConcurrentBag(); async Task FaultyHandler(string msg, CancellationToken token) { @@ -226,16 +232,19 @@ async Task FaultyHandler(string msg, CancellationToken token) throw new Exception("Simulated consumer crash before acknowledgment."); } - await _messagingProvider.SubscribeAsync(_testTopic, _testConsumerGroup, FaultyHandler, cts.Token); - await Task.Delay(500); // Ensure setup + await _messagingProvider.SubscribeAsync(_testTopic, _testConsumerGroup, FaultyHandler, _cts.Token); + await Task.Delay(500); - // Act: Publish message - await _messagingProvider.PublishAsync(_testTopic, messageToSend, cts.Token); + // Act + await _messagingProvider.PublishAsync(_testTopic, messageToSend, _cts.Token); - // Wait for redelivery - await Task.Delay(3000); + for (int i = 0; i < 20; i++) + { + if (receivedMessages.Count > 1) break; + await Task.Delay(500); + } - // Assert: The message should be received multiple times due to reattempts + // Assert Assert.True(receivedMessages.Count > 1, "Message should be redelivered at least once."); } @@ -245,26 +254,43 @@ public async Task CompetingConsumers_ShouldDeliverOnlyOnce_WhenMultipleConsumers // Arrange var receivedMessages = new ConcurrentDictionary(); var messageToSend = "Competing Consumer Test"; - var cts = new CancellationTokenSource(TimeSpan.FromSeconds(10)); + var allSubscribersProcessed = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); async Task MessageHandler(string msg, CancellationToken token) { receivedMessages.AddOrUpdate("received", 1, (key, value) => value + 1); + + // If any consumer has received more than 1 message, fail immediately + if (receivedMessages["received"] > 1) + { + allSubscribersProcessed.TrySetException(new Exception("More than one consumer in the group received the message!")); + } + else + { + allSubscribersProcessed.TrySetResult(true); + } } - // Multiple competing consumers in the same group - await _messagingProvider.SubscribeAsync(_testTopic, _testConsumerGroup, MessageHandler, cts.Token); - await _messagingProvider.SubscribeAsync(_testTopic, _testConsumerGroup, MessageHandler, cts.Token); - await _messagingProvider.SubscribeAsync(_testTopic, _testConsumerGroup, MessageHandler, cts.Token); - await Task.Delay(500); // Allow time for consumers to start + await _messagingProvider.SubscribeAsync(_testTopic, _testConsumerGroup, MessageHandler, _cts.Token); + await _messagingProvider.SubscribeAsync(_testTopic, _testConsumerGroup, MessageHandler, _cts.Token); + await _messagingProvider.SubscribeAsync(_testTopic, _testConsumerGroup, MessageHandler, _cts.Token); + await Task.Delay(500); // Act - await _messagingProvider.PublishAsync(_testTopic, messageToSend, cts.Token); + await _messagingProvider.PublishAsync(_testTopic, messageToSend, _cts.Token); - // Wait for processing - await Task.Delay(1000); + await Task.Delay(3000); + + try + { + await allSubscribersProcessed.Task.WaitAsync(TimeSpan.FromSeconds(5)); + } + catch (TimeoutException) + { + Assert.Fail("Timed out waiting for message processing."); + } - // Assert: Only one of the competing consumers should receive the message + // Assert Assert.Equal(1, receivedMessages.GetValueOrDefault("received", 0)); } @@ -273,35 +299,63 @@ public async Task CompetingConsumers_ShouldDistributeEvenly_WhenMultipleConsumer { // Arrange var receivedMessages = new ConcurrentDictionary(); - var totalMessages = 10; - var cts = new CancellationTokenSource(TimeSpan.FromSeconds(15)); + var totalMessages = 100; + var numConsumers = 10; + var variancePercentage = 0.1; + var perConsumerMessageCount = new ConcurrentDictionary(); + var allMessagesProcessed = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + + async Task CreateConsumer() + { + var consumerId = Guid.NewGuid(); - async Task MessageHandler(string msg, CancellationToken token) + async Task MessageHandler(string msg, CancellationToken token) + { + perConsumerMessageCount.AddOrUpdate(consumerId, 1, (_, count) => count + 1); + + _logger.LogDebug($"Consumer {consumerId} received a message."); + + if (perConsumerMessageCount.Values.Sum() >= totalMessages) + { + allMessagesProcessed.TrySetResult(true); + } + } + + await _messagingProvider.SubscribeAsync(_testTopic, _testConsumerGroup, MessageHandler, _cts.Token); + } + + for (int i = 0; i < numConsumers; i++) { - receivedMessages.AddOrUpdate(msg, 1, (key, value) => value + 1); + await CreateConsumer(); } - // Multiple competing consumers in the same group - await _messagingProvider.SubscribeAsync(_testTopic, _testConsumerGroup, MessageHandler, cts.Token); - await _messagingProvider.SubscribeAsync(_testTopic, _testConsumerGroup, MessageHandler, cts.Token); - await _messagingProvider.SubscribeAsync(_testTopic, _testConsumerGroup, MessageHandler, cts.Token); - await Task.Delay(500); // Allow time for consumers to start - - // Act: Publish multiple messages + // Act for (int i = 0; i < totalMessages; i++) { - await _messagingProvider.PublishAsync(_testTopic, $"Message {i}", cts.Token); + await _messagingProvider.PublishAsync(_testTopic, "Test Message", _cts.Token); + } + + try + { + await allMessagesProcessed.Task.WaitAsync(TimeSpan.FromSeconds(10)); + } + catch (TimeoutException) + { + _logger.LogDebug("Timed out waiting for consumers to receive all messages."); + Assert.Fail($"Consumers only processed {perConsumerMessageCount.Values.Sum()} of {totalMessages} messages."); } - // Wait for processing - await Task.Delay(2000); + // Assert + var messageCounts = perConsumerMessageCount.Values.ToList(); + var expectedPerConsumer = totalMessages / numConsumers; + var variance = (int)(expectedPerConsumer * variancePercentage); + var minAllowed = expectedPerConsumer - variance; + var maxAllowed = expectedPerConsumer + variance; - // Assert: Messages should be evenly distributed across consumers - var messageCounts = receivedMessages.Values; - var minReceived = messageCounts.Min(); - var maxReceived = messageCounts.Max(); - - Assert.True(maxReceived - minReceived <= 1, "Messages should be evenly distributed among competing consumers."); + foreach (var count in messageCounts) + { + Assert.InRange(count, minAllowed, maxAllowed); + } } } } From 94c3702a81085e50c3be78582a2fb6b36553c2e1 Mon Sep 17 00:00:00 2001 From: David Runemalm Date: Tue, 4 Mar 2025 18:12:23 +0700 Subject: [PATCH 087/109] Refactor tests namespaces. --- .../{ => Base}/Domain/Model/TestAggregateRoot.cs | 2 +- src/OpenDDD.Tests/{ => Base}/Domain/Model/TestEntity.cs | 2 +- .../{ => Base}/Domain/Model/TestValueObject.cs | 2 +- src/OpenDDD.Tests/Base/IntegrationTests.cs | 2 +- src/OpenDDD.Tests/{ => Base}/Logging/XunitLogger.cs | 2 +- .../{ => Base}/Logging/XunitLoggingProvider.cs | 2 +- .../Configurations/TestAggregateRootConfiguration.cs | 2 +- .../EfCore/Configurations/TestEntityConfiguration.cs | 2 +- .../EfCore/Configurations/TestValueObjectConfiguration.cs | 2 +- .../EfCore/DbContext/Postgres/PostgresTestDbContext.cs | 2 +- .../EfCore/DbContext/Sqlite/SqliteTestDbContext.cs | 2 +- .../EfCore/Postgres/PostgresEfCoreRepositoryTests.cs | 2 +- .../EfCore/Sqlite/SqliteEfCoreRepositoryTests.cs | 2 +- .../OpenDdd/InMemory/InMemoryOpenDddRepositoryTests.cs | 2 +- .../OpenDdd/Postgres/PostgresOpenDddRepositoryTests.cs | 2 +- .../Events/Azure/AzureServiceBusMessagingProviderTests.cs | 8 ++++---- .../Infrastructure/Events/DomainPublisherTests.cs | 2 +- .../Infrastructure/Events/EventSerializerTests.cs | 5 ++--- .../Events/InMemory/InMemoryMessagingProviderTests.cs | 2 +- .../Infrastructure/Events/IntegrationPublisherTests.cs | 3 +-- .../Events/Kafka/KafkaMessagingProviderTests.cs | 6 +++--- .../Events/RabbitMq/RabbitMqMessagingProviderTests.cs | 3 +-- .../{ => Unit}/Infrastructure/Events/TestEvent.cs | 2 +- .../OpenDdd/Expressions/JsonbExpressionParserTests.cs | 3 +-- .../Serializers/OpenDddAggregateSerializerTests.cs | 7 +++---- .../OpenDdd/Serializers/OpenDddSerializerTests.cs | 3 +-- 26 files changed, 34 insertions(+), 40 deletions(-) rename src/OpenDDD.Tests/{ => Base}/Domain/Model/TestAggregateRoot.cs (95%) rename src/OpenDDD.Tests/{ => Base}/Domain/Model/TestEntity.cs (92%) rename src/OpenDDD.Tests/{ => Base}/Domain/Model/TestValueObject.cs (90%) rename src/OpenDDD.Tests/{ => Base}/Logging/XunitLogger.cs (96%) rename src/OpenDDD.Tests/{ => Base}/Logging/XunitLoggingProvider.cs (93%) rename src/OpenDDD.Tests/{ => Unit}/Infrastructure/Events/Azure/AzureServiceBusMessagingProviderTests.cs (97%) rename src/OpenDDD.Tests/{ => Unit}/Infrastructure/Events/DomainPublisherTests.cs (97%) rename src/OpenDDD.Tests/{ => Unit}/Infrastructure/Events/EventSerializerTests.cs (95%) rename src/OpenDDD.Tests/{ => Unit}/Infrastructure/Events/InMemory/InMemoryMessagingProviderTests.cs (98%) rename src/OpenDDD.Tests/{ => Unit}/Infrastructure/Events/IntegrationPublisherTests.cs (97%) rename src/OpenDDD.Tests/{ => Unit}/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs (98%) rename src/OpenDDD.Tests/{ => Unit}/Infrastructure/Events/RabbitMq/RabbitMqMessagingProviderTests.cs (98%) rename src/OpenDDD.Tests/{ => Unit}/Infrastructure/Events/TestEvent.cs (90%) rename src/OpenDDD.Tests/{ => Unit}/Infrastructure/Persistence/OpenDdd/Expressions/JsonbExpressionParserTests.cs (98%) rename src/OpenDDD.Tests/{ => Unit}/Infrastructure/Persistence/OpenDdd/Serializers/OpenDddAggregateSerializerTests.cs (93%) rename src/OpenDDD.Tests/{ => Unit}/Infrastructure/Persistence/OpenDdd/Serializers/OpenDddSerializerTests.cs (95%) diff --git a/src/OpenDDD.Tests/Domain/Model/TestAggregateRoot.cs b/src/OpenDDD.Tests/Base/Domain/Model/TestAggregateRoot.cs similarity index 95% rename from src/OpenDDD.Tests/Domain/Model/TestAggregateRoot.cs rename to src/OpenDDD.Tests/Base/Domain/Model/TestAggregateRoot.cs index 26ec64d..d39102a 100644 --- a/src/OpenDDD.Tests/Domain/Model/TestAggregateRoot.cs +++ b/src/OpenDDD.Tests/Base/Domain/Model/TestAggregateRoot.cs @@ -1,6 +1,6 @@ using OpenDDD.Domain.Model.Base; -namespace OpenDDD.Tests.Domain.Model +namespace OpenDDD.Tests.Base.Domain.Model { public class TestAggregateRoot : AggregateRootBase { diff --git a/src/OpenDDD.Tests/Domain/Model/TestEntity.cs b/src/OpenDDD.Tests/Base/Domain/Model/TestEntity.cs similarity index 92% rename from src/OpenDDD.Tests/Domain/Model/TestEntity.cs rename to src/OpenDDD.Tests/Base/Domain/Model/TestEntity.cs index b0c344f..07d13ea 100644 --- a/src/OpenDDD.Tests/Domain/Model/TestEntity.cs +++ b/src/OpenDDD.Tests/Base/Domain/Model/TestEntity.cs @@ -1,6 +1,6 @@ using OpenDDD.Domain.Model.Base; -namespace OpenDDD.Tests.Domain.Model +namespace OpenDDD.Tests.Base.Domain.Model { public class TestEntity : EntityBase { diff --git a/src/OpenDDD.Tests/Domain/Model/TestValueObject.cs b/src/OpenDDD.Tests/Base/Domain/Model/TestValueObject.cs similarity index 90% rename from src/OpenDDD.Tests/Domain/Model/TestValueObject.cs rename to src/OpenDDD.Tests/Base/Domain/Model/TestValueObject.cs index 07c00e1..f1efad9 100644 --- a/src/OpenDDD.Tests/Domain/Model/TestValueObject.cs +++ b/src/OpenDDD.Tests/Base/Domain/Model/TestValueObject.cs @@ -1,6 +1,6 @@ using OpenDDD.Domain.Model; -namespace OpenDDD.Tests.Domain.Model +namespace OpenDDD.Tests.Base.Domain.Model { public class TestValueObject : IValueObject { diff --git a/src/OpenDDD.Tests/Base/IntegrationTests.cs b/src/OpenDDD.Tests/Base/IntegrationTests.cs index 81fda57..d7fc11d 100644 --- a/src/OpenDDD.Tests/Base/IntegrationTests.cs +++ b/src/OpenDDD.Tests/Base/IntegrationTests.cs @@ -1,5 +1,5 @@ using Microsoft.Extensions.Logging; -using OpenDDD.Tests.Logging; +using OpenDDD.Tests.Base.Logging; using Xunit.Abstractions; namespace OpenDDD.Tests.Base diff --git a/src/OpenDDD.Tests/Logging/XunitLogger.cs b/src/OpenDDD.Tests/Base/Logging/XunitLogger.cs similarity index 96% rename from src/OpenDDD.Tests/Logging/XunitLogger.cs rename to src/OpenDDD.Tests/Base/Logging/XunitLogger.cs index 3fed942..f1bd8d4 100644 --- a/src/OpenDDD.Tests/Logging/XunitLogger.cs +++ b/src/OpenDDD.Tests/Base/Logging/XunitLogger.cs @@ -1,7 +1,7 @@ using Microsoft.Extensions.Logging; using Xunit.Abstractions; -namespace OpenDDD.Tests.Logging +namespace OpenDDD.Tests.Base.Logging { public class XunitLogger : ILogger { diff --git a/src/OpenDDD.Tests/Logging/XunitLoggingProvider.cs b/src/OpenDDD.Tests/Base/Logging/XunitLoggingProvider.cs similarity index 93% rename from src/OpenDDD.Tests/Logging/XunitLoggingProvider.cs rename to src/OpenDDD.Tests/Base/Logging/XunitLoggingProvider.cs index e5283e4..9abf476 100644 --- a/src/OpenDDD.Tests/Logging/XunitLoggingProvider.cs +++ b/src/OpenDDD.Tests/Base/Logging/XunitLoggingProvider.cs @@ -1,7 +1,7 @@ using Microsoft.Extensions.Logging; using Xunit.Abstractions; -namespace OpenDDD.Tests.Logging +namespace OpenDDD.Tests.Base.Logging { public class XunitLoggerProvider : ILoggerProvider { diff --git a/src/OpenDDD.Tests/Integration/Infrastructure/Persistence/EfCore/Configurations/TestAggregateRootConfiguration.cs b/src/OpenDDD.Tests/Integration/Infrastructure/Persistence/EfCore/Configurations/TestAggregateRootConfiguration.cs index 49aa41f..7ba2510 100644 --- a/src/OpenDDD.Tests/Integration/Infrastructure/Persistence/EfCore/Configurations/TestAggregateRootConfiguration.cs +++ b/src/OpenDDD.Tests/Integration/Infrastructure/Persistence/EfCore/Configurations/TestAggregateRootConfiguration.cs @@ -1,7 +1,7 @@ using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore.Metadata.Builders; using OpenDDD.Infrastructure.Persistence.EfCore.Base; -using OpenDDD.Tests.Domain.Model; +using OpenDDD.Tests.Base.Domain.Model; namespace OpenDDD.Tests.Integration.Infrastructure.Persistence.EfCore.Configurations { diff --git a/src/OpenDDD.Tests/Integration/Infrastructure/Persistence/EfCore/Configurations/TestEntityConfiguration.cs b/src/OpenDDD.Tests/Integration/Infrastructure/Persistence/EfCore/Configurations/TestEntityConfiguration.cs index db4830b..4d08546 100644 --- a/src/OpenDDD.Tests/Integration/Infrastructure/Persistence/EfCore/Configurations/TestEntityConfiguration.cs +++ b/src/OpenDDD.Tests/Integration/Infrastructure/Persistence/EfCore/Configurations/TestEntityConfiguration.cs @@ -1,6 +1,6 @@ using Microsoft.EntityFrameworkCore.Metadata.Builders; using OpenDDD.Infrastructure.Persistence.EfCore.Base; -using OpenDDD.Tests.Domain.Model; +using OpenDDD.Tests.Base.Domain.Model; namespace OpenDDD.Tests.Integration.Infrastructure.Persistence.EfCore.Configurations { diff --git a/src/OpenDDD.Tests/Integration/Infrastructure/Persistence/EfCore/Configurations/TestValueObjectConfiguration.cs b/src/OpenDDD.Tests/Integration/Infrastructure/Persistence/EfCore/Configurations/TestValueObjectConfiguration.cs index 07447cc..52bda60 100644 --- a/src/OpenDDD.Tests/Integration/Infrastructure/Persistence/EfCore/Configurations/TestValueObjectConfiguration.cs +++ b/src/OpenDDD.Tests/Integration/Infrastructure/Persistence/EfCore/Configurations/TestValueObjectConfiguration.cs @@ -1,6 +1,6 @@ using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore.Metadata.Builders; -using OpenDDD.Tests.Domain.Model; +using OpenDDD.Tests.Base.Domain.Model; namespace OpenDDD.Tests.Integration.Infrastructure.Persistence.EfCore.Configurations { diff --git a/src/OpenDDD.Tests/Integration/Infrastructure/Persistence/EfCore/DbContext/Postgres/PostgresTestDbContext.cs b/src/OpenDDD.Tests/Integration/Infrastructure/Persistence/EfCore/DbContext/Postgres/PostgresTestDbContext.cs index 0178a59..f8d0d6e 100644 --- a/src/OpenDDD.Tests/Integration/Infrastructure/Persistence/EfCore/DbContext/Postgres/PostgresTestDbContext.cs +++ b/src/OpenDDD.Tests/Integration/Infrastructure/Persistence/EfCore/DbContext/Postgres/PostgresTestDbContext.cs @@ -2,7 +2,7 @@ using Microsoft.Extensions.Logging; using OpenDDD.API.Options; using OpenDDD.Infrastructure.Persistence.EfCore.Base; -using OpenDDD.Tests.Domain.Model; +using OpenDDD.Tests.Base.Domain.Model; namespace OpenDDD.Tests.Integration.Infrastructure.Persistence.EfCore.DbContext.Postgres { diff --git a/src/OpenDDD.Tests/Integration/Infrastructure/Persistence/EfCore/DbContext/Sqlite/SqliteTestDbContext.cs b/src/OpenDDD.Tests/Integration/Infrastructure/Persistence/EfCore/DbContext/Sqlite/SqliteTestDbContext.cs index 1be7873..515ad76 100644 --- a/src/OpenDDD.Tests/Integration/Infrastructure/Persistence/EfCore/DbContext/Sqlite/SqliteTestDbContext.cs +++ b/src/OpenDDD.Tests/Integration/Infrastructure/Persistence/EfCore/DbContext/Sqlite/SqliteTestDbContext.cs @@ -2,7 +2,7 @@ using Microsoft.Extensions.Logging; using OpenDDD.API.Options; using OpenDDD.Infrastructure.Persistence.EfCore.Base; -using OpenDDD.Tests.Domain.Model; +using OpenDDD.Tests.Base.Domain.Model; namespace OpenDDD.Tests.Integration.Infrastructure.Persistence.EfCore.DbContext.Sqlite { diff --git a/src/OpenDDD.Tests/Integration/Infrastructure/Repository/EfCore/Postgres/PostgresEfCoreRepositoryTests.cs b/src/OpenDDD.Tests/Integration/Infrastructure/Repository/EfCore/Postgres/PostgresEfCoreRepositoryTests.cs index 1ab4bf9..d9cda77 100644 --- a/src/OpenDDD.Tests/Integration/Infrastructure/Repository/EfCore/Postgres/PostgresEfCoreRepositoryTests.cs +++ b/src/OpenDDD.Tests/Integration/Infrastructure/Repository/EfCore/Postgres/PostgresEfCoreRepositoryTests.cs @@ -14,7 +14,7 @@ using OpenDDD.Infrastructure.Repository.EfCore; using OpenDDD.Infrastructure.TransactionalOutbox; using OpenDDD.Tests.Base; -using OpenDDD.Tests.Domain.Model; +using OpenDDD.Tests.Base.Domain.Model; using OpenDDD.Tests.Integration.Infrastructure.Persistence.EfCore.DbContext.Postgres; namespace OpenDDD.Tests.Integration.Infrastructure.Repository.EfCore.Postgres diff --git a/src/OpenDDD.Tests/Integration/Infrastructure/Repository/EfCore/Sqlite/SqliteEfCoreRepositoryTests.cs b/src/OpenDDD.Tests/Integration/Infrastructure/Repository/EfCore/Sqlite/SqliteEfCoreRepositoryTests.cs index e5dbe45..8f27603 100644 --- a/src/OpenDDD.Tests/Integration/Infrastructure/Repository/EfCore/Sqlite/SqliteEfCoreRepositoryTests.cs +++ b/src/OpenDDD.Tests/Integration/Infrastructure/Repository/EfCore/Sqlite/SqliteEfCoreRepositoryTests.cs @@ -13,7 +13,7 @@ using OpenDDD.Infrastructure.Repository.EfCore; using OpenDDD.Infrastructure.TransactionalOutbox; using OpenDDD.Tests.Base; -using OpenDDD.Tests.Domain.Model; +using OpenDDD.Tests.Base.Domain.Model; using OpenDDD.Tests.Integration.Infrastructure.Persistence.EfCore.DbContext.Sqlite; namespace OpenDDD.Tests.Integration.Infrastructure.Repository.EfCore.Sqlite diff --git a/src/OpenDDD.Tests/Integration/Infrastructure/Repository/OpenDdd/InMemory/InMemoryOpenDddRepositoryTests.cs b/src/OpenDDD.Tests/Integration/Infrastructure/Repository/OpenDdd/InMemory/InMemoryOpenDddRepositoryTests.cs index 181e0aa..8ba871d 100644 --- a/src/OpenDDD.Tests/Integration/Infrastructure/Repository/OpenDdd/InMemory/InMemoryOpenDddRepositoryTests.cs +++ b/src/OpenDDD.Tests/Integration/Infrastructure/Repository/OpenDdd/InMemory/InMemoryOpenDddRepositoryTests.cs @@ -8,7 +8,7 @@ using OpenDDD.Infrastructure.Persistence.Storage.InMemory; using OpenDDD.Infrastructure.Repository.OpenDdd.InMemory; using OpenDDD.Tests.Base; -using OpenDDD.Tests.Domain.Model; +using OpenDDD.Tests.Base.Domain.Model; namespace OpenDDD.Tests.Integration.Infrastructure.Repository.OpenDdd.InMemory { diff --git a/src/OpenDDD.Tests/Integration/Infrastructure/Repository/OpenDdd/Postgres/PostgresOpenDddRepositoryTests.cs b/src/OpenDDD.Tests/Integration/Infrastructure/Repository/OpenDdd/Postgres/PostgresOpenDddRepositoryTests.cs index 8242ece..c57d53e 100644 --- a/src/OpenDDD.Tests/Integration/Infrastructure/Repository/OpenDdd/Postgres/PostgresOpenDddRepositoryTests.cs +++ b/src/OpenDDD.Tests/Integration/Infrastructure/Repository/OpenDdd/Postgres/PostgresOpenDddRepositoryTests.cs @@ -8,7 +8,7 @@ using OpenDDD.Infrastructure.Persistence.Serializers; using OpenDDD.Infrastructure.Repository.OpenDdd.Postgres; using OpenDDD.Tests.Base; -using OpenDDD.Tests.Domain.Model; +using OpenDDD.Tests.Base.Domain.Model; namespace OpenDDD.Tests.Integration.Infrastructure.Repository.OpenDdd.Postgres { diff --git a/src/OpenDDD.Tests/Infrastructure/Events/Azure/AzureServiceBusMessagingProviderTests.cs b/src/OpenDDD.Tests/Unit/Infrastructure/Events/Azure/AzureServiceBusMessagingProviderTests.cs similarity index 97% rename from src/OpenDDD.Tests/Infrastructure/Events/Azure/AzureServiceBusMessagingProviderTests.cs rename to src/OpenDDD.Tests/Unit/Infrastructure/Events/Azure/AzureServiceBusMessagingProviderTests.cs index 4443d3a..ef5874d 100644 --- a/src/OpenDDD.Tests/Infrastructure/Events/Azure/AzureServiceBusMessagingProviderTests.cs +++ b/src/OpenDDD.Tests/Unit/Infrastructure/Events/Azure/AzureServiceBusMessagingProviderTests.cs @@ -1,12 +1,12 @@ -using Microsoft.Extensions.Logging; -using Moq; -using Azure; +using Azure; using Azure.Messaging.ServiceBus; using Azure.Messaging.ServiceBus.Administration; +using Microsoft.Extensions.Logging; +using Moq; using OpenDDD.Infrastructure.Events.Azure; using OpenDDD.Tests.Base; -namespace OpenDDD.Tests.Infrastructure.Events.Azure +namespace OpenDDD.Tests.Unit.Infrastructure.Events.Azure { public class AzureServiceBusMessagingProviderTests : UnitTests { diff --git a/src/OpenDDD.Tests/Infrastructure/Events/DomainPublisherTests.cs b/src/OpenDDD.Tests/Unit/Infrastructure/Events/DomainPublisherTests.cs similarity index 97% rename from src/OpenDDD.Tests/Infrastructure/Events/DomainPublisherTests.cs rename to src/OpenDDD.Tests/Unit/Infrastructure/Events/DomainPublisherTests.cs index 951037b..f89122b 100644 --- a/src/OpenDDD.Tests/Infrastructure/Events/DomainPublisherTests.cs +++ b/src/OpenDDD.Tests/Unit/Infrastructure/Events/DomainPublisherTests.cs @@ -3,7 +3,7 @@ using OpenDDD.Infrastructure.Events; using OpenDDD.Tests.Base; -namespace OpenDDD.Tests.Infrastructure.Events +namespace OpenDDD.Tests.Unit.Infrastructure.Events { public class DomainPublisherTests : UnitTests { diff --git a/src/OpenDDD.Tests/Infrastructure/Events/EventSerializerTests.cs b/src/OpenDDD.Tests/Unit/Infrastructure/Events/EventSerializerTests.cs similarity index 95% rename from src/OpenDDD.Tests/Infrastructure/Events/EventSerializerTests.cs rename to src/OpenDDD.Tests/Unit/Infrastructure/Events/EventSerializerTests.cs index 6a06e8e..5cf2c8b 100644 --- a/src/OpenDDD.Tests/Infrastructure/Events/EventSerializerTests.cs +++ b/src/OpenDDD.Tests/Unit/Infrastructure/Events/EventSerializerTests.cs @@ -1,8 +1,7 @@ -using Xunit; -using OpenDDD.Infrastructure.Events; +using OpenDDD.Infrastructure.Events; using OpenDDD.Tests.Base; -namespace OpenDDD.Tests.Infrastructure.Events +namespace OpenDDD.Tests.Unit.Infrastructure.Events { public class EventSerializerTests : UnitTests { diff --git a/src/OpenDDD.Tests/Infrastructure/Events/InMemory/InMemoryMessagingProviderTests.cs b/src/OpenDDD.Tests/Unit/Infrastructure/Events/InMemory/InMemoryMessagingProviderTests.cs similarity index 98% rename from src/OpenDDD.Tests/Infrastructure/Events/InMemory/InMemoryMessagingProviderTests.cs rename to src/OpenDDD.Tests/Unit/Infrastructure/Events/InMemory/InMemoryMessagingProviderTests.cs index 5d267b5..4f55f9e 100644 --- a/src/OpenDDD.Tests/Infrastructure/Events/InMemory/InMemoryMessagingProviderTests.cs +++ b/src/OpenDDD.Tests/Unit/Infrastructure/Events/InMemory/InMemoryMessagingProviderTests.cs @@ -3,7 +3,7 @@ using OpenDDD.Infrastructure.Events.InMemory; using OpenDDD.Tests.Base; -namespace OpenDDD.Tests.Infrastructure.Events.InMemory +namespace OpenDDD.Tests.Unit.Infrastructure.Events.InMemory { public class InMemoryMessagingProviderTests : UnitTests { diff --git a/src/OpenDDD.Tests/Infrastructure/Events/IntegrationPublisherTests.cs b/src/OpenDDD.Tests/Unit/Infrastructure/Events/IntegrationPublisherTests.cs similarity index 97% rename from src/OpenDDD.Tests/Infrastructure/Events/IntegrationPublisherTests.cs rename to src/OpenDDD.Tests/Unit/Infrastructure/Events/IntegrationPublisherTests.cs index 46cc2a8..9cdc015 100644 --- a/src/OpenDDD.Tests/Infrastructure/Events/IntegrationPublisherTests.cs +++ b/src/OpenDDD.Tests/Unit/Infrastructure/Events/IntegrationPublisherTests.cs @@ -1,10 +1,9 @@ using FluentAssertions; -using Xunit; using OpenDDD.Domain.Model; using OpenDDD.Infrastructure.Events; using OpenDDD.Tests.Base; -namespace OpenDDD.Tests.Infrastructure.Events +namespace OpenDDD.Tests.Unit.Infrastructure.Events { public class IntegrationPublisherTests : UnitTests { diff --git a/src/OpenDDD.Tests/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs b/src/OpenDDD.Tests/Unit/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs similarity index 98% rename from src/OpenDDD.Tests/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs rename to src/OpenDDD.Tests/Unit/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs index 4c5224b..8d339bb 100644 --- a/src/OpenDDD.Tests/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs +++ b/src/OpenDDD.Tests/Unit/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs @@ -1,11 +1,11 @@ -using Microsoft.Extensions.Logging; +using Confluent.Kafka; +using Microsoft.Extensions.Logging; using Moq; using OpenDDD.Infrastructure.Events.Kafka; using OpenDDD.Infrastructure.Events.Kafka.Factories; using OpenDDD.Tests.Base; -using Confluent.Kafka; -namespace OpenDDD.Tests.Infrastructure.Events.Kafka +namespace OpenDDD.Tests.Unit.Infrastructure.Events.Kafka { public class KafkaMessagingProviderTests : UnitTests { diff --git a/src/OpenDDD.Tests/Infrastructure/Events/RabbitMq/RabbitMqMessagingProviderTests.cs b/src/OpenDDD.Tests/Unit/Infrastructure/Events/RabbitMq/RabbitMqMessagingProviderTests.cs similarity index 98% rename from src/OpenDDD.Tests/Infrastructure/Events/RabbitMq/RabbitMqMessagingProviderTests.cs rename to src/OpenDDD.Tests/Unit/Infrastructure/Events/RabbitMq/RabbitMqMessagingProviderTests.cs index b66e46a..3bfc2a2 100644 --- a/src/OpenDDD.Tests/Infrastructure/Events/RabbitMq/RabbitMqMessagingProviderTests.cs +++ b/src/OpenDDD.Tests/Unit/Infrastructure/Events/RabbitMq/RabbitMqMessagingProviderTests.cs @@ -1,12 +1,11 @@ using Microsoft.Extensions.Logging; -using Xunit; using Moq; using OpenDDD.Infrastructure.Events.RabbitMq; using OpenDDD.Infrastructure.Events.RabbitMq.Factories; using OpenDDD.Tests.Base; using RabbitMQ.Client; -namespace OpenDDD.Tests.Infrastructure.Events.RabbitMq +namespace OpenDDD.Tests.Unit.Infrastructure.Events.RabbitMq { public class RabbitMqMessagingProviderTests : UnitTests { diff --git a/src/OpenDDD.Tests/Infrastructure/Events/TestEvent.cs b/src/OpenDDD.Tests/Unit/Infrastructure/Events/TestEvent.cs similarity index 90% rename from src/OpenDDD.Tests/Infrastructure/Events/TestEvent.cs rename to src/OpenDDD.Tests/Unit/Infrastructure/Events/TestEvent.cs index ecf00c0..048da8b 100644 --- a/src/OpenDDD.Tests/Infrastructure/Events/TestEvent.cs +++ b/src/OpenDDD.Tests/Unit/Infrastructure/Events/TestEvent.cs @@ -1,6 +1,6 @@ using OpenDDD.Domain.Model; -namespace OpenDDD.Tests.Infrastructure.Events +namespace OpenDDD.Tests.Unit.Infrastructure.Events { public class TestEvent : IDomainEvent { diff --git a/src/OpenDDD.Tests/Infrastructure/Persistence/OpenDdd/Expressions/JsonbExpressionParserTests.cs b/src/OpenDDD.Tests/Unit/Infrastructure/Persistence/OpenDdd/Expressions/JsonbExpressionParserTests.cs similarity index 98% rename from src/OpenDDD.Tests/Infrastructure/Persistence/OpenDdd/Expressions/JsonbExpressionParserTests.cs rename to src/OpenDDD.Tests/Unit/Infrastructure/Persistence/OpenDdd/Expressions/JsonbExpressionParserTests.cs index b43102e..8278f51 100644 --- a/src/OpenDDD.Tests/Infrastructure/Persistence/OpenDdd/Expressions/JsonbExpressionParserTests.cs +++ b/src/OpenDDD.Tests/Unit/Infrastructure/Persistence/OpenDdd/Expressions/JsonbExpressionParserTests.cs @@ -1,10 +1,9 @@ using System.Linq.Expressions; -using Xunit; using FluentAssertions; using OpenDDD.Infrastructure.Persistence.OpenDdd.Expressions; using OpenDDD.Tests.Base; -namespace OpenDDD.Tests.Infrastructure.Persistence.OpenDdd.Expressions +namespace OpenDDD.Tests.Unit.Infrastructure.Persistence.OpenDdd.Expressions { public class JsonbExpressionParserTests : UnitTests { diff --git a/src/OpenDDD.Tests/Infrastructure/Persistence/OpenDdd/Serializers/OpenDddAggregateSerializerTests.cs b/src/OpenDDD.Tests/Unit/Infrastructure/Persistence/OpenDdd/Serializers/OpenDddAggregateSerializerTests.cs similarity index 93% rename from src/OpenDDD.Tests/Infrastructure/Persistence/OpenDdd/Serializers/OpenDddAggregateSerializerTests.cs rename to src/OpenDDD.Tests/Unit/Infrastructure/Persistence/OpenDdd/Serializers/OpenDddAggregateSerializerTests.cs index 09b6769..eba8b40 100644 --- a/src/OpenDDD.Tests/Infrastructure/Persistence/OpenDdd/Serializers/OpenDddAggregateSerializerTests.cs +++ b/src/OpenDDD.Tests/Unit/Infrastructure/Persistence/OpenDdd/Serializers/OpenDddAggregateSerializerTests.cs @@ -1,9 +1,8 @@ -using Xunit; -using OpenDDD.Infrastructure.Persistence.OpenDdd.Serializers; +using OpenDDD.Infrastructure.Persistence.OpenDdd.Serializers; using OpenDDD.Tests.Base; -using OpenDDD.Tests.Domain.Model; +using OpenDDD.Tests.Base.Domain.Model; -namespace OpenDDD.Tests.Infrastructure.Persistence.OpenDdd.Serializers +namespace OpenDDD.Tests.Unit.Infrastructure.Persistence.OpenDdd.Serializers { public class OpenDddAggregateSerializerTests : UnitTests { diff --git a/src/OpenDDD.Tests/Infrastructure/Persistence/OpenDdd/Serializers/OpenDddSerializerTests.cs b/src/OpenDDD.Tests/Unit/Infrastructure/Persistence/OpenDdd/Serializers/OpenDddSerializerTests.cs similarity index 95% rename from src/OpenDDD.Tests/Infrastructure/Persistence/OpenDdd/Serializers/OpenDddSerializerTests.cs rename to src/OpenDDD.Tests/Unit/Infrastructure/Persistence/OpenDdd/Serializers/OpenDddSerializerTests.cs index 1320c83..7212305 100644 --- a/src/OpenDDD.Tests/Infrastructure/Persistence/OpenDdd/Serializers/OpenDddSerializerTests.cs +++ b/src/OpenDDD.Tests/Unit/Infrastructure/Persistence/OpenDdd/Serializers/OpenDddSerializerTests.cs @@ -1,8 +1,7 @@ using OpenDDD.Infrastructure.Persistence.OpenDdd.Serializers; using OpenDDD.Tests.Base; -using Xunit; -namespace OpenDDD.Tests.Infrastructure.Persistence.OpenDdd.Serializers +namespace OpenDDD.Tests.Unit.Infrastructure.Persistence.OpenDdd.Serializers { public class OpenDddSerializerTests : UnitTests { From 90ea7c75ea597887118e1669da82b0043c2c2328 Mon Sep 17 00:00:00 2001 From: David Runemalm Date: Tue, 4 Mar 2025 18:14:28 +0700 Subject: [PATCH 088/109] Increase timeout in azure test. --- .../Events/Azure/AzureServiceBusMessagingProviderTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/OpenDDD.Tests/Integration/Infrastructure/Events/Azure/AzureServiceBusMessagingProviderTests.cs b/src/OpenDDD.Tests/Integration/Infrastructure/Events/Azure/AzureServiceBusMessagingProviderTests.cs index defb370..1898fd8 100644 --- a/src/OpenDDD.Tests/Integration/Infrastructure/Events/Azure/AzureServiceBusMessagingProviderTests.cs +++ b/src/OpenDDD.Tests/Integration/Infrastructure/Events/Azure/AzureServiceBusMessagingProviderTests.cs @@ -16,7 +16,7 @@ public class AzureServiceBusMessagingProviderTests : IntegrationTests, IAsyncLif private readonly ILogger _logger; private readonly ServiceBusClient _serviceBusClient; private readonly AzureServiceBusMessagingProvider _messagingProvider; - private readonly CancellationTokenSource _cts = new(TimeSpan.FromSeconds(60)); + private readonly CancellationTokenSource _cts = new(TimeSpan.FromSeconds(120)); public AzureServiceBusMessagingProviderTests(ITestOutputHelper testOutputHelper) : base(testOutputHelper, enableLogging: true) From be2fb3f3b5117734f8b378a322351a3f679960e7 Mon Sep 17 00:00:00 2001 From: David Runemalm Date: Thu, 6 Mar 2025 17:07:14 +0700 Subject: [PATCH 089/109] Fix issue with kafka messaging provider not distributing messages among kafka consumers. --- Makefile | 18 +++++++ .../Kafka/KafkaMessagingProviderTests.cs | 9 ++-- .../Events/Kafka/Factories/KafkaConsumer.cs | 19 +++++-- .../Kafka/Factories/KafkaConsumerFactory.cs | 3 +- .../Events/Kafka/KafkaMessagingProvider.cs | 52 ++++++++++++------- 5 files changed, 72 insertions(+), 29 deletions(-) diff --git a/Makefile b/Makefile index 5eb43fb..1e239ab 100644 --- a/Makefile +++ b/Makefile @@ -431,6 +431,24 @@ kafka-broker-status: ##@Kafka Show Kafka broker status kafka-list-consumer-groups: ##@Kafka List active Kafka consumer groups @docker exec -it $(KAFKA_CONTAINER) /opt/kafka/bin/kafka-consumer-groups.sh --bootstrap-server localhost:9092 --list +.PHONY: kafka-describe-consumer-groups +kafka-describe-consumer-groups: ##@Kafka List detailed info for all consumer groups + @docker exec -it $(KAFKA_CONTAINER) kafka-consumer-groups.sh --bootstrap-server $(KAFKA_BROKER) --all-groups --describe + +.PHONY: kafka-describe-consumer-group +kafka-describe-consumer-group: ##@Kafka Describe Kafka consumer group (requires GROUP=) +ifndef GROUP + $(error Consumer group not specified. Usage: make kafka-describe-consumer-group GROUP=) +endif + @docker exec -it $(KAFKA_CONTAINER) kafka-consumer-groups.sh --bootstrap-server $(KAFKA_BROKER) --group $(GROUP) --describe + +.PHONY: kafka-check-lag +kafka-check-lag: ##@Kafka Check Kafka consumer lag for a group (requires GROUP=) +ifndef GROUP + $(error Consumer group not specified. Usage: make kafka-check-lag GROUP=) +endif + @docker exec -it $(KAFKA_CONTAINER) kafka-consumer-groups.sh --bootstrap-server $(KAFKA_BROKER) --group $(GROUP) --describe | grep -E 'TOPIC|LAG' + .PHONY: kafka-consume kafka-consume: ##@Kafka Consume messages from a Kafka topic (uses NAME) ifndef NAME diff --git a/src/OpenDDD.Tests/Integration/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs b/src/OpenDDD.Tests/Integration/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs index 8f7b6d1..cc22771 100644 --- a/src/OpenDDD.Tests/Integration/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs +++ b/src/OpenDDD.Tests/Integration/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs @@ -303,14 +303,14 @@ async Task MessageHandler(string msg, CancellationToken token) $"Expected only one consumer to receive the message, but {receivedMessages.Count} consumers received it."); } - [Fact(Skip = "Skipping this test temporarily due to not working with > 1 partitions.")] + [Fact] public async Task CompetingConsumers_ShouldDistributeEvenly_WhenMultipleConsumersInGroup() { // Arrange var topicName = $"test-topic-{Guid.NewGuid()}"; var consumerGroup = "test-consumer-group"; - var totalMessages = 10; - var numConsumers = 10; + var totalMessages = 100; + var numConsumers = 2; var variancePercentage = 0.1; // Allow 10% variance var perConsumerMessageCount = new ConcurrentDictionary(); // Track messages per consumer var allMessagesProcessed = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); @@ -318,6 +318,7 @@ public async Task CompetingConsumers_ShouldDistributeEvenly_WhenMultipleConsumer async Task CreateConsumer() { var consumerId = Guid.NewGuid(); + _logger.LogDebug($"Creating consumer with unique ID: {consumerId}"); async Task MessageHandler(string msg, CancellationToken token) { @@ -340,8 +341,6 @@ async Task MessageHandler(string msg, CancellationToken token) } await WaitForKafkaConsumerGroupStable(consumerGroup, _cts.Token); - - await Task.Delay(2000); // Act for (int i = 0; i < totalMessages; i++) diff --git a/src/OpenDDD/Infrastructure/Events/Kafka/Factories/KafkaConsumer.cs b/src/OpenDDD/Infrastructure/Events/Kafka/Factories/KafkaConsumer.cs index f0236c0..4da577a 100644 --- a/src/OpenDDD/Infrastructure/Events/Kafka/Factories/KafkaConsumer.cs +++ b/src/OpenDDD/Infrastructure/Events/Kafka/Factories/KafkaConsumer.cs @@ -1,5 +1,5 @@ -using Confluent.Kafka; -using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging; +using Confluent.Kafka; namespace OpenDDD.Infrastructure.Events.Kafka.Factories { @@ -28,7 +28,7 @@ public void Subscribe(string topic) _consumer.Subscribe(topic); SubscribedTopics.Add(topic); } - + public void StartProcessing(Func messageHandler, CancellationToken globalToken) { if (_consumerTask != null) return; @@ -85,6 +85,9 @@ public async Task StopProcessingAsync() await _consumerTask; _consumerTask = null; } + + _consumer.Close(); + _consumer.Dispose(); } public void Dispose() @@ -92,7 +95,15 @@ public void Dispose() if (_disposed) return; _disposed = true; - _consumer.Close(); + try + { + _consumer.Close(); + } + catch (ObjectDisposedException) + { + _logger.LogWarning("Attempted to close a disposed Kafka consumer."); + } + _consumer.Dispose(); _cts.Dispose(); } diff --git a/src/OpenDDD/Infrastructure/Events/Kafka/Factories/KafkaConsumerFactory.cs b/src/OpenDDD/Infrastructure/Events/Kafka/Factories/KafkaConsumerFactory.cs index f042ee3..4ac12e7 100644 --- a/src/OpenDDD/Infrastructure/Events/Kafka/Factories/KafkaConsumerFactory.cs +++ b/src/OpenDDD/Infrastructure/Events/Kafka/Factories/KafkaConsumerFactory.cs @@ -22,8 +22,9 @@ public virtual KafkaConsumer Create(string consumerGroup) var consumerConfig = new ConsumerConfig { BootstrapServers = _bootstrapServers, - ClientId = "OpenDDD", + ClientId = $"OpenDDD-{Guid.NewGuid()}", GroupId = consumerGroup, + PartitionAssignmentStrategy = PartitionAssignmentStrategy.RoundRobin, EnableAutoCommit = false, AutoOffsetReset = AutoOffsetReset.Latest, MaxPollIntervalMs = 300000, // Max time consumer can take to process a message before kafka removes it from group diff --git a/src/OpenDDD/Infrastructure/Events/Kafka/KafkaMessagingProvider.cs b/src/OpenDDD/Infrastructure/Events/Kafka/KafkaMessagingProvider.cs index aced766..aae533d 100644 --- a/src/OpenDDD/Infrastructure/Events/Kafka/KafkaMessagingProvider.cs +++ b/src/OpenDDD/Infrastructure/Events/Kafka/KafkaMessagingProvider.cs @@ -14,7 +14,7 @@ public class KafkaMessagingProvider : IMessagingProvider, IAsyncDisposable private readonly bool _autoCreateTopics; private readonly IKafkaConsumerFactory _consumerFactory; private readonly ILogger _logger; - private readonly ConcurrentDictionary _consumers = new(); + private readonly ConcurrentDictionary> _consumers = new(); private readonly CancellationTokenSource _cts = new(); private bool _disposed; @@ -73,31 +73,36 @@ public async Task SubscribeAsync( } } - var kafkaConsumer = _consumers.GetOrAdd(groupKey, _ => - { - var newConsumer = _consumerFactory.Create(consumerGroup); - newConsumer.Subscribe(topic); - return newConsumer; - }); + var consumers = _consumers.GetOrAdd(groupKey, _ => new ConcurrentBag()); + + var newConsumer = _consumerFactory.Create(consumerGroup); + newConsumer.Subscribe(topic); + consumers.Add(newConsumer); - _logger.LogDebug("Subscribed to Kafka topic '{Topic}' with consumer group '{ConsumerGroup}'", topic, consumerGroup); + _logger.LogDebug("Subscribed a new consumer to Kafka topic '{Topic}' with consumer group '{ConsumerGroup}'", topic, consumerGroup); - kafkaConsumer.StartProcessing(messageHandler, _cts.Token); + newConsumer.StartProcessing(messageHandler, _cts.Token); } public async Task UnsubscribeAsync(string topic, string consumerGroup, CancellationToken cancellationToken) { var groupKey = GetGroupKey(topic, consumerGroup); - if (!_consumers.TryRemove(groupKey, out var kafkaConsumer)) + if (!_consumers.TryGetValue(groupKey, out var consumers) || !consumers.Any()) { - _logger.LogWarning("No active consumer found for consumer group '{ConsumerGroup}' on topic '{Topic}'. It may have already stopped.", consumerGroup, topic); + _logger.LogWarning("No active consumers found for consumer group '{ConsumerGroup}' on topic '{Topic}'.", consumerGroup, topic); return; } - _logger.LogDebug("Stopping consumer for topic '{Topic}' and consumer group '{ConsumerGroup}'...", topic, consumerGroup); - await kafkaConsumer.StopProcessingAsync(); - kafkaConsumer.Dispose(); + _logger.LogDebug("Stopping all consumers for topic '{Topic}' and consumer group '{ConsumerGroup}'...", topic, consumerGroup); + + foreach (var consumer in consumers) + { + await consumer.StopProcessingAsync(); + consumer.Dispose(); + } + + _consumers.TryRemove(groupKey, out _); } public async Task PublishAsync(string topic, string message, CancellationToken cancellationToken) @@ -132,7 +137,7 @@ private async Task CreateTopicIfNotExistsAsync(string topic, CancellationToken c _logger.LogDebug("Creating Kafka topic: {Topic}", topic); await _adminClient.CreateTopicsAsync(new[] { - new TopicSpecification { Name = topic, NumPartitions = 1, ReplicationFactor = 1 } + new TopicSpecification { Name = topic, NumPartitions = 2, ReplicationFactor = 1 } }, null); for (int i = 0; i < 30; i++) @@ -169,15 +174,24 @@ public async ValueTask DisposeAsync() _logger.LogDebug("Disposing KafkaMessagingProvider..."); _cts.Cancel(); - - var tasks = _consumers.Values.Select(c => c.StopProcessingAsync()).ToList(); + + var tasks = _consumers.Values + .SelectMany(consumers => consumers) + .Select(c => c.StopProcessingAsync()) + .ToList(); + await Task.WhenAll(tasks); - foreach (var consumer in _consumers.Values) + foreach (var consumerList in _consumers.Values) { - consumer.Dispose(); + foreach (var consumer in consumerList) + { + consumer.Dispose(); + } } + _consumers.Clear(); + _producer.Dispose(); _adminClient.Dispose(); } From b2387ff6f493c7aaf5b5dbc3ada0d3492050606d Mon Sep 17 00:00:00 2001 From: David Runemalm Date: Thu, 6 Mar 2025 17:16:07 +0700 Subject: [PATCH 090/109] Rename test. --- .../AzureServiceBusMessagingProviderTests.cs | 2 +- .../InMemoryMessagingProviderTests.cs | 2 +- .../Kafka/KafkaMessagingProviderTests.cs | 2 +- .../RabbitMqMessagingProviderTests.cs | 2 +- .../Kafka/KafkaMessagingProviderTests.cs | 19 ------------------- .../Events/Kafka/KafkaMessagingProvider.cs | 2 +- 6 files changed, 5 insertions(+), 24 deletions(-) diff --git a/src/OpenDDD.Tests/Integration/Infrastructure/Events/Azure/AzureServiceBusMessagingProviderTests.cs b/src/OpenDDD.Tests/Integration/Infrastructure/Events/Azure/AzureServiceBusMessagingProviderTests.cs index 1898fd8..75362df 100644 --- a/src/OpenDDD.Tests/Integration/Infrastructure/Events/Azure/AzureServiceBusMessagingProviderTests.cs +++ b/src/OpenDDD.Tests/Integration/Infrastructure/Events/Azure/AzureServiceBusMessagingProviderTests.cs @@ -234,7 +234,7 @@ async Task MessageHandler(string msg, CancellationToken token) } [Fact] - public async Task CompetingConsumers_ShouldDistributeEvenly_WhenMultipleConsumersInGroup() + public async Task CompetingConsumers_ShouldDistributeMessages_WhenMultipleConsumersInGroup() { // Arrange var topicName = $"test-topic-{Guid.NewGuid()}"; diff --git a/src/OpenDDD.Tests/Integration/Infrastructure/Events/InMemory/InMemoryMessagingProviderTests.cs b/src/OpenDDD.Tests/Integration/Infrastructure/Events/InMemory/InMemoryMessagingProviderTests.cs index 2380d2a..49b716c 100644 --- a/src/OpenDDD.Tests/Integration/Infrastructure/Events/InMemory/InMemoryMessagingProviderTests.cs +++ b/src/OpenDDD.Tests/Integration/Infrastructure/Events/InMemory/InMemoryMessagingProviderTests.cs @@ -171,7 +171,7 @@ async Task MessageHandler(string msg, CancellationToken token) } [Fact] - public async Task CompetingConsumers_ShouldDistributeEvenly_WhenMultipleConsumersInGroup() + public async Task CompetingConsumers_ShouldDistributeMessages_WhenMultipleConsumersInGroup() { // Arrange var topicName = "test-topic-even-distribution"; diff --git a/src/OpenDDD.Tests/Integration/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs b/src/OpenDDD.Tests/Integration/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs index cc22771..b7a6047 100644 --- a/src/OpenDDD.Tests/Integration/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs +++ b/src/OpenDDD.Tests/Integration/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs @@ -304,7 +304,7 @@ async Task MessageHandler(string msg, CancellationToken token) } [Fact] - public async Task CompetingConsumers_ShouldDistributeEvenly_WhenMultipleConsumersInGroup() + public async Task CompetingConsumers_ShouldDistributeMessages_WhenMultipleConsumersInGroup() { // Arrange var topicName = $"test-topic-{Guid.NewGuid()}"; diff --git a/src/OpenDDD.Tests/Integration/Infrastructure/Events/RabbitMq/RabbitMqMessagingProviderTests.cs b/src/OpenDDD.Tests/Integration/Infrastructure/Events/RabbitMq/RabbitMqMessagingProviderTests.cs index ea9d796..c94e2c5 100644 --- a/src/OpenDDD.Tests/Integration/Infrastructure/Events/RabbitMq/RabbitMqMessagingProviderTests.cs +++ b/src/OpenDDD.Tests/Integration/Infrastructure/Events/RabbitMq/RabbitMqMessagingProviderTests.cs @@ -295,7 +295,7 @@ async Task MessageHandler(string msg, CancellationToken token) } [Fact] - public async Task CompetingConsumers_ShouldDistributeEvenly_WhenMultipleConsumersInGroup() + public async Task CompetingConsumers_ShouldDistributeMessages_WhenMultipleConsumersInGroup() { // Arrange var receivedMessages = new ConcurrentDictionary(); diff --git a/src/OpenDDD.Tests/Unit/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs b/src/OpenDDD.Tests/Unit/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs index 8d339bb..87bec7e 100644 --- a/src/OpenDDD.Tests/Unit/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs +++ b/src/OpenDDD.Tests/Unit/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs @@ -122,24 +122,5 @@ public async Task PublishAsync_ShouldThrowException_WhenMessageIsInvalid(string await Assert.ThrowsAsync(() => _provider.PublishAsync(Topic, invalidMessage, CancellationToken.None)); } - - [Fact] - public async Task DisposeAsync_ShouldDisposeAllConsumers_AndKafkaClients() - { - // Arrange - await _provider.SubscribeAsync(Topic, ConsumerGroup, async (_, _) => await Task.CompletedTask, CancellationToken.None); - - // Ensure the background task has enough time to start - await Task.Delay(100); - - // Act - await _provider.DisposeAsync(); - - // Assert - _mockConsumer.Verify(c => c.Close(), Times.Once); - _mockConsumer.Verify(c => c.Dispose(), Times.Once); - _mockProducer.Verify(p => p.Dispose(), Times.Once); - _mockAdminClient.Verify(a => a.Dispose(), Times.Once); - } } } diff --git a/src/OpenDDD/Infrastructure/Events/Kafka/KafkaMessagingProvider.cs b/src/OpenDDD/Infrastructure/Events/Kafka/KafkaMessagingProvider.cs index aae533d..bd18fe1 100644 --- a/src/OpenDDD/Infrastructure/Events/Kafka/KafkaMessagingProvider.cs +++ b/src/OpenDDD/Infrastructure/Events/Kafka/KafkaMessagingProvider.cs @@ -137,7 +137,7 @@ private async Task CreateTopicIfNotExistsAsync(string topic, CancellationToken c _logger.LogDebug("Creating Kafka topic: {Topic}", topic); await _adminClient.CreateTopicsAsync(new[] { - new TopicSpecification { Name = topic, NumPartitions = 2, ReplicationFactor = 1 } + new TopicSpecification { Name = topic, NumPartitions = 1, ReplicationFactor = 1 } }, null); for (int i = 0; i < 30; i++) From 6baa4f870437503673d103d9b02c7620731a52da Mon Sep 17 00:00:00 2001 From: David Runemalm Date: Thu, 6 Mar 2025 17:27:05 +0700 Subject: [PATCH 091/109] Change kafka default partition count. --- .../Infrastructure/Events/Kafka/KafkaMessagingProvider.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/OpenDDD/Infrastructure/Events/Kafka/KafkaMessagingProvider.cs b/src/OpenDDD/Infrastructure/Events/Kafka/KafkaMessagingProvider.cs index bd18fe1..aae533d 100644 --- a/src/OpenDDD/Infrastructure/Events/Kafka/KafkaMessagingProvider.cs +++ b/src/OpenDDD/Infrastructure/Events/Kafka/KafkaMessagingProvider.cs @@ -137,7 +137,7 @@ private async Task CreateTopicIfNotExistsAsync(string topic, CancellationToken c _logger.LogDebug("Creating Kafka topic: {Topic}", topic); await _adminClient.CreateTopicsAsync(new[] { - new TopicSpecification { Name = topic, NumPartitions = 1, ReplicationFactor = 1 } + new TopicSpecification { Name = topic, NumPartitions = 2, ReplicationFactor = 1 } }, null); for (int i = 0; i < 30; i++) From a6eb9c2cf8c536f91eb021efad1db69742c580f3 Mon Sep 17 00:00:00 2001 From: David Runemalm Date: Fri, 7 Mar 2025 16:03:14 +0700 Subject: [PATCH 092/109] Refactor tests. --- .../AzureServiceBusMessagingProviderTests.cs | 4 +- .../InMemoryMessagingProviderTests.cs | 6 +- .../Kafka/KafkaMessagingProviderTests.cs | 14 +- .../RabbitMqMessagingProviderTests.cs | 4 +- .../InMemoryMessagingProviderTests.cs | 2 +- .../Kafka/KafkaMessagingProviderTests.cs | 15 +- .../OpenDddServiceCollectionExtensions.cs | 1 - .../Azure/AzureServiceBusMessagingProvider.cs | 62 ++++----- .../Azure/AzureServiceBusSubscription.cs | 23 +++ .../Events/Base/ISubscription.cs | 9 ++ .../Events/Base/Subscription.cs | 23 +++ .../Events/IMessagingProvider.cs | 13 +- .../InMemory/InMemoryMessagingProvider.cs | 131 +++++++++--------- .../Events/InMemory/InMemorySubscription.cs | 26 ++++ .../Events/Kafka/Factories/KafkaConsumer.cs | 8 +- .../Events/Kafka/KafkaMessagingProvider.cs | 90 +++++------- .../Events/Kafka/KafkaSubscription.cs | 16 +++ .../RabbitMq/RabbitMqCustomAsyncConsumer.cs | 31 ++++- .../RabbitMq/RabbitMqMessagingProvider.cs | 49 ++++--- .../Events/RabbitMq/RabbitMqSubscription.cs | 16 +++ 20 files changed, 326 insertions(+), 217 deletions(-) create mode 100644 src/OpenDDD/Infrastructure/Events/Azure/AzureServiceBusSubscription.cs create mode 100644 src/OpenDDD/Infrastructure/Events/Base/ISubscription.cs create mode 100644 src/OpenDDD/Infrastructure/Events/Base/Subscription.cs create mode 100644 src/OpenDDD/Infrastructure/Events/InMemory/InMemorySubscription.cs create mode 100644 src/OpenDDD/Infrastructure/Events/Kafka/KafkaSubscription.cs create mode 100644 src/OpenDDD/Infrastructure/Events/RabbitMq/RabbitMqSubscription.cs diff --git a/src/OpenDDD.Tests/Integration/Infrastructure/Events/Azure/AzureServiceBusMessagingProviderTests.cs b/src/OpenDDD.Tests/Integration/Infrastructure/Events/Azure/AzureServiceBusMessagingProviderTests.cs index 75362df..01e6ace 100644 --- a/src/OpenDDD.Tests/Integration/Infrastructure/Events/Azure/AzureServiceBusMessagingProviderTests.cs +++ b/src/OpenDDD.Tests/Integration/Infrastructure/Events/Azure/AzureServiceBusMessagingProviderTests.cs @@ -121,14 +121,14 @@ public async Task AtLeastOnceGurantee_ShouldDeliverToLateSubscriber_WhenSubscrib var messageToSend = "Persistent Message Test"; var messageReceivedTcs = new TaskCompletionSource(); - await _messagingProvider.SubscribeAsync(topicName, subscriptionName, async (msg, token) => + var firstSubscription = await _messagingProvider.SubscribeAsync(topicName, subscriptionName, async (msg, token) => { Assert.Fail("First subscription should not receive the message."); }, _cts.Token); await Task.Delay(500); - await _messagingProvider.UnsubscribeAsync(topicName, subscriptionName, _cts.Token); + await _messagingProvider.UnsubscribeAsync(firstSubscription, _cts.Token); await Task.Delay(500); diff --git a/src/OpenDDD.Tests/Integration/Infrastructure/Events/InMemory/InMemoryMessagingProviderTests.cs b/src/OpenDDD.Tests/Integration/Infrastructure/Events/InMemory/InMemoryMessagingProviderTests.cs index 49b716c..6eb9e34 100644 --- a/src/OpenDDD.Tests/Integration/Infrastructure/Events/InMemory/InMemoryMessagingProviderTests.cs +++ b/src/OpenDDD.Tests/Integration/Infrastructure/Events/InMemory/InMemoryMessagingProviderTests.cs @@ -40,14 +40,14 @@ public async Task AtLeastOnceGuarantee_ShouldDeliverToLateSubscriber_WhenSubscri var lateSubscriberReceived = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - await _messagingProvider.SubscribeAsync(topicName, groupName, async (msg, token) => + var firstSubscription = await _messagingProvider.SubscribeAsync(topicName, groupName, async (msg, token) => { Assert.Fail("First subscription should not receive the message."); }, _cts.Token); await Task.Delay(500); - await _messagingProvider.UnsubscribeAsync(topicName, groupName, _cts.Token); + await _messagingProvider.UnsubscribeAsync(firstSubscription, _cts.Token); // Act await _messagingProvider.PublishAsync(topicName, messageToSend, _cts.Token); @@ -178,7 +178,7 @@ public async Task CompetingConsumers_ShouldDistributeMessages_WhenMultipleConsum var consumerGroup = "test-consumer-group"; var totalMessages = 100; var numConsumers = 2; - var variancePercentage = 0.2; + var variancePercentage = 0.3; var perConsumerMessageCount = new ConcurrentDictionary(); var allMessagesProcessed = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); diff --git a/src/OpenDDD.Tests/Integration/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs b/src/OpenDDD.Tests/Integration/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs index b7a6047..4a43935 100644 --- a/src/OpenDDD.Tests/Integration/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs +++ b/src/OpenDDD.Tests/Integration/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs @@ -37,7 +37,6 @@ public KafkaMessagingProviderTests(ITestOutputHelper testOutputHelper) _consumerFactory = new KafkaConsumerFactory(_bootstrapServers, _consumerLogger); _messagingProvider = new KafkaMessagingProvider( - _bootstrapServers, _adminClient, _producer, _consumerFactory, @@ -98,7 +97,6 @@ public async Task AutoCreateTopic_ShouldCreateTopicOnSubscribe_WhenSettingEnable var topicName = $"test-topic-{Guid.NewGuid()}"; var consumerGroup = "test-consumer-group"; - // Ensure topic does not exist var metadata = _adminClient.GetMetadata(TimeSpan.FromSeconds(5)); metadata.Topics.Any(t => t.Topic == topicName).Should().BeFalse(); @@ -134,7 +132,6 @@ public async Task AutoCreateTopic_ShouldNotCreateTopicOnSubscribe_WhenSettingDis var consumerGroup = "test-consumer-group"; var messagingProvider = new KafkaMessagingProvider( - _bootstrapServers, _adminClient, _producer, _consumerFactory, @@ -162,7 +159,7 @@ public async Task AtLeastOnceGurantee_ShouldDeliverToLateSubscriber_WhenSubscrib var lateSubscriberReceived = new TaskCompletionSource(); ConcurrentBag _receivedMessages = new(); - await _messagingProvider.SubscribeAsync(topicName, consumerGroup, async (msg, token) => + var firstSubscription = await _messagingProvider.SubscribeAsync(topicName, consumerGroup, async (msg, token) => { firstSubscriberReceived.SetResult(true); }, _cts.Token); @@ -171,7 +168,7 @@ await _messagingProvider.SubscribeAsync(topicName, consumerGroup, async (msg, to await _messagingProvider.PublishAsync(topicName, messageToSend, _cts.Token); await firstSubscriberReceived.Task.WaitAsync(TimeSpan.FromSeconds(10)); - await _messagingProvider.UnsubscribeAsync(topicName, consumerGroup, _cts.Token); + await _messagingProvider.UnsubscribeAsync(firstSubscription, _cts.Token); await Task.Delay(5000); @@ -276,7 +273,6 @@ async Task MessageHandler(string msg, CancellationToken token) messageProcessed.TrySetResult(true); } - // Subscribe multiple consumers in the same group await _messagingProvider.SubscribeAsync(topicName, consumerGroup, MessageHandler, _cts.Token); await _messagingProvider.SubscribeAsync(topicName, consumerGroup, MessageHandler, _cts.Token); @@ -295,10 +291,9 @@ async Task MessageHandler(string msg, CancellationToken token) Assert.Fail("No consumer received the message."); } - // Wait a little longer to ensure no second consumer processes the same message await Task.Delay(5000); - // Assert: Exactly one consumer should have received the message + // Assert receivedMessages.Count.Should().Be(1, $"Expected only one consumer to receive the message, but {receivedMessages.Count} consumers received it."); } @@ -311,7 +306,7 @@ public async Task CompetingConsumers_ShouldDistributeMessages_WhenMultipleConsum var consumerGroup = "test-consumer-group"; var totalMessages = 100; var numConsumers = 2; - var variancePercentage = 0.1; // Allow 10% variance + var variancePercentage = 0.2; var perConsumerMessageCount = new ConcurrentDictionary(); // Track messages per consumer var allMessagesProcessed = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); @@ -334,7 +329,6 @@ async Task MessageHandler(string msg, CancellationToken token) await _messagingProvider.SubscribeAsync(topicName, consumerGroup, MessageHandler, _cts.Token); } - // Subscribe multiple consumers for (int i = 0; i < numConsumers; i++) { await CreateConsumer(); diff --git a/src/OpenDDD.Tests/Integration/Infrastructure/Events/RabbitMq/RabbitMqMessagingProviderTests.cs b/src/OpenDDD.Tests/Integration/Infrastructure/Events/RabbitMq/RabbitMqMessagingProviderTests.cs index c94e2c5..6302fb4 100644 --- a/src/OpenDDD.Tests/Integration/Infrastructure/Events/RabbitMq/RabbitMqMessagingProviderTests.cs +++ b/src/OpenDDD.Tests/Integration/Infrastructure/Events/RabbitMq/RabbitMqMessagingProviderTests.cs @@ -160,13 +160,13 @@ public async Task AtLeastOnceGurantee_ShouldDeliverToLateSubscriber_WhenSubscrib var lateSubscriberReceived = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - await _messagingProvider.SubscribeAsync(_testTopic, _testConsumerGroup, async (msg, token) => + var firstSubscription = await _messagingProvider.SubscribeAsync(_testTopic, _testConsumerGroup, async (msg, token) => { Assert.Fail("First subscription should not receive the message."); }, _cts.Token); await Task.Delay(500); - await _messagingProvider.UnsubscribeAsync(_testTopic, _testConsumerGroup, _cts.Token); + await _messagingProvider.UnsubscribeAsync(firstSubscription, _cts.Token); await Task.Delay(500); // Act diff --git a/src/OpenDDD.Tests/Unit/Infrastructure/Events/InMemory/InMemoryMessagingProviderTests.cs b/src/OpenDDD.Tests/Unit/Infrastructure/Events/InMemory/InMemoryMessagingProviderTests.cs index 4f55f9e..a666d2b 100644 --- a/src/OpenDDD.Tests/Unit/Infrastructure/Events/InMemory/InMemoryMessagingProviderTests.cs +++ b/src/OpenDDD.Tests/Unit/Infrastructure/Events/InMemory/InMemoryMessagingProviderTests.cs @@ -52,7 +52,7 @@ await Assert.ThrowsAsync(() => [Fact] public async Task SubscribeAsync_ShouldThrowException_WhenHandlerIsNull() { - await Assert.ThrowsAsync(() => + await Assert.ThrowsAsync(() => _messagingProvider.SubscribeAsync(Topic, ConsumerGroup, null!, CancellationToken.None)); } diff --git a/src/OpenDDD.Tests/Unit/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs b/src/OpenDDD.Tests/Unit/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs index 87bec7e..181c2f3 100644 --- a/src/OpenDDD.Tests/Unit/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs +++ b/src/OpenDDD.Tests/Unit/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs @@ -31,7 +31,6 @@ public KafkaMessagingProviderTests() _mockConsumerLogger = new Mock>(); _provider = new KafkaMessagingProvider( - BootstrapServers, _mockAdminClient.Object, _mockProducer.Object, _mockConsumerFactory.Object, @@ -58,22 +57,20 @@ public KafkaMessagingProviderTests() } [Theory] - [InlineData(null, "adminClient", "producer", "consumerFactory", "logger")] - [InlineData("bootstrapServers", null, "producer", "consumerFactory", "logger")] - [InlineData("bootstrapServers", "adminClient", null, "consumerFactory", "logger")] - [InlineData("bootstrapServers", "adminClient", "producer", null, "logger")] - [InlineData("bootstrapServers", "adminClient", "producer", "consumerFactory", null)] + [InlineData(null, "producer", "consumerFactory", "logger")] + [InlineData("adminClient", null, "consumerFactory", "logger")] + [InlineData("adminClient", "producer", null, "logger")] + [InlineData("adminClient", "producer", "consumerFactory", null)] public void Constructor_ShouldThrowException_WhenDependenciesAreNull( - string? bootstrapServers, string? adminClient, string? producer, string? consumerFactory, string? logger) + string? adminClient, string? producer, string? consumerFactory, string? logger) { - var bs = bootstrapServers is null ? null! : BootstrapServers; var mockAdmin = adminClient is null ? null! : _mockAdminClient.Object; var mockProducer = producer is null ? null! : _mockProducer.Object; var mockConsumerFactory = consumerFactory is null ? null! : _mockConsumerFactory.Object; var mockLogger = logger is null ? null! : _mockLogger.Object; Assert.Throws(() => - new KafkaMessagingProvider(bs, mockAdmin, mockProducer, mockConsumerFactory, true, mockLogger)); + new KafkaMessagingProvider(mockAdmin, mockProducer, mockConsumerFactory, true, mockLogger)); } [Theory] diff --git a/src/OpenDDD/API/Extensions/OpenDddServiceCollectionExtensions.cs b/src/OpenDDD/API/Extensions/OpenDddServiceCollectionExtensions.cs index 53dc91c..6bc2856 100644 --- a/src/OpenDDD/API/Extensions/OpenDddServiceCollectionExtensions.cs +++ b/src/OpenDDD/API/Extensions/OpenDddServiceCollectionExtensions.cs @@ -340,7 +340,6 @@ private static void AddKafka(this IServiceCollection services) var logger = provider.GetRequiredService>(); var consumerLogger = provider.GetRequiredService>(); return new KafkaMessagingProvider( - kafkaOptions.BootstrapServers, new AdminClientBuilder(new AdminClientConfig { BootstrapServers = kafkaOptions.BootstrapServers, ClientId = "OpenDDD" }).Build(), new ProducerBuilder(new ProducerConfig { BootstrapServers = kafkaOptions.BootstrapServers, ClientId = "OpenDDD" }).Build(), new KafkaConsumerFactory(kafkaOptions.BootstrapServers, consumerLogger), diff --git a/src/OpenDDD/Infrastructure/Events/Azure/AzureServiceBusMessagingProvider.cs b/src/OpenDDD/Infrastructure/Events/Azure/AzureServiceBusMessagingProvider.cs index fc25fd4..aeb0515 100644 --- a/src/OpenDDD/Infrastructure/Events/Azure/AzureServiceBusMessagingProvider.cs +++ b/src/OpenDDD/Infrastructure/Events/Azure/AzureServiceBusMessagingProvider.cs @@ -1,4 +1,6 @@ -using Microsoft.Extensions.Logging; +using System.Collections.Concurrent; +using Microsoft.Extensions.Logging; +using OpenDDD.Infrastructure.Events.Base; using Azure.Messaging.ServiceBus; using Azure.Messaging.ServiceBus.Administration; @@ -10,7 +12,7 @@ public class AzureServiceBusMessagingProvider : IMessagingProvider, IAsyncDispos private readonly ServiceBusAdministrationClient _adminClient; private readonly bool _autoCreateTopics; private readonly ILogger _logger; - private readonly List _processors = new(); + private readonly ConcurrentDictionary _subscriptions = new(); private bool _disposed; public AzureServiceBusMessagingProvider( @@ -25,7 +27,7 @@ public AzureServiceBusMessagingProvider( _logger = logger ?? throw new ArgumentNullException(nameof(logger)); } - public async Task SubscribeAsync(string topic, string consumerGroup, Func messageHandler, CancellationToken cancellationToken = default) + public async Task SubscribeAsync(string topic, string consumerGroup, Func messageHandler, CancellationToken cancellationToken = default) { if (string.IsNullOrWhiteSpace(topic)) { @@ -59,7 +61,6 @@ public async Task SubscribeAsync(string topic, string consumerGroup, Func { @@ -73,41 +74,30 @@ public async Task SubscribeAsync(string topic, string consumerGroup, Func - p.EntityPath.Equals($"{topic}/Subscriptions/{subscriptionName}", StringComparison.OrdinalIgnoreCase)); - - if (processor != null) - { - _processors.Remove(processor); - _logger.LogInformation("Stopping and disposing message processor for topic '{Topic}' and subscription '{Subscription}'", topic, subscriptionName); + _logger.LogInformation("Unsubscribing from Azure Service Bus topic '{Topic}' and subscription '{Subscription}', Subscription ID: {SubscriptionId}", serviceBusSubscription.Topic, serviceBusSubscription.ConsumerGroup, serviceBusSubscription.Id); - await processor.StopProcessingAsync(cancellationToken); - await processor.DisposeAsync(); - } - else - { - _logger.LogWarning("No active subscription found for topic '{Topic}' and subscription '{Subscription}'", topic, subscriptionName); - } + await serviceBusSubscription.Consumer.StopProcessingAsync(cancellationToken); + await serviceBusSubscription.DisposeAsync(); } public async Task PublishAsync(string topic, string message, CancellationToken cancellationToken = default) @@ -157,14 +147,16 @@ public async ValueTask DisposeAsync() _logger.LogDebug("Disposing AzureServiceBusMessagingProvider..."); - foreach (var processor in _processors) + foreach (var subscription in _subscriptions.Values) { - _logger.LogDebug("Stopping message processor..."); - await processor.StopProcessingAsync(); - await processor.DisposeAsync(); + if (subscription.Consumer.IsProcessing) + { + await subscription.Consumer.StopProcessingAsync(CancellationToken.None); + } + await subscription.DisposeAsync(); } - _logger.LogDebug("Disposing ServiceBusClient..."); + _subscriptions.Clear(); await _client.DisposeAsync(); } } diff --git a/src/OpenDDD/Infrastructure/Events/Azure/AzureServiceBusSubscription.cs b/src/OpenDDD/Infrastructure/Events/Azure/AzureServiceBusSubscription.cs new file mode 100644 index 0000000..7495d5b --- /dev/null +++ b/src/OpenDDD/Infrastructure/Events/Azure/AzureServiceBusSubscription.cs @@ -0,0 +1,23 @@ +using OpenDDD.Infrastructure.Events.Base; +using Azure.Messaging.ServiceBus; + +namespace OpenDDD.Infrastructure.Events.Azure +{ + public class AzureServiceBusSubscription : Subscription + { + public AzureServiceBusSubscription(string topic, string consumerGroup, ServiceBusProcessor processor) + : base(topic, consumerGroup, processor) + { + + } + + public override async ValueTask DisposeAsync() + { + if (Consumer.IsProcessing) + { + await Consumer.StopProcessingAsync(); + } + await Consumer.DisposeAsync(); + } + } +} diff --git a/src/OpenDDD/Infrastructure/Events/Base/ISubscription.cs b/src/OpenDDD/Infrastructure/Events/Base/ISubscription.cs new file mode 100644 index 0000000..2bb6df7 --- /dev/null +++ b/src/OpenDDD/Infrastructure/Events/Base/ISubscription.cs @@ -0,0 +1,9 @@ +namespace OpenDDD.Infrastructure.Events.Base +{ + public interface ISubscription : IAsyncDisposable + { + string Id { get; } + string Topic { get; } + string ConsumerGroup { get; } + } +} diff --git a/src/OpenDDD/Infrastructure/Events/Base/Subscription.cs b/src/OpenDDD/Infrastructure/Events/Base/Subscription.cs new file mode 100644 index 0000000..72a8dbb --- /dev/null +++ b/src/OpenDDD/Infrastructure/Events/Base/Subscription.cs @@ -0,0 +1,23 @@ +namespace OpenDDD.Infrastructure.Events.Base +{ + public class Subscription : ISubscription where TConsumer : IAsyncDisposable + { + public string Id { get; } = Guid.NewGuid().ToString(); + public string Topic { get; } + public string ConsumerGroup { get; } + public TConsumer? Consumer { get; } + + public Subscription(string topic, string consumerGroup, TConsumer? consumer) + { + Topic = topic ?? throw new ArgumentException(nameof(topic)); + ConsumerGroup = consumerGroup ?? throw new ArgumentException(nameof(consumerGroup)); + Consumer = consumer; + } + + public virtual async ValueTask DisposeAsync() + { + await Consumer.DisposeAsync(); + await ValueTask.CompletedTask; + } + } +} diff --git a/src/OpenDDD/Infrastructure/Events/IMessagingProvider.cs b/src/OpenDDD/Infrastructure/Events/IMessagingProvider.cs index 7454a50..a930fc3 100644 --- a/src/OpenDDD/Infrastructure/Events/IMessagingProvider.cs +++ b/src/OpenDDD/Infrastructure/Events/IMessagingProvider.cs @@ -1,9 +1,16 @@ -namespace OpenDDD.Infrastructure.Events +using OpenDDD.Infrastructure.Events.Base; + +namespace OpenDDD.Infrastructure.Events { public interface IMessagingProvider { - Task SubscribeAsync(string topic, string consumerGroup, Func messageHandler, CancellationToken cancellationToken = default); - Task UnsubscribeAsync(string topic, string consumerGroup, CancellationToken cancellationToken = default); + Task SubscribeAsync( + string topic, + string consumerGroup, + Func messageHandler, + CancellationToken cancellationToken = default); + + Task UnsubscribeAsync(ISubscription subscription, CancellationToken cancellationToken = default); Task PublishAsync(string topic, string message, CancellationToken cancellationToken = default); } } diff --git a/src/OpenDDD/Infrastructure/Events/InMemory/InMemoryMessagingProvider.cs b/src/OpenDDD/Infrastructure/Events/InMemory/InMemoryMessagingProvider.cs index a912304..586f912 100644 --- a/src/OpenDDD/Infrastructure/Events/InMemory/InMemoryMessagingProvider.cs +++ b/src/OpenDDD/Infrastructure/Events/InMemory/InMemoryMessagingProvider.cs @@ -1,13 +1,14 @@ using System.Collections.Concurrent; using Microsoft.Extensions.Logging; +using OpenDDD.Infrastructure.Events.Base; namespace OpenDDD.Infrastructure.Events.InMemory { public class InMemoryMessagingProvider : IMessagingProvider, IAsyncDisposable { private readonly ConcurrentDictionary> _messageLog = new(); - private readonly ConcurrentDictionary _consumerOffsets = new(); - private readonly ConcurrentDictionary>> _subscribers = new(); + private readonly ConcurrentDictionary _consumerGroupOffsets = new(); + private readonly ConcurrentDictionary _subscriptions = new(); private readonly ConcurrentQueue<(string Topic, string Message, string ConsumerGroup, int RetryCount)> _retryQueue = new(); private readonly ILogger _logger; private readonly TimeSpan _initialRetryDelay = TimeSpan.FromSeconds(1); @@ -21,65 +22,57 @@ public InMemoryMessagingProvider(ILogger logger) _retryTask = Task.Run(ProcessRetries, _cts.Token); // Start retry processing loop } - private static string GetGroupKey(string topic, string consumerGroup) - { - if (string.IsNullOrWhiteSpace(topic)) - throw new ArgumentException("Topic cannot be null or empty.", nameof(topic)); - - if (string.IsNullOrWhiteSpace(consumerGroup)) - throw new ArgumentException("Consumer group cannot be null or empty.", nameof(consumerGroup)); - - return $"{topic}:{consumerGroup}"; - } - - public Task SubscribeAsync(string topic, string consumerGroup, Func messageHandler, CancellationToken ct) + public async Task SubscribeAsync(string topic, string consumerGroup, Func messageHandler, CancellationToken ct = default) { if (messageHandler is null) - throw new ArgumentNullException(nameof(messageHandler)); + throw new ArgumentException(nameof(messageHandler)); + + var subscription = new InMemorySubscription(topic, consumerGroup, messageHandler); + _subscriptions[subscription.Id] = subscription; - var groupKey = GetGroupKey(topic, consumerGroup); - var handlers = _subscribers.GetOrAdd(groupKey, _ => new ConcurrentBag>()); + _logger.LogDebug("Subscribed to topic: {Topic} in listener group: {ConsumerGroup}, Subscription ID: {SubscriptionId}", + topic, consumerGroup, subscription.Id); - handlers.Add(messageHandler); - _logger.LogDebug("Subscribed to topic: {Topic} in listener group: {ConsumerGroup}", topic, consumerGroup); + var groupKey = $"{topic}:{consumerGroup}"; - if (!_consumerOffsets.ContainsKey(groupKey)) + if (!_consumerGroupOffsets.ContainsKey(groupKey)) { - var messageCount = _messageLog.TryGetValue(topic, out var messages) ? messages.Count : 0; - _consumerOffsets[groupKey] = messageCount; - _logger.LogDebug("Consumer group '{ConsumerGroup}' is subscribing for the first time, starting at offset {Offset}.", consumerGroup, messageCount); - return Task.CompletedTask; + _consumerGroupOffsets[groupKey] = _messageLog.TryGetValue(topic, out var messages) ? messages.Count : 0; + _logger.LogDebug("First subscription in consumer group '{ConsumerGroup}', starting at offset {Offset}.", + consumerGroup, _consumerGroupOffsets[groupKey]); } - - if (_messageLog.TryGetValue(topic, out var storedMessages)) + else { - var offset = _consumerOffsets[groupKey]; - var unseenMessages = storedMessages.Skip(offset).ToList(); - - foreach (var msg in unseenMessages) + var offset = _consumerGroupOffsets[groupKey]; + if (_messageLog.TryGetValue(topic, out var storedMessages)) { - _ = Task.Run(async () => await messageHandler(msg, ct), ct); - _consumerOffsets[groupKey]++; + var unseenMessages = storedMessages.Skip(offset).ToList(); + foreach (var msg in unseenMessages) + { + _ = Task.Run(async () => await messageHandler(msg, ct), ct); + _consumerGroupOffsets[groupKey]++; + } } } - return Task.CompletedTask; + return subscription; } - public Task UnsubscribeAsync(string topic, string consumerGroup, CancellationToken cancellationToken = default) + public async Task UnsubscribeAsync(ISubscription subscription, CancellationToken cancellationToken = default) { - var groupKey = GetGroupKey(topic, consumerGroup); + if (subscription == null) + throw new ArgumentNullException(nameof(subscription)); - if (_subscribers.TryRemove(groupKey, out _)) + if (!_subscriptions.TryRemove(subscription.Id, out _)) { - _logger.LogDebug("Unsubscribed from topic: {Topic} in listener group: {ConsumerGroup}", topic, consumerGroup); - } - else - { - _logger.LogWarning("No active subscriptions found for topic: {Topic} in listener group: {ConsumerGroup}", topic, consumerGroup); + _logger.LogWarning("No active subscription found with ID {SubscriptionId}", subscription.Id); + return; } - return Task.CompletedTask; + _logger.LogDebug("Unsubscribed from topic: {Topic} in listener group: {ConsumerGroup}, Subscription ID: {SubscriptionId}", + subscription.Topic, subscription.ConsumerGroup, subscription.Id); + + await subscription.DisposeAsync(); } public Task PublishAsync(string topic, string message, CancellationToken ct) @@ -96,28 +89,28 @@ public Task PublishAsync(string topic, string message, CancellationToken ct) messages.Add(message); } - var matchingGroups = _subscribers.Keys.Where(key => key.StartsWith($"{topic}:")).ToList(); + var groupSubscriptions = _subscriptions.Values + .Where(s => s.Topic == topic) + .GroupBy(s => s.ConsumerGroup); - foreach (var groupKey in matchingGroups) + foreach (var group in groupSubscriptions) { - if (_subscribers.TryGetValue(groupKey, out var handlers) && handlers.Any()) - { - var handler = handlers.OrderBy(_ => Guid.NewGuid()).First(); + var subscription = group.OrderBy(_ => Guid.NewGuid()).FirstOrDefault(); + if (subscription == null) continue; - _ = Task.Run(async () => + _ = Task.Run(async () => + { + try { - try - { - await handler(message, ct); - _consumerOffsets.AddOrUpdate(groupKey, 0, (_, currentOffset) => currentOffset + 1); // Update offset - } - catch (Exception ex) - { - _logger.LogError(ex, $"Error in handler for topic '{topic}': {ex.Message}"); - _retryQueue.Enqueue((topic, message, groupKey.Split(':')[1], 1)); - } - }, ct); - } + await subscription.MessageHandler(message, ct); + _consumerGroupOffsets.AddOrUpdate($"{topic}:{subscription.ConsumerGroup}", 0, (_, currentOffset) => currentOffset + 1); + } + catch (Exception ex) + { + _logger.LogError(ex, $"Error in handler for topic '{topic}' in consumer group '{subscription.ConsumerGroup}': {ex.Message}"); + _retryQueue.Enqueue((topic, message, subscription.ConsumerGroup, 1)); + } + }, ct); } return Task.CompletedTask; @@ -137,16 +130,14 @@ private async Task ProcessRetries() continue; } - var groupKey = GetGroupKey(topic, consumerGroup); - if (_subscribers.TryGetValue(groupKey, out var handlers) && handlers.Any()) + var subscription = _subscriptions.Values.FirstOrDefault(s => s.Topic == topic && s.ConsumerGroup == consumerGroup); + if (subscription != null) { - var handler = handlers.OrderBy(_ => Guid.NewGuid()).First(); - - await Task.Delay(ComputeBackoff(retryCount), _cts.Token); // Exponential backoff + await Task.Delay(ComputeBackoff(retryCount), _cts.Token); try { - await handler(message, _cts.Token); + await subscription.MessageHandler(message, _cts.Token); } catch (Exception ex) { @@ -170,6 +161,14 @@ private TimeSpan ComputeBackoff(int retryCount) public async ValueTask DisposeAsync() { _cts.Cancel(); + + foreach (var subscription in _subscriptions.Values) + { + await subscription.DisposeAsync(); + } + + _subscriptions.Clear(); + try { await _retryTask; diff --git a/src/OpenDDD/Infrastructure/Events/InMemory/InMemorySubscription.cs b/src/OpenDDD/Infrastructure/Events/InMemory/InMemorySubscription.cs new file mode 100644 index 0000000..35422d7 --- /dev/null +++ b/src/OpenDDD/Infrastructure/Events/InMemory/InMemorySubscription.cs @@ -0,0 +1,26 @@ +using OpenDDD.Infrastructure.Events.Base; + +namespace OpenDDD.Infrastructure.Events.InMemory +{ + public class InMemorySubscription : Subscription + { + public Func MessageHandler { get; } + + public InMemorySubscription(string topic, string consumerGroup, Func messageHandler) + : base(topic, consumerGroup, null) + { + if (string.IsNullOrWhiteSpace(topic)) + throw new ArgumentException("Topic cannot be null or empty.", nameof(topic)); + + if (string.IsNullOrWhiteSpace(consumerGroup)) + throw new ArgumentException("Consumer group cannot be null or empty.", nameof(consumerGroup)); + + MessageHandler = messageHandler; + } + + public override ValueTask DisposeAsync() + { + return ValueTask.CompletedTask; + } + } +} diff --git a/src/OpenDDD/Infrastructure/Events/Kafka/Factories/KafkaConsumer.cs b/src/OpenDDD/Infrastructure/Events/Kafka/Factories/KafkaConsumer.cs index 4da577a..3373570 100644 --- a/src/OpenDDD/Infrastructure/Events/Kafka/Factories/KafkaConsumer.cs +++ b/src/OpenDDD/Infrastructure/Events/Kafka/Factories/KafkaConsumer.cs @@ -3,7 +3,7 @@ namespace OpenDDD.Infrastructure.Events.Kafka.Factories { - public class KafkaConsumer : IDisposable + public class KafkaConsumer : IAsyncDisposable { private readonly IConsumer _consumer; private readonly ILogger _logger; @@ -90,9 +90,9 @@ public async Task StopProcessingAsync() _consumer.Dispose(); } - public void Dispose() + public ValueTask DisposeAsync() { - if (_disposed) return; + if (_disposed) return ValueTask.CompletedTask; _disposed = true; try @@ -106,6 +106,8 @@ public void Dispose() _consumer.Dispose(); _cts.Dispose(); + + return ValueTask.CompletedTask; } } } diff --git a/src/OpenDDD/Infrastructure/Events/Kafka/KafkaMessagingProvider.cs b/src/OpenDDD/Infrastructure/Events/Kafka/KafkaMessagingProvider.cs index aae533d..cf5a3b0 100644 --- a/src/OpenDDD/Infrastructure/Events/Kafka/KafkaMessagingProvider.cs +++ b/src/OpenDDD/Infrastructure/Events/Kafka/KafkaMessagingProvider.cs @@ -1,5 +1,6 @@ using System.Collections.Concurrent; using Microsoft.Extensions.Logging; +using OpenDDD.Infrastructure.Events.Base; using OpenDDD.Infrastructure.Events.Kafka.Factories; using Confluent.Kafka; using Confluent.Kafka.Admin; @@ -8,36 +9,34 @@ namespace OpenDDD.Infrastructure.Events.Kafka { public class KafkaMessagingProvider : IMessagingProvider, IAsyncDisposable { - private readonly string _bootstrapServers; private readonly IProducer _producer; private readonly IAdminClient _adminClient; private readonly bool _autoCreateTopics; private readonly IKafkaConsumerFactory _consumerFactory; private readonly ILogger _logger; - private readonly ConcurrentDictionary> _consumers = new(); + private readonly ConcurrentDictionary _subscriptions = new(); private readonly CancellationTokenSource _cts = new(); private bool _disposed; public KafkaMessagingProvider( - string bootstrapServers, IAdminClient adminClient, IProducer producer, IKafkaConsumerFactory consumerFactory, bool autoCreateTopics, ILogger logger) { - if (string.IsNullOrWhiteSpace(bootstrapServers)) - throw new ArgumentNullException(nameof(bootstrapServers)); - - _bootstrapServers = bootstrapServers; _adminClient = adminClient ?? throw new ArgumentNullException(nameof(adminClient)); _producer = producer ?? throw new ArgumentNullException(nameof(producer)); _consumerFactory = consumerFactory ?? throw new ArgumentNullException(nameof(consumerFactory)); _autoCreateTopics = autoCreateTopics; _logger = logger ?? throw new ArgumentNullException(nameof(logger)); } - - private static string GetGroupKey(string topic, string consumerGroup) + + public async Task SubscribeAsync( + string topic, + string consumerGroup, + Func messageHandler, + CancellationToken cancellationToken = default) { if (string.IsNullOrWhiteSpace(topic)) throw new ArgumentException("Topic cannot be null or empty.", nameof(topic)); @@ -45,20 +44,9 @@ private static string GetGroupKey(string topic, string consumerGroup) if (string.IsNullOrWhiteSpace(consumerGroup)) throw new ArgumentException("Consumer group cannot be null or empty.", nameof(consumerGroup)); - return $"{topic}:{consumerGroup}"; - } - - public async Task SubscribeAsync( - string topic, - string consumerGroup, - Func messageHandler, - CancellationToken cancellationToken) - { if (messageHandler is null) throw new ArgumentNullException(nameof(messageHandler)); - var groupKey = GetGroupKey(topic, consumerGroup); - if (_autoCreateTopics) { await CreateTopicIfNotExistsAsync(topic, cancellationToken); @@ -73,39 +61,35 @@ public async Task SubscribeAsync( } } - var consumers = _consumers.GetOrAdd(groupKey, _ => new ConcurrentBag()); - - var newConsumer = _consumerFactory.Create(consumerGroup); - newConsumer.Subscribe(topic); - consumers.Add(newConsumer); + var consumer = _consumerFactory.Create(consumerGroup); + consumer.Subscribe(topic); + consumer.StartProcessing(messageHandler, _cts.Token); - _logger.LogDebug("Subscribed a new consumer to Kafka topic '{Topic}' with consumer group '{ConsumerGroup}'", topic, consumerGroup); + var subscription = new KafkaSubscription(topic, consumerGroup, consumer); + _subscriptions[subscription.Id] = subscription; - newConsumer.StartProcessing(messageHandler, _cts.Token); + _logger.LogDebug("Subscribed a new consumer to Kafka topic '{Topic}' with consumer group '{ConsumerGroup}', Subscription ID: {SubscriptionId}", topic, consumerGroup, subscription.Id); + return subscription; } - - public async Task UnsubscribeAsync(string topic, string consumerGroup, CancellationToken cancellationToken) + + public async Task UnsubscribeAsync(ISubscription subscription, CancellationToken cancellationToken = default) { - var groupKey = GetGroupKey(topic, consumerGroup); + if (subscription == null) + throw new ArgumentNullException(nameof(subscription)); - if (!_consumers.TryGetValue(groupKey, out var consumers) || !consumers.Any()) + if (!_subscriptions.TryRemove(subscription.Id, out var removedSubscription)) { - _logger.LogWarning("No active consumers found for consumer group '{ConsumerGroup}' on topic '{Topic}'.", consumerGroup, topic); + _logger.LogWarning("No active subscription found with ID {SubscriptionId}", subscription.Id); return; } - _logger.LogDebug("Stopping all consumers for topic '{Topic}' and consumer group '{ConsumerGroup}'...", topic, consumerGroup); - - foreach (var consumer in consumers) - { - await consumer.StopProcessingAsync(); - consumer.Dispose(); - } + _logger.LogDebug("Unsubscribing from Kafka topic '{Topic}' with consumer group '{ConsumerGroup}', Subscription ID: {SubscriptionId}", removedSubscription.Topic, removedSubscription.ConsumerGroup, removedSubscription.Id); - _consumers.TryRemove(groupKey, out _); + await removedSubscription.Consumer.StopProcessingAsync(); + await removedSubscription.DisposeAsync(); } - public async Task PublishAsync(string topic, string message, CancellationToken cancellationToken) + public async Task PublishAsync(string topic, string message, CancellationToken cancellationToken = default) { if (string.IsNullOrWhiteSpace(topic)) throw new ArgumentException("Topic cannot be null or empty.", nameof(topic)); @@ -121,7 +105,7 @@ public async Task PublishAsync(string topic, string message, CancellationToken c await _producer.ProduceAsync(topic, new Message { Value = message }, cancellationToken); _logger.LogDebug("Published message to Kafka topic '{Topic}'", topic); } - + private async Task CreateTopicIfNotExistsAsync(string topic, CancellationToken cancellationToken) { try @@ -165,7 +149,7 @@ await _adminClient.CreateTopicsAsync(new[] throw new InvalidOperationException($"Failed to create topic '{topic}'", ex); } } - + public async ValueTask DisposeAsync() { if (_disposed) return; @@ -175,23 +159,15 @@ public async ValueTask DisposeAsync() _cts.Cancel(); - var tasks = _consumers.Values - .SelectMany(consumers => consumers) - .Select(c => c.StopProcessingAsync()) - .ToList(); - - await Task.WhenAll(tasks); - - foreach (var consumerList in _consumers.Values) + var disposeTasks = _subscriptions.Values.Select(async sub => { - foreach (var consumer in consumerList) - { - consumer.Dispose(); - } - } + await sub.Consumer.StopProcessingAsync(); + await sub.DisposeAsync(); + }).ToList(); - _consumers.Clear(); + await Task.WhenAll(disposeTasks); + _subscriptions.Clear(); _producer.Dispose(); _adminClient.Dispose(); } diff --git a/src/OpenDDD/Infrastructure/Events/Kafka/KafkaSubscription.cs b/src/OpenDDD/Infrastructure/Events/Kafka/KafkaSubscription.cs new file mode 100644 index 0000000..59577d4 --- /dev/null +++ b/src/OpenDDD/Infrastructure/Events/Kafka/KafkaSubscription.cs @@ -0,0 +1,16 @@ +using OpenDDD.Infrastructure.Events.Base; +using OpenDDD.Infrastructure.Events.Kafka.Factories; + +namespace OpenDDD.Infrastructure.Events.Kafka +{ + public class KafkaSubscription : Subscription + { + public KafkaSubscription(string topic, string consumerGroup, KafkaConsumer consumer) + : base(topic, consumerGroup, consumer) { } + + public override async ValueTask DisposeAsync() + { + await Consumer.DisposeAsync(); + } + } +} diff --git a/src/OpenDDD/Infrastructure/Events/RabbitMq/RabbitMqCustomAsyncConsumer.cs b/src/OpenDDD/Infrastructure/Events/RabbitMq/RabbitMqCustomAsyncConsumer.cs index 13e33d4..db347bc 100644 --- a/src/OpenDDD/Infrastructure/Events/RabbitMq/RabbitMqCustomAsyncConsumer.cs +++ b/src/OpenDDD/Infrastructure/Events/RabbitMq/RabbitMqCustomAsyncConsumer.cs @@ -5,10 +5,12 @@ namespace OpenDDD.Infrastructure.Events.RabbitMq { - public class RabbitMqCustomAsyncConsumer : IAsyncBasicConsumer + public class RabbitMqCustomAsyncConsumer : IAsyncBasicConsumer, IAsyncDisposable { private readonly Func _messageHandler; private readonly ILogger _logger; + private readonly IChannel _channel; + private string? _consumerTag; private bool _disposed; public RabbitMqCustomAsyncConsumer( @@ -16,12 +18,26 @@ public RabbitMqCustomAsyncConsumer( Func messageHandler, ILogger logger) { - Channel = channel ?? throw new ArgumentNullException(nameof(channel)); + _channel = channel ?? throw new ArgumentNullException(nameof(channel)); _messageHandler = messageHandler ?? throw new ArgumentNullException(nameof(messageHandler)); _logger = logger ?? throw new ArgumentNullException(nameof(logger)); } - public IChannel? Channel { get; } + public IChannel Channel => _channel; + + public async Task StartConsumingAsync(string queueName, CancellationToken cancellationToken) + { + _consumerTag = await _channel.BasicConsumeAsync(queueName, autoAck: false, this, cancellationToken); + _logger.LogInformation("Started consuming messages from queue '{QueueName}' with consumer tag '{ConsumerTag}'", queueName, _consumerTag); + } + + public async Task StopConsumingAsync(CancellationToken cancellationToken) + { + if (_consumerTag is null || _disposed) return; + + await _channel.BasicCancelAsync(_consumerTag, false, cancellationToken); + _logger.LogInformation("Stopped consuming messages for consumer tag '{ConsumerTag}'", _consumerTag); + } public async Task HandleBasicDeliverAsync( string consumerTag, @@ -71,5 +87,14 @@ public Task HandleChannelShutdownAsync(object channel, ShutdownEventArgs reason) _logger.LogWarning("Channel was shut down. Reason: {Reason}", reason.ReplyText); return Task.CompletedTask; } + + public async ValueTask DisposeAsync() + { + if (_disposed) + return; + + _disposed = true; + await StopConsumingAsync(CancellationToken.None); + } } } diff --git a/src/OpenDDD/Infrastructure/Events/RabbitMq/RabbitMqMessagingProvider.cs b/src/OpenDDD/Infrastructure/Events/RabbitMq/RabbitMqMessagingProvider.cs index 6cf9b4f..4af4006 100644 --- a/src/OpenDDD/Infrastructure/Events/RabbitMq/RabbitMqMessagingProvider.cs +++ b/src/OpenDDD/Infrastructure/Events/RabbitMq/RabbitMqMessagingProvider.cs @@ -1,6 +1,7 @@ using System.Collections.Concurrent; using System.Text; using Microsoft.Extensions.Logging; +using OpenDDD.Infrastructure.Events.Base; using OpenDDD.Infrastructure.Events.RabbitMq.Factories; using RabbitMQ.Client; @@ -13,8 +14,7 @@ public class RabbitMqMessagingProvider : IMessagingProvider, IAsyncDisposable private readonly ILogger _logger; private IConnection? _connection; private IChannel? _channel; - private readonly ConcurrentBag _consumers = new(); - private readonly ConcurrentDictionary _consumerTags = new(); + private readonly ConcurrentDictionary _subscriptions = new(); public RabbitMqMessagingProvider( IConnectionFactory factory, @@ -26,7 +26,7 @@ public RabbitMqMessagingProvider( _logger = logger ?? throw new ArgumentNullException(nameof(logger)); } - public async Task SubscribeAsync(string topic, string consumerGroup, Func messageHandler, CancellationToken cancellationToken = default) + public async Task SubscribeAsync(string topic, string consumerGroup, Func messageHandler, CancellationToken cancellationToken = default) { if (string.IsNullOrWhiteSpace(topic)) throw new ArgumentException("Topic cannot be null or empty.", nameof(topic)); @@ -47,34 +47,32 @@ public async Task SubscribeAsync(string topic, string consumerGroup, Func + { + public RabbitMqSubscription(string topic, string consumerGroup, RabbitMqCustomAsyncConsumer consumer) + : base(topic, consumerGroup, consumer) { } + + public override async ValueTask DisposeAsync() + { + await Consumer.StopConsumingAsync(CancellationToken.None); + await Consumer.DisposeAsync(); + } + } +} From 7aed97351ce7dd309d93d7e4356ecf59e996807e Mon Sep 17 00:00:00 2001 From: David Runemalm Date: Fri, 7 Mar 2025 16:20:54 +0700 Subject: [PATCH 093/109] Increase delay to attempt to fix failing test in github workflow. --- .../Events/Kafka/KafkaMessagingProviderTests.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/OpenDDD.Tests/Integration/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs b/src/OpenDDD.Tests/Integration/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs index 4a43935..7919d72 100644 --- a/src/OpenDDD.Tests/Integration/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs +++ b/src/OpenDDD.Tests/Integration/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs @@ -170,12 +170,12 @@ public async Task AtLeastOnceGurantee_ShouldDeliverToLateSubscriber_WhenSubscrib await _messagingProvider.UnsubscribeAsync(firstSubscription, _cts.Token); - await Task.Delay(5000); + await Task.Delay(10000, _cts.Token); // Late subscriber await _messagingProvider.PublishAsync(topicName, messageToSend, _cts.Token); - await Task.Delay(5000); + await Task.Delay(10000, _cts.Token); await _messagingProvider.SubscribeAsync(topicName, consumerGroup, async (msg, token) => { From a05463163d713fc3a9d3c866300a202a3d18e6e2 Mon Sep 17 00:00:00 2001 From: David Runemalm Date: Fri, 7 Mar 2025 17:04:59 +0700 Subject: [PATCH 094/109] Rename rabbit consumer class. --- .../Events/RabbitMq/Factories/IRabbitMqConsumerFactory.cs | 2 +- .../Events/RabbitMq/Factories/RabbitMqConsumerFactory.cs | 4 ++-- .../{RabbitMqCustomAsyncConsumer.cs => RabbitMqConsumer.cs} | 4 ++-- .../Infrastructure/Events/RabbitMq/RabbitMqSubscription.cs | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) rename src/OpenDDD/Infrastructure/Events/RabbitMq/{RabbitMqCustomAsyncConsumer.cs => RabbitMqConsumer.cs} (96%) diff --git a/src/OpenDDD/Infrastructure/Events/RabbitMq/Factories/IRabbitMqConsumerFactory.cs b/src/OpenDDD/Infrastructure/Events/RabbitMq/Factories/IRabbitMqConsumerFactory.cs index 2cb754d..7241f30 100644 --- a/src/OpenDDD/Infrastructure/Events/RabbitMq/Factories/IRabbitMqConsumerFactory.cs +++ b/src/OpenDDD/Infrastructure/Events/RabbitMq/Factories/IRabbitMqConsumerFactory.cs @@ -4,6 +4,6 @@ namespace OpenDDD.Infrastructure.Events.RabbitMq.Factories { public interface IRabbitMqConsumerFactory { - RabbitMqCustomAsyncConsumer CreateConsumer(IChannel channel, Func messageHandler); + RabbitMqConsumer CreateConsumer(IChannel channel, Func messageHandler); } } diff --git a/src/OpenDDD/Infrastructure/Events/RabbitMq/Factories/RabbitMqConsumerFactory.cs b/src/OpenDDD/Infrastructure/Events/RabbitMq/Factories/RabbitMqConsumerFactory.cs index 92b9d2f..f92543a 100644 --- a/src/OpenDDD/Infrastructure/Events/RabbitMq/Factories/RabbitMqConsumerFactory.cs +++ b/src/OpenDDD/Infrastructure/Events/RabbitMq/Factories/RabbitMqConsumerFactory.cs @@ -12,12 +12,12 @@ public RabbitMqConsumerFactory(ILogger logger) _logger = logger ?? throw new ArgumentNullException(nameof(logger)); } - public RabbitMqCustomAsyncConsumer CreateConsumer(IChannel channel, Func messageHandler) + public RabbitMqConsumer CreateConsumer(IChannel channel, Func messageHandler) { if (channel == null) throw new ArgumentNullException(nameof(channel)); if (messageHandler == null) throw new ArgumentNullException(nameof(messageHandler)); - return new RabbitMqCustomAsyncConsumer(channel, messageHandler, _logger); + return new RabbitMqConsumer(channel, messageHandler, _logger); } } } diff --git a/src/OpenDDD/Infrastructure/Events/RabbitMq/RabbitMqCustomAsyncConsumer.cs b/src/OpenDDD/Infrastructure/Events/RabbitMq/RabbitMqConsumer.cs similarity index 96% rename from src/OpenDDD/Infrastructure/Events/RabbitMq/RabbitMqCustomAsyncConsumer.cs rename to src/OpenDDD/Infrastructure/Events/RabbitMq/RabbitMqConsumer.cs index db347bc..e5008d6 100644 --- a/src/OpenDDD/Infrastructure/Events/RabbitMq/RabbitMqCustomAsyncConsumer.cs +++ b/src/OpenDDD/Infrastructure/Events/RabbitMq/RabbitMqConsumer.cs @@ -5,7 +5,7 @@ namespace OpenDDD.Infrastructure.Events.RabbitMq { - public class RabbitMqCustomAsyncConsumer : IAsyncBasicConsumer, IAsyncDisposable + public class RabbitMqConsumer : IAsyncBasicConsumer, IAsyncDisposable { private readonly Func _messageHandler; private readonly ILogger _logger; @@ -13,7 +13,7 @@ public class RabbitMqCustomAsyncConsumer : IAsyncBasicConsumer, IAsyncDisposable private string? _consumerTag; private bool _disposed; - public RabbitMqCustomAsyncConsumer( + public RabbitMqConsumer( IChannel channel, Func messageHandler, ILogger logger) diff --git a/src/OpenDDD/Infrastructure/Events/RabbitMq/RabbitMqSubscription.cs b/src/OpenDDD/Infrastructure/Events/RabbitMq/RabbitMqSubscription.cs index 5c13666..5766a1b 100644 --- a/src/OpenDDD/Infrastructure/Events/RabbitMq/RabbitMqSubscription.cs +++ b/src/OpenDDD/Infrastructure/Events/RabbitMq/RabbitMqSubscription.cs @@ -2,9 +2,9 @@ namespace OpenDDD.Infrastructure.Events.RabbitMq { - public class RabbitMqSubscription : Subscription + public class RabbitMqSubscription : Subscription { - public RabbitMqSubscription(string topic, string consumerGroup, RabbitMqCustomAsyncConsumer consumer) + public RabbitMqSubscription(string topic, string consumerGroup, RabbitMqConsumer consumer) : base(topic, consumerGroup, consumer) { } public override async ValueTask DisposeAsync() From 479191f5433ff1c9df1a8a26d75763388d543c46 Mon Sep 17 00:00:00 2001 From: David Runemalm Date: Fri, 7 Mar 2025 17:49:45 +0700 Subject: [PATCH 095/109] Attempt to fix test in pipeline. --- .../Kafka/KafkaMessagingProviderTests.cs | 21 +++++++++++++------ .../Events/Kafka/Factories/KafkaConsumer.cs | 2 +- .../Events/Kafka/KafkaSubscription.cs | 6 ++++++ 3 files changed, 22 insertions(+), 7 deletions(-) diff --git a/src/OpenDDD.Tests/Integration/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs b/src/OpenDDD.Tests/Integration/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs index 7919d72..b8a71d3 100644 --- a/src/OpenDDD.Tests/Integration/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs +++ b/src/OpenDDD.Tests/Integration/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs @@ -19,7 +19,7 @@ public class KafkaMessagingProviderTests : IntegrationTests, IAsyncLifetime private readonly ILogger _logger; private readonly ILogger _consumerLogger; private readonly KafkaMessagingProvider _messagingProvider; - private readonly CancellationTokenSource _cts = new(TimeSpan.FromSeconds(60)); + private readonly CancellationTokenSource _cts = new(TimeSpan.FromSeconds(120)); public KafkaMessagingProviderTests(ITestOutputHelper testOutputHelper) : base(testOutputHelper, enableLogging: true) @@ -163,21 +163,26 @@ public async Task AtLeastOnceGurantee_ShouldDeliverToLateSubscriber_WhenSubscrib { firstSubscriberReceived.SetResult(true); }, _cts.Token); + await WaitForKafkaConsumerGroupStable(consumerGroup, _cts.Token); + await Task.Delay(10000, _cts.Token); + await _messagingProvider.PublishAsync(topicName, messageToSend, _cts.Token); - await firstSubscriberReceived.Task.WaitAsync(TimeSpan.FromSeconds(10)); + + await firstSubscriberReceived.Task.WaitAsync(TimeSpan.FromSeconds(30)); + + await Task.Delay(10000, _cts.Token); await _messagingProvider.UnsubscribeAsync(firstSubscription, _cts.Token); await Task.Delay(10000, _cts.Token); - - // Late subscriber - await _messagingProvider.PublishAsync(topicName, messageToSend, _cts.Token); + await _messagingProvider.PublishAsync(topicName, messageToSend, _cts.Token); await Task.Delay(10000, _cts.Token); - await _messagingProvider.SubscribeAsync(topicName, consumerGroup, async (msg, token) => + // Late subscriber + var lateSubscription = await _messagingProvider.SubscribeAsync(topicName, consumerGroup, async (msg, token) => { _receivedMessages.Add(msg); lateSubscriberReceived.TrySetResult(true); @@ -185,6 +190,10 @@ await _messagingProvider.SubscribeAsync(topicName, consumerGroup, async (msg, to await WaitForKafkaConsumerGroupStable(consumerGroup, _cts.Token); + await Task.Delay(10000, _cts.Token); + + (lateSubscription as KafkaSubscription).PrintDebugInfo(); + await lateSubscriberReceived.Task.WaitAsync(TimeSpan.FromSeconds(30)); // Assert diff --git a/src/OpenDDD/Infrastructure/Events/Kafka/Factories/KafkaConsumer.cs b/src/OpenDDD/Infrastructure/Events/Kafka/Factories/KafkaConsumer.cs index 3373570..a99b2cc 100644 --- a/src/OpenDDD/Infrastructure/Events/Kafka/Factories/KafkaConsumer.cs +++ b/src/OpenDDD/Infrastructure/Events/Kafka/Factories/KafkaConsumer.cs @@ -5,7 +5,7 @@ namespace OpenDDD.Infrastructure.Events.Kafka.Factories { public class KafkaConsumer : IAsyncDisposable { - private readonly IConsumer _consumer; + public readonly IConsumer _consumer; private readonly ILogger _logger; private readonly CancellationTokenSource _cts = new(); private Task? _consumerTask; diff --git a/src/OpenDDD/Infrastructure/Events/Kafka/KafkaSubscription.cs b/src/OpenDDD/Infrastructure/Events/Kafka/KafkaSubscription.cs index 59577d4..8f58cff 100644 --- a/src/OpenDDD/Infrastructure/Events/Kafka/KafkaSubscription.cs +++ b/src/OpenDDD/Infrastructure/Events/Kafka/KafkaSubscription.cs @@ -12,5 +12,11 @@ public override async ValueTask DisposeAsync() { await Consumer.DisposeAsync(); } + + public void PrintDebugInfo() + { + var assignments = Consumer._consumer.Assignment; + Console.WriteLine($"Consumer assigned partitions: {string.Join(", ", assignments.Select(a => a.Partition.Value))}"); + } } } From 8e6b69fdca7106fcb5a3a8380a1cc7e80b8ec110 Mon Sep 17 00:00:00 2001 From: David Runemalm Date: Fri, 7 Mar 2025 18:47:01 +0700 Subject: [PATCH 096/109] Auto-commit initial offset on partitions with no messages on unsubscribe in kafka provider. --- .../Kafka/KafkaMessagingProviderTests.cs | 14 ++-- .../Events/Kafka/Factories/KafkaConsumer.cs | 65 ++++++++++++++++++- .../Events/Kafka/KafkaSubscription.cs | 6 -- 3 files changed, 70 insertions(+), 15 deletions(-) diff --git a/src/OpenDDD.Tests/Integration/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs b/src/OpenDDD.Tests/Integration/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs index b8a71d3..78624ae 100644 --- a/src/OpenDDD.Tests/Integration/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs +++ b/src/OpenDDD.Tests/Integration/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs @@ -166,23 +166,23 @@ public async Task AtLeastOnceGurantee_ShouldDeliverToLateSubscriber_WhenSubscrib await WaitForKafkaConsumerGroupStable(consumerGroup, _cts.Token); - await Task.Delay(10000, _cts.Token); + await Task.Delay(500, _cts.Token); await _messagingProvider.PublishAsync(topicName, messageToSend, _cts.Token); await firstSubscriberReceived.Task.WaitAsync(TimeSpan.FromSeconds(30)); - await Task.Delay(10000, _cts.Token); + await Task.Delay(500, _cts.Token); await _messagingProvider.UnsubscribeAsync(firstSubscription, _cts.Token); - await Task.Delay(10000, _cts.Token); + await Task.Delay(500, _cts.Token); await _messagingProvider.PublishAsync(topicName, messageToSend, _cts.Token); - await Task.Delay(10000, _cts.Token); + await Task.Delay(5000, _cts.Token); // Late subscriber - var lateSubscription = await _messagingProvider.SubscribeAsync(topicName, consumerGroup, async (msg, token) => + await _messagingProvider.SubscribeAsync(topicName, consumerGroup, async (msg, token) => { _receivedMessages.Add(msg); lateSubscriberReceived.TrySetResult(true); @@ -190,9 +190,7 @@ public async Task AtLeastOnceGurantee_ShouldDeliverToLateSubscriber_WhenSubscrib await WaitForKafkaConsumerGroupStable(consumerGroup, _cts.Token); - await Task.Delay(10000, _cts.Token); - - (lateSubscription as KafkaSubscription).PrintDebugInfo(); + await Task.Delay(500, _cts.Token); await lateSubscriberReceived.Task.WaitAsync(TimeSpan.FromSeconds(30)); diff --git a/src/OpenDDD/Infrastructure/Events/Kafka/Factories/KafkaConsumer.cs b/src/OpenDDD/Infrastructure/Events/Kafka/Factories/KafkaConsumer.cs index a99b2cc..5c38814 100644 --- a/src/OpenDDD/Infrastructure/Events/Kafka/Factories/KafkaConsumer.cs +++ b/src/OpenDDD/Infrastructure/Events/Kafka/Factories/KafkaConsumer.cs @@ -5,7 +5,7 @@ namespace OpenDDD.Infrastructure.Events.Kafka.Factories { public class KafkaConsumer : IAsyncDisposable { - public readonly IConsumer _consumer; + private readonly IConsumer _consumer; private readonly ILogger _logger; private readonly CancellationTokenSource _cts = new(); private Task? _consumerTask; @@ -86,10 +86,73 @@ public async Task StopProcessingAsync() _consumerTask = null; } + // Make sure all partitions have initial commit offset + await CommitInitialOffsetsAsync(); + _consumer.Close(); _consumer.Dispose(); } + private async Task CommitInitialOffsetsAsync() + { + foreach (var partition in _consumer.Assignment) + { + // Check if an offset is already committed + var committedOffsets = _consumer.Committed(new[] { partition }, TimeSpan.FromSeconds(5)); + var currentOffset = committedOffsets?.FirstOrDefault()?.Offset; + + if (currentOffset == null || currentOffset == Offset.Unset) + { + // Query watermark offsets (LOW = earliest, HIGH = next available offset) + var watermarkOffsets = _consumer.QueryWatermarkOffsets(partition, TimeSpan.FromSeconds(5)); + + if (watermarkOffsets.High == 0) // No messages ever written + { + _logger.LogDebug("Partition {Partition} has no messages. Sending placeholder message.", partition); + + // Publish a placeholder message to the **specific partition** + using var producer = new ProducerBuilder(new ProducerConfig { BootstrapServers = "localhost:9092" }).Build(); + var deliveryResult = await producer.ProduceAsync( + new TopicPartition(partition.Topic, partition.Partition), // Ensure correct partition + new Message { Value = "__init__" }); + + producer.Flush(TimeSpan.FromSeconds(5)); + + _logger.LogDebug("Sent __init__ message to partition {Partition}. Offset: {Offset}", partition, deliveryResult.Offset); + + // Poll Kafka until watermark updates or timeout occurs + var timeout = TimeSpan.FromSeconds(5); + var startTime = DateTime.UtcNow; + + while (DateTime.UtcNow - startTime < timeout) + { + await Task.Delay(100); // Small delay to avoid excessive polling + watermarkOffsets = _consumer.QueryWatermarkOffsets(partition, TimeSpan.FromSeconds(5)); + + if (watermarkOffsets.High > 0) // Message registered + break; + } + + if (watermarkOffsets.High == 0) + { + throw new TimeoutException($"Kafka did not register the __init__ message for partition {partition} within 5 seconds."); + } + + // Ensure the new high watermark is exactly 1 + var initialOffset = watermarkOffsets.High; + if (initialOffset.Value != 1) + { + throw new Exception($"Expected initial offset to be 1, but got {initialOffset.Value}"); + } + + _consumer.Commit(new[] { new TopicPartitionOffset(partition, initialOffset) }); + + _logger.LogDebug("Committed initial offset {Offset} for partition {Partition}", initialOffset, partition); + } + } + } + } + public ValueTask DisposeAsync() { if (_disposed) return ValueTask.CompletedTask; diff --git a/src/OpenDDD/Infrastructure/Events/Kafka/KafkaSubscription.cs b/src/OpenDDD/Infrastructure/Events/Kafka/KafkaSubscription.cs index 8f58cff..59577d4 100644 --- a/src/OpenDDD/Infrastructure/Events/Kafka/KafkaSubscription.cs +++ b/src/OpenDDD/Infrastructure/Events/Kafka/KafkaSubscription.cs @@ -12,11 +12,5 @@ public override async ValueTask DisposeAsync() { await Consumer.DisposeAsync(); } - - public void PrintDebugInfo() - { - var assignments = Consumer._consumer.Assignment; - Console.WriteLine($"Consumer assigned partitions: {string.Join(", ", assignments.Select(a => a.Partition.Value))}"); - } } } From cb26cd23d759cf87ccc5cb793edeedfecea8668e Mon Sep 17 00:00:00 2001 From: David Runemalm Date: Sat, 8 Mar 2025 08:51:33 +0700 Subject: [PATCH 097/109] Add auto-create-topics support to RabbitMQ provider as well. --- docs/configuration.rst | 11 ++- docs/userguide.rst | 3 +- .../Bookstore/src/Bookstore/appsettings.json | 3 +- .../RabbitMqMessagingProviderTests.cs | 97 +++++++++++++++---- .../RabbitMqMessagingProviderTests.cs | 3 +- .../OpenDddServiceCollectionExtensions.cs | 6 +- src/OpenDDD/API/Options/OpenDddOptions.cs | 13 ++- .../Options/OpenDddRabbitMqOptions.cs | 1 + .../RabbitMq/RabbitMqMessagingProvider.cs | 59 ++++++++++- 9 files changed, 168 insertions(+), 28 deletions(-) diff --git a/docs/configuration.rst b/docs/configuration.rst index dd4072b..2a54e7a 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -49,7 +49,8 @@ An example configuration in `appsettings.json`: "Port": 5672, "Username": "guest", "Password": "guest", - "VirtualHost": "/" + "VirtualHost": "/", + "AutoCreateTopics": true }, "Kafka": { "BootstrapServers": "localhost:9092", @@ -141,14 +142,18 @@ OpenDDD.NET supports multiple messaging providers: port: 5672, username: "guest", password: "guest", - virtualHost: "/" + virtualHost: "/", + autoCreateTopics: true ); **Kafka**: .. code-block:: csharp - options.UseKafka("localhost:9092"); + options.UseKafka( + "localhost:9092", + autoCreateTopics: true + ); **Azure Service Bus**: diff --git a/docs/userguide.rst b/docs/userguide.rst index 9c23a1d..99c9979 100644 --- a/docs/userguide.rst +++ b/docs/userguide.rst @@ -599,7 +599,8 @@ Add the following configuration to your `appsettings.json` file to customize Ope "Port": 5672, "Username": "guest", "Password": "guest", - "VirtualHost": "/" + "VirtualHost": "/", + "AutoCreateTopics": true }, "Kafka": { "BootstrapServers": "localhost:9092", diff --git a/samples/Bookstore/src/Bookstore/appsettings.json b/samples/Bookstore/src/Bookstore/appsettings.json index 4a43318..14b80e6 100644 --- a/samples/Bookstore/src/Bookstore/appsettings.json +++ b/samples/Bookstore/src/Bookstore/appsettings.json @@ -32,7 +32,8 @@ "Port": 5672, "Username": "guest", "Password": "guest", - "VirtualHost": "/" + "VirtualHost": "/", + "AutoCreateTopics": true }, "Kafka": { "BootstrapServers": "localhost:9092", diff --git a/src/OpenDDD.Tests/Integration/Infrastructure/Events/RabbitMq/RabbitMqMessagingProviderTests.cs b/src/OpenDDD.Tests/Integration/Infrastructure/Events/RabbitMq/RabbitMqMessagingProviderTests.cs index 6302fb4..d86287d 100644 --- a/src/OpenDDD.Tests/Integration/Infrastructure/Events/RabbitMq/RabbitMqMessagingProviderTests.cs +++ b/src/OpenDDD.Tests/Integration/Infrastructure/Events/RabbitMq/RabbitMqMessagingProviderTests.cs @@ -1,4 +1,5 @@ using System.Collections.Concurrent; +using FluentAssertions; using Microsoft.Extensions.Logging; using Xunit.Abstractions; using OpenDDD.Infrastructure.Events.RabbitMq; @@ -38,7 +39,11 @@ public RabbitMqMessagingProviderTests(ITestOutputHelper testOutputHelper) }; _consumerFactory = new RabbitMqConsumerFactory(_logger); - _messagingProvider = new RabbitMqMessagingProvider(_connectionFactory, _consumerFactory, _logger); + _messagingProvider = new RabbitMqMessagingProvider( + _connectionFactory, + _consumerFactory, + autoCreateTopics: true, + _logger); } public async Task InitializeAsync() @@ -126,29 +131,65 @@ private async Task CleanupExchangesAndQueuesAsync() public async Task AutoCreateTopic_ShouldCreateTopicOnSubscribe_WhenSettingEnabled() { // Arrange - await VerifyExchangeAndQueueDoNotExist(); + var topicName = $"test-topic-{Guid.NewGuid()}"; + var consumerGroup = "test-consumer-group"; + + var messagingProvider = new RabbitMqMessagingProvider( + _connectionFactory, + _consumerFactory, + autoCreateTopics: true, // Auto-create enabled + _logger); + + var exchangeExistsBefore = await ExchangeExistsAsync(topicName, _cts.Token); + exchangeExistsBefore.Should().BeFalse("The exchange should not exist before subscribing."); // Act - await _messagingProvider.SubscribeAsync(_testTopic, _testConsumerGroup, (msg, token) => Task.CompletedTask); + await messagingProvider.SubscribeAsync(topicName, consumerGroup, async (msg, token) => + await Task.CompletedTask, _cts.Token); + + var timeout = TimeSpan.FromSeconds(30); + var pollingInterval = TimeSpan.FromMilliseconds(500); + var startTime = DateTime.UtcNow; - // Assert - try + bool exchangeExists = false; + while (DateTime.UtcNow - startTime < timeout) { - await _channel!.ExchangeDeclarePassiveAsync(_testTopic, CancellationToken.None); - } - catch (OperationInterruptedException) - { - Assert.Fail($"Exchange '{_testTopic}' does not exist."); + if (await ExchangeExistsAsync(topicName, _cts.Token)) + { + exchangeExists = true; + break; + } + await Task.Delay(pollingInterval, _cts.Token); } - try - { - await _channel!.QueueDeclarePassiveAsync($"{_testConsumerGroup}.{_testTopic}", CancellationToken.None); - } - catch (OperationInterruptedException) + // Assert + exchangeExists.Should().BeTrue("RabbitMQ should create the exchange automatically when subscribing."); + } + + [Fact] + public async Task AutoCreateTopic_ShouldNotCreateTopicOnSubscribe_WhenSettingDisabled() + { + // Arrange + var topicName = $"test-topic-{Guid.NewGuid()}"; + var consumerGroup = "test-consumer-group"; + + var messagingProvider = new RabbitMqMessagingProvider( + _connectionFactory, + _consumerFactory, + autoCreateTopics: false, // Auto-create disabled + _logger); + + var exchangeExistsBefore = await ExchangeExistsAsync(topicName, _cts.Token); + exchangeExistsBefore.Should().BeFalse("The exchange should not exist before subscribing."); + + // Act & Assert + var exception = await Assert.ThrowsAsync(async () => { - Assert.Fail($"Queue '{_testConsumerGroup}.{_testTopic}' does not exist."); - } + await messagingProvider.SubscribeAsync(topicName, consumerGroup, async (msg, token) => + await Task.CompletedTask, _cts.Token); + }); + + exception.Message.Should().Contain($"Topic '{topicName}' does not exist."); } [Fact] @@ -357,5 +398,27 @@ async Task MessageHandler(string msg, CancellationToken token) Assert.InRange(count, minAllowed, maxAllowed); } } + + private async Task ExchangeExistsAsync(string exchange, CancellationToken cancellationToken) + { + try + { + // Use a temporary channel to check exchange existence, since old one might have stale topic data + using var tempChannel = await _connection!.CreateChannelAsync(null, cancellationToken); + await tempChannel.ExchangeDeclarePassiveAsync(exchange, cancellationToken); + + return true; + } + catch (OperationInterruptedException ex) when (ex.ShutdownReason?.ReplyCode == 404) + { + _logger.LogDebug("Exchange '{Exchange}' does not exist yet.", exchange); + return false; + } + catch (Exception ex) + { + _logger.LogError(ex, "Unexpected error while checking if exchange '{Exchange}' exists."); + throw; + } + } } } diff --git a/src/OpenDDD.Tests/Unit/Infrastructure/Events/RabbitMq/RabbitMqMessagingProviderTests.cs b/src/OpenDDD.Tests/Unit/Infrastructure/Events/RabbitMq/RabbitMqMessagingProviderTests.cs index 3bfc2a2..3549224 100644 --- a/src/OpenDDD.Tests/Unit/Infrastructure/Events/RabbitMq/RabbitMqMessagingProviderTests.cs +++ b/src/OpenDDD.Tests/Unit/Infrastructure/Events/RabbitMq/RabbitMqMessagingProviderTests.cs @@ -26,6 +26,7 @@ public RabbitMqMessagingProviderTests() _provider = new RabbitMqMessagingProvider( _mockConnectionFactory.Object, _mockConsumerFactory.Object, + autoCreateTopics: true, _mockLogger.Object ); } @@ -42,7 +43,7 @@ public void Constructor_ShouldThrowException_WhenDependenciesAreNull( var mockLogger = logger is null ? null! : _mockLogger.Object; Assert.Throws(() => - new RabbitMqMessagingProvider(mockConnectionFactory, mockConsumerFactory, mockLogger)); + new RabbitMqMessagingProvider(mockConnectionFactory, mockConsumerFactory, autoCreateTopics: true, mockLogger)); } [Theory] diff --git a/src/OpenDDD/API/Extensions/OpenDddServiceCollectionExtensions.cs b/src/OpenDDD/API/Extensions/OpenDddServiceCollectionExtensions.cs index 6bc2856..eac6823 100644 --- a/src/OpenDDD/API/Extensions/OpenDddServiceCollectionExtensions.cs +++ b/src/OpenDDD/API/Extensions/OpenDddServiceCollectionExtensions.cs @@ -323,7 +323,11 @@ private static void AddRabbitMq(this IServiceCollection services) var consumerFactory = new RabbitMqConsumerFactory(logger); - return new RabbitMqMessagingProvider(connectionFactory, consumerFactory, logger); + return new RabbitMqMessagingProvider( + connectionFactory, + consumerFactory, + rabbitMqOptions.AutoCreateTopics, + logger); }); } diff --git a/src/OpenDDD/API/Options/OpenDddOptions.cs b/src/OpenDDD/API/Options/OpenDddOptions.cs index d7bf397..b9524ef 100644 --- a/src/OpenDDD/API/Options/OpenDddOptions.cs +++ b/src/OpenDDD/API/Options/OpenDddOptions.cs @@ -62,7 +62,13 @@ public OpenDddOptions UseInMemoryMessaging() return this; } - public OpenDddOptions UseRabbitMq(string hostName, int port, string username, string password, string virtualHost = "/") + public OpenDddOptions UseRabbitMq( + string hostName, + int port, + string username, + string password, + string virtualHost = "/", + bool autoCreateTopics = true) { MessagingProvider = "RabbitMq"; RabbitMq = new OpenDddRabbitMqOptions @@ -71,12 +77,13 @@ public OpenDddOptions UseRabbitMq(string hostName, int port, string username, st Port = port, Username = username, Password = password, - VirtualHost = virtualHost + VirtualHost = virtualHost, + AutoCreateTopics = autoCreateTopics }; return this; } - public OpenDddOptions UseKafka(string bootstrapServers) + public OpenDddOptions UseKafka(string bootstrapServers, bool autoCreateTopics = true) { MessagingProvider = "Kafka"; Kafka = new OpenDddKafkaOptions { BootstrapServers = bootstrapServers }; diff --git a/src/OpenDDD/Infrastructure/Events/RabbitMq/Options/OpenDddRabbitMqOptions.cs b/src/OpenDDD/Infrastructure/Events/RabbitMq/Options/OpenDddRabbitMqOptions.cs index 7944563..9d2ab6b 100644 --- a/src/OpenDDD/Infrastructure/Events/RabbitMq/Options/OpenDddRabbitMqOptions.cs +++ b/src/OpenDDD/Infrastructure/Events/RabbitMq/Options/OpenDddRabbitMqOptions.cs @@ -7,5 +7,6 @@ public class OpenDddRabbitMqOptions public string Username { get; set; } = "guest"; public string Password { get; set; } = "guest"; public string VirtualHost { get; set; } = "/"; + public bool AutoCreateTopics { get; set; } = true; } } diff --git a/src/OpenDDD/Infrastructure/Events/RabbitMq/RabbitMqMessagingProvider.cs b/src/OpenDDD/Infrastructure/Events/RabbitMq/RabbitMqMessagingProvider.cs index 4af4006..606c416 100644 --- a/src/OpenDDD/Infrastructure/Events/RabbitMq/RabbitMqMessagingProvider.cs +++ b/src/OpenDDD/Infrastructure/Events/RabbitMq/RabbitMqMessagingProvider.cs @@ -4,6 +4,7 @@ using OpenDDD.Infrastructure.Events.Base; using OpenDDD.Infrastructure.Events.RabbitMq.Factories; using RabbitMQ.Client; +using RabbitMQ.Client.Exceptions; namespace OpenDDD.Infrastructure.Events.RabbitMq { @@ -12,6 +13,7 @@ public class RabbitMqMessagingProvider : IMessagingProvider, IAsyncDisposable private readonly IConnectionFactory _connectionFactory; private readonly IRabbitMqConsumerFactory _consumerFactory; private readonly ILogger _logger; + private readonly bool _autoCreateTopics; private IConnection? _connection; private IChannel? _channel; private readonly ConcurrentDictionary _subscriptions = new(); @@ -19,10 +21,12 @@ public class RabbitMqMessagingProvider : IMessagingProvider, IAsyncDisposable public RabbitMqMessagingProvider( IConnectionFactory factory, IRabbitMqConsumerFactory consumerFactory, + bool autoCreateTopics, ILogger logger) { _connectionFactory = factory ?? throw new ArgumentNullException(nameof(factory)); _consumerFactory = consumerFactory ?? throw new ArgumentNullException(nameof(consumerFactory)); + _autoCreateTopics = autoCreateTopics; _logger = logger ?? throw new ArgumentNullException(nameof(logger)); } @@ -41,6 +45,22 @@ public async Task SubscribeAsync(string topic, string consumerGro if (_channel is null) throw new InvalidOperationException("RabbitMQ channel is not available."); + bool exchangeExists = await ExchangeExistsAsync(topic, cancellationToken); + + if (!exchangeExists) + { + if (_autoCreateTopics) + { + await _channel.ExchangeDeclareAsync(topic, ExchangeType.Fanout, durable: true, autoDelete: false, cancellationToken: cancellationToken); + _logger.LogInformation("Auto-created exchange (topic): {Topic}", topic); + } + else + { + _logger.LogError("Cannot subscribe to non-existent topic: {Topic}", topic); + throw new InvalidOperationException($"Topic '{topic}' does not exist."); + } + } + await _channel.ExchangeDeclareAsync(topic, ExchangeType.Fanout, durable: true, autoDelete: false, cancellationToken: cancellationToken); var queueName = $"{consumerGroup}.{topic}"; await _channel.QueueDeclareAsync(queueName, durable: true, exclusive: false, autoDelete: false, cancellationToken: cancellationToken); @@ -87,7 +107,21 @@ public async Task PublishAsync(string topic, string message, CancellationToken c if (_channel is null) throw new InvalidOperationException("RabbitMQ channel is not available."); - await _channel.ExchangeDeclareAsync(topic, ExchangeType.Fanout, durable: true, autoDelete: false, cancellationToken: cancellationToken); + bool exchangeExists = await ExchangeExistsAsync(topic, cancellationToken); + + if (!exchangeExists) + { + if (_autoCreateTopics) + { + await _channel.ExchangeDeclareAsync(topic, ExchangeType.Fanout, durable: true, autoDelete: false, cancellationToken: cancellationToken); + _logger.LogInformation("Auto-created exchange (topic): {Topic}", topic); + } + else + { + _logger.LogError("Cannot publish to non-existent topic: {Topic}", topic); + throw new InvalidOperationException($"Topic '{topic}' does not exist."); + } + } var body = Encoding.UTF8.GetBytes(message); await _channel.BasicPublishAsync(topic, "", body, cancellationToken: cancellationToken); @@ -102,6 +136,29 @@ private async Task EnsureConnectedAsync(CancellationToken cancellationToken) _connection = await _connectionFactory.CreateConnectionAsync(cancellationToken); _channel = await _connection.CreateChannelAsync(null, cancellationToken); } + + private async Task ExchangeExistsAsync(string exchange, CancellationToken cancellationToken) + { + try + { + await _channel!.ExchangeDeclarePassiveAsync(exchange, cancellationToken); + return true; + } + catch (OperationInterruptedException ex) when (ex.ShutdownReason?.ReplyCode == 404) + { + _logger.LogDebug("Exchange '{Exchange}' does not exist.", exchange); + + // Since the channel was closed, reopen it before returning + await EnsureConnectedAsync(cancellationToken); + + return false; + } + catch (Exception ex) + { + _logger.LogError(ex, "Unexpected error while checking if exchange '{Exchange}' exists.", exchange); + throw; + } + } public async ValueTask DisposeAsync() { From 213f25048285078eaf9366c027676436b2277c69 Mon Sep 17 00:00:00 2001 From: David Runemalm Date: Sat, 8 Mar 2025 08:54:27 +0700 Subject: [PATCH 098/109] Refactor rabbit provider by breaking out helper method for reuse. --- .../RabbitMq/RabbitMqMessagingProvider.cs | 53 +++++++++---------- 1 file changed, 25 insertions(+), 28 deletions(-) diff --git a/src/OpenDDD/Infrastructure/Events/RabbitMq/RabbitMqMessagingProvider.cs b/src/OpenDDD/Infrastructure/Events/RabbitMq/RabbitMqMessagingProvider.cs index 606c416..5198282 100644 --- a/src/OpenDDD/Infrastructure/Events/RabbitMq/RabbitMqMessagingProvider.cs +++ b/src/OpenDDD/Infrastructure/Events/RabbitMq/RabbitMqMessagingProvider.cs @@ -45,22 +45,11 @@ public async Task SubscribeAsync(string topic, string consumerGro if (_channel is null) throw new InvalidOperationException("RabbitMQ channel is not available."); - bool exchangeExists = await ExchangeExistsAsync(topic, cancellationToken); - - if (!exchangeExists) + if (_autoCreateTopics) { - if (_autoCreateTopics) - { - await _channel.ExchangeDeclareAsync(topic, ExchangeType.Fanout, durable: true, autoDelete: false, cancellationToken: cancellationToken); - _logger.LogInformation("Auto-created exchange (topic): {Topic}", topic); - } - else - { - _logger.LogError("Cannot subscribe to non-existent topic: {Topic}", topic); - throw new InvalidOperationException($"Topic '{topic}' does not exist."); - } + await CreateTopicIfNotExistsAsync(topic, cancellationToken); } - + await _channel.ExchangeDeclareAsync(topic, ExchangeType.Fanout, durable: true, autoDelete: false, cancellationToken: cancellationToken); var queueName = $"{consumerGroup}.{topic}"; await _channel.QueueDeclareAsync(queueName, durable: true, exclusive: false, autoDelete: false, cancellationToken: cancellationToken); @@ -76,7 +65,7 @@ public async Task SubscribeAsync(string topic, string consumerGro return subscription; } - + public async Task UnsubscribeAsync(ISubscription subscription, CancellationToken cancellationToken = default) { if (subscription == null) @@ -107,20 +96,9 @@ public async Task PublishAsync(string topic, string message, CancellationToken c if (_channel is null) throw new InvalidOperationException("RabbitMQ channel is not available."); - bool exchangeExists = await ExchangeExistsAsync(topic, cancellationToken); - - if (!exchangeExists) + if (_autoCreateTopics) { - if (_autoCreateTopics) - { - await _channel.ExchangeDeclareAsync(topic, ExchangeType.Fanout, durable: true, autoDelete: false, cancellationToken: cancellationToken); - _logger.LogInformation("Auto-created exchange (topic): {Topic}", topic); - } - else - { - _logger.LogError("Cannot publish to non-existent topic: {Topic}", topic); - throw new InvalidOperationException($"Topic '{topic}' does not exist."); - } + await CreateTopicIfNotExistsAsync(topic, cancellationToken); } var body = Encoding.UTF8.GetBytes(message); @@ -137,6 +115,25 @@ private async Task EnsureConnectedAsync(CancellationToken cancellationToken) _channel = await _connection.CreateChannelAsync(null, cancellationToken); } + private async Task CreateTopicIfNotExistsAsync(string topic, CancellationToken cancellationToken) + { + bool exchangeExists = await ExchangeExistsAsync(topic, cancellationToken); + + if (!exchangeExists) + { + if (_autoCreateTopics) + { + await _channel.ExchangeDeclareAsync(topic, ExchangeType.Fanout, durable: true, autoDelete: false, cancellationToken: cancellationToken); + _logger.LogInformation("Auto-created exchange (topic): {Topic}", topic); + } + else + { + _logger.LogError("Cannot subscribe to non-existent topic: {Topic}", topic); + throw new InvalidOperationException($"Topic '{topic}' does not exist."); + } + } + } + private async Task ExchangeExistsAsync(string exchange, CancellationToken cancellationToken) { try From bfb96ef79c9f2946a28193f2fa1a17f8fd19c521 Mon Sep 17 00:00:00 2001 From: David Runemalm Date: Sat, 8 Mar 2025 09:26:10 +0700 Subject: [PATCH 099/109] Refactor code and tests for messaging providers. --- .../AzureServiceBusMessagingProviderTests.cs | 3 +- .../Kafka/KafkaMessagingProviderTests.cs | 4 +- .../RabbitMqMessagingProviderTests.cs | 13 ++++- .../Azure/AzureServiceBusMessagingProvider.cs | 44 ++++++---------- .../Events/Kafka/KafkaMessagingProvider.cs | 52 +++++-------------- .../RabbitMq/RabbitMqMessagingProvider.cs | 44 ++++++---------- 6 files changed, 62 insertions(+), 98 deletions(-) diff --git a/src/OpenDDD.Tests/Integration/Infrastructure/Events/Azure/AzureServiceBusMessagingProviderTests.cs b/src/OpenDDD.Tests/Integration/Infrastructure/Events/Azure/AzureServiceBusMessagingProviderTests.cs index 01e6ace..bee513d 100644 --- a/src/OpenDDD.Tests/Integration/Infrastructure/Events/Azure/AzureServiceBusMessagingProviderTests.cs +++ b/src/OpenDDD.Tests/Integration/Infrastructure/Events/Azure/AzureServiceBusMessagingProviderTests.cs @@ -5,6 +5,7 @@ using OpenDDD.Tests.Base; using Azure.Messaging.ServiceBus; using Azure.Messaging.ServiceBus.Administration; +using FluentAssertions; namespace OpenDDD.Tests.Integration.Infrastructure.Events.Azure { @@ -108,7 +109,7 @@ public async Task AutoCreateTopic_ShouldNotCreateTopicOnSubscribe_WhenSettingDis Assert.False(await _adminClient.TopicExistsAsync(topicName), "Topic should not have been created."); - Assert.Equal($"Cannot subscribe to topic '{topicName}' because it does not exist and auto-creation is disabled.", exception.Message); + exception.Message.Should().Be($"Topic '{topicName}' does not exist. Enable 'autoCreateTopics' to create topics automatically."); } [Fact] diff --git a/src/OpenDDD.Tests/Integration/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs b/src/OpenDDD.Tests/Integration/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs index 78624ae..612246d 100644 --- a/src/OpenDDD.Tests/Integration/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs +++ b/src/OpenDDD.Tests/Integration/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs @@ -139,13 +139,13 @@ public async Task AutoCreateTopic_ShouldNotCreateTopicOnSubscribe_WhenSettingDis _logger); // Act & Assert - var exception = await Assert.ThrowsAsync(async () => + var exception = await Assert.ThrowsAsync(async () => { await messagingProvider.SubscribeAsync(topicName, consumerGroup, async (msg, token) => await Task.CompletedTask, _cts.Token); }); - exception.Message.Should().Contain($"Topic '{topicName}' does not exist."); + exception.Message.Should().Be($"Topic '{topicName}' does not exist. Enable 'autoCreateTopics' to create topics automatically."); } [Fact] diff --git a/src/OpenDDD.Tests/Integration/Infrastructure/Events/RabbitMq/RabbitMqMessagingProviderTests.cs b/src/OpenDDD.Tests/Integration/Infrastructure/Events/RabbitMq/RabbitMqMessagingProviderTests.cs index d86287d..7be87c6 100644 --- a/src/OpenDDD.Tests/Integration/Infrastructure/Events/RabbitMq/RabbitMqMessagingProviderTests.cs +++ b/src/OpenDDD.Tests/Integration/Infrastructure/Events/RabbitMq/RabbitMqMessagingProviderTests.cs @@ -189,7 +189,7 @@ await messagingProvider.SubscribeAsync(topicName, consumerGroup, async (msg, tok await Task.CompletedTask, _cts.Token); }); - exception.Message.Should().Contain($"Topic '{topicName}' does not exist."); + exception.Message.Should().Be($"Topic '{topicName}' does not exist. Enable 'autoCreateTopics' to create topics automatically."); } [Fact] @@ -399,6 +399,14 @@ async Task MessageHandler(string msg, CancellationToken token) } } + private async Task EnsureConnectedAsync(CancellationToken cancellationToken) + { + if (_connection is { IsOpen: true } && _channel is { IsOpen: true }) return; + + _connection = await _connectionFactory.CreateConnectionAsync(cancellationToken); + _channel = await _connection.CreateChannelAsync(null, cancellationToken); + } + private async Task ExchangeExistsAsync(string exchange, CancellationToken cancellationToken) { try @@ -412,6 +420,9 @@ private async Task ExchangeExistsAsync(string exchange, CancellationToken catch (OperationInterruptedException ex) when (ex.ShutdownReason?.ReplyCode == 404) { _logger.LogDebug("Exchange '{Exchange}' does not exist yet.", exchange); + + await EnsureConnectedAsync(cancellationToken); + return false; } catch (Exception ex) diff --git a/src/OpenDDD/Infrastructure/Events/Azure/AzureServiceBusMessagingProvider.cs b/src/OpenDDD/Infrastructure/Events/Azure/AzureServiceBusMessagingProvider.cs index aeb0515..ef755ab 100644 --- a/src/OpenDDD/Infrastructure/Events/Azure/AzureServiceBusMessagingProvider.cs +++ b/src/OpenDDD/Infrastructure/Events/Azure/AzureServiceBusMessagingProvider.cs @@ -44,23 +44,11 @@ public async Task SubscribeAsync(string topic, string consumerGro throw new ArgumentNullException(nameof(messageHandler)); } - var subscriptionName = consumerGroup; + await EnsureTopicExistsAsync(topic, cancellationToken); - if (_autoCreateTopics) - { - await CreateTopicIfNotExistsAsync(topic, cancellationToken); - } - - if (!await _adminClient.TopicExistsAsync(topic, cancellationToken)) - { - var errorMessage = $"Cannot subscribe to topic '{topic}' because it does not exist and auto-creation is disabled."; - _logger.LogError(errorMessage); - throw new InvalidOperationException(errorMessage); - } - - await CreateSubscriptionIfNotExistsAsync(topic, subscriptionName, cancellationToken); + await CreateSubscriptionIfNotExistsAsync(topic, consumerGroup, cancellationToken); - var processor = _client.CreateProcessor(topic, subscriptionName); + var processor = _client.CreateProcessor(topic, consumerGroup); processor.ProcessMessageAsync += async args => { @@ -70,14 +58,14 @@ public async Task SubscribeAsync(string topic, string consumerGro processor.ProcessErrorAsync += args => { - _logger.LogError(args.Exception, "Error processing message in subscription {SubscriptionName}", subscriptionName); + _logger.LogError(args.Exception, "Error processing message in subscription {SubscriptionName}", consumerGroup); return Task.CompletedTask; }; var subscription = new AzureServiceBusSubscription(topic, consumerGroup, processor); _subscriptions[subscription.Id] = subscription; - _logger.LogInformation("Starting message processor for topic '{Topic}' and subscription '{Subscription}', Subscription ID: {SubscriptionId}", topic, subscriptionName, subscription.Id); + _logger.LogInformation("Starting message processor for topic '{Topic}' and subscription '{Subscription}', Subscription ID: {SubscriptionId}", topic, consumerGroup, subscription.Id); await processor.StartProcessingAsync(cancellationToken); return subscription; @@ -103,31 +91,31 @@ public async Task UnsubscribeAsync(ISubscription subscription, CancellationToken public async Task PublishAsync(string topic, string message, CancellationToken cancellationToken = default) { if (string.IsNullOrWhiteSpace(topic)) - { throw new ArgumentException("Topic cannot be null or empty.", nameof(topic)); - } if (string.IsNullOrWhiteSpace(message)) - { throw new ArgumentException("Message cannot be null or empty.", nameof(message)); - } - if (_autoCreateTopics) - { - await CreateTopicIfNotExistsAsync(topic, cancellationToken); - } + await EnsureTopicExistsAsync(topic, cancellationToken); var sender = _client.CreateSender(topic); await sender.SendMessageAsync(new ServiceBusMessage(message), cancellationToken); _logger.LogInformation("Published message to topic '{Topic}'", topic); } - private async Task CreateTopicIfNotExistsAsync(string topic, CancellationToken cancellationToken) + private async Task EnsureTopicExistsAsync(string topic, CancellationToken cancellationToken) { if (!await _adminClient.TopicExistsAsync(topic, cancellationToken)) { - await _adminClient.CreateTopicAsync(topic, cancellationToken); - _logger.LogInformation("Created topic: {Topic}", topic); + if (_autoCreateTopics) + { + await _adminClient.CreateTopicAsync(topic, cancellationToken); + _logger.LogInformation("Created topic: {Topic}", topic); + } + else + { + throw new InvalidOperationException($"Topic '{topic}' does not exist. Enable 'autoCreateTopics' to create topics automatically."); + } } } diff --git a/src/OpenDDD/Infrastructure/Events/Kafka/KafkaMessagingProvider.cs b/src/OpenDDD/Infrastructure/Events/Kafka/KafkaMessagingProvider.cs index cf5a3b0..6297c0e 100644 --- a/src/OpenDDD/Infrastructure/Events/Kafka/KafkaMessagingProvider.cs +++ b/src/OpenDDD/Infrastructure/Events/Kafka/KafkaMessagingProvider.cs @@ -47,19 +47,7 @@ public async Task SubscribeAsync( if (messageHandler is null) throw new ArgumentNullException(nameof(messageHandler)); - if (_autoCreateTopics) - { - await CreateTopicIfNotExistsAsync(topic, cancellationToken); - } - else - { - var metadata = _adminClient.GetMetadata(TimeSpan.FromSeconds(5)); - if (!metadata.Topics.Any(t => t.Topic == topic)) - { - _logger.LogError("Cannot subscribe to non-existent topic: {Topic}", topic); - throw new KafkaException(new Error(ErrorCode.UnknownTopicOrPart, $"Topic '{topic}' does not exist.")); - } - } + await EnsureTopicExistsAsync(topic, cancellationToken); var consumer = _consumerFactory.Create(consumerGroup); consumer.Subscribe(topic); @@ -97,57 +85,45 @@ public async Task PublishAsync(string topic, string message, CancellationToken c if (string.IsNullOrWhiteSpace(message)) throw new ArgumentException("Message cannot be null or empty.", nameof(message)); - if (_autoCreateTopics) - { - await CreateTopicIfNotExistsAsync(topic, cancellationToken); - } + await EnsureTopicExistsAsync(topic, cancellationToken); await _producer.ProduceAsync(topic, new Message { Value = message }, cancellationToken); _logger.LogDebug("Published message to Kafka topic '{Topic}'", topic); } - private async Task CreateTopicIfNotExistsAsync(string topic, CancellationToken cancellationToken) + private async Task EnsureTopicExistsAsync(string topic, CancellationToken cancellationToken) { - try - { - var metadata = _adminClient.GetMetadata(TimeSpan.FromSeconds(5)); + var metadata = _adminClient.GetMetadata(TimeSpan.FromSeconds(5)); - if (metadata.Topics.Any(t => t.Topic == topic)) - { - _logger.LogDebug("Topic '{Topic}' already exists. Skipping creation.", topic); - return; - } + if (metadata.Topics.Any(t => t.Topic == topic)) + { + _logger.LogDebug("Topic '{Topic}' already exists.", topic); + return; + } + if (_autoCreateTopics) + { _logger.LogDebug("Creating Kafka topic: {Topic}", topic); await _adminClient.CreateTopicsAsync(new[] { new TopicSpecification { Name = topic, NumPartitions = 2, ReplicationFactor = 1 } }, null); + // Wait for the topic to be available for (int i = 0; i < 30; i++) { await Task.Delay(500, cancellationToken); metadata = _adminClient.GetMetadata(TimeSpan.FromSeconds(1)); - if (metadata.Topics.Any(t => t.Topic == topic)) { _logger.LogDebug("Kafka topic '{Topic}' is now available.", topic); return; } } - throw new KafkaException(new Error(ErrorCode.UnknownTopicOrPart, $"Failed to create topic '{topic}' within timeout.")); } - catch (KafkaException ex) - { - _logger.LogError(ex, "Kafka error while creating topic {Topic}: {Message}", topic, ex.Message); - throw; - } - catch (Exception ex) - { - _logger.LogError(ex, "Unexpected error while creating Kafka topic {Topic}", topic); - throw new InvalidOperationException($"Failed to create topic '{topic}'", ex); - } + + throw new InvalidOperationException($"Topic '{topic}' does not exist. Enable 'autoCreateTopics' to create topics automatically."); } public async ValueTask DisposeAsync() diff --git a/src/OpenDDD/Infrastructure/Events/RabbitMq/RabbitMqMessagingProvider.cs b/src/OpenDDD/Infrastructure/Events/RabbitMq/RabbitMqMessagingProvider.cs index 5198282..410b54d 100644 --- a/src/OpenDDD/Infrastructure/Events/RabbitMq/RabbitMqMessagingProvider.cs +++ b/src/OpenDDD/Infrastructure/Events/RabbitMq/RabbitMqMessagingProvider.cs @@ -42,15 +42,8 @@ public async Task SubscribeAsync(string topic, string consumerGro throw new ArgumentNullException(nameof(messageHandler), "Message handler cannot be null."); await EnsureConnectedAsync(cancellationToken); - - if (_channel is null) throw new InvalidOperationException("RabbitMQ channel is not available."); - - if (_autoCreateTopics) - { - await CreateTopicIfNotExistsAsync(topic, cancellationToken); - } + await EnsureTopicExistsAsync(topic, cancellationToken); - await _channel.ExchangeDeclareAsync(topic, ExchangeType.Fanout, durable: true, autoDelete: false, cancellationToken: cancellationToken); var queueName = $"{consumerGroup}.{topic}"; await _channel.QueueDeclareAsync(queueName, durable: true, exclusive: false, autoDelete: false, cancellationToken: cancellationToken); await _channel.QueueBindAsync(queueName, topic, "", cancellationToken: cancellationToken); @@ -93,13 +86,7 @@ public async Task PublishAsync(string topic, string message, CancellationToken c throw new ArgumentException("Message cannot be null or empty.", nameof(message)); await EnsureConnectedAsync(cancellationToken); - - if (_channel is null) throw new InvalidOperationException("RabbitMQ channel is not available."); - - if (_autoCreateTopics) - { - await CreateTopicIfNotExistsAsync(topic, cancellationToken); - } + await EnsureTopicExistsAsync(topic, cancellationToken); var body = Encoding.UTF8.GetBytes(message); await _channel.BasicPublishAsync(topic, "", body, cancellationToken: cancellationToken); @@ -115,22 +102,24 @@ private async Task EnsureConnectedAsync(CancellationToken cancellationToken) _channel = await _connection.CreateChannelAsync(null, cancellationToken); } - private async Task CreateTopicIfNotExistsAsync(string topic, CancellationToken cancellationToken) + private async Task EnsureTopicExistsAsync(string topic, CancellationToken cancellationToken) { bool exchangeExists = await ExchangeExistsAsync(topic, cancellationToken); - if (!exchangeExists) + if (exchangeExists) + { + _logger.LogDebug("Exchange '{Topic}' already exists.", topic); + return; + } + + if (_autoCreateTopics) + { + await _channel.ExchangeDeclareAsync(topic, ExchangeType.Fanout, durable: true, autoDelete: false, cancellationToken: cancellationToken); + _logger.LogInformation("Auto-created exchange (topic): {Topic}", topic); + } + else { - if (_autoCreateTopics) - { - await _channel.ExchangeDeclareAsync(topic, ExchangeType.Fanout, durable: true, autoDelete: false, cancellationToken: cancellationToken); - _logger.LogInformation("Auto-created exchange (topic): {Topic}", topic); - } - else - { - _logger.LogError("Cannot subscribe to non-existent topic: {Topic}", topic); - throw new InvalidOperationException($"Topic '{topic}' does not exist."); - } + throw new InvalidOperationException($"Topic '{topic}' does not exist. Enable 'autoCreateTopics' to create topics automatically."); } } @@ -145,7 +134,6 @@ private async Task ExchangeExistsAsync(string exchange, CancellationToken { _logger.LogDebug("Exchange '{Exchange}' does not exist.", exchange); - // Since the channel was closed, reopen it before returning await EnsureConnectedAsync(cancellationToken); return false; From c7a93ee7476715a7f6c031add9e4f75f0b8e5a22 Mon Sep 17 00:00:00 2001 From: David Runemalm Date: Sat, 8 Mar 2025 09:45:17 +0700 Subject: [PATCH 100/109] Refactor messaging provider tests a bit. --- .../Azure/AzureServiceBusMessagingProviderTests.cs | 12 ++++-------- .../Events/Kafka/KafkaMessagingProviderTests.cs | 3 +++ .../RabbitMq/RabbitMqMessagingProviderTests.cs | 6 +++--- 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/src/OpenDDD.Tests/Integration/Infrastructure/Events/Azure/AzureServiceBusMessagingProviderTests.cs b/src/OpenDDD.Tests/Integration/Infrastructure/Events/Azure/AzureServiceBusMessagingProviderTests.cs index bee513d..894fd15 100644 --- a/src/OpenDDD.Tests/Integration/Infrastructure/Events/Azure/AzureServiceBusMessagingProviderTests.cs +++ b/src/OpenDDD.Tests/Integration/Infrastructure/Events/Azure/AzureServiceBusMessagingProviderTests.cs @@ -72,10 +72,8 @@ public async Task AutoCreateTopic_ShouldCreateTopicOnSubscribe_WhenSettingEnable var topicName = $"test-topic-{Guid.NewGuid()}"; var subscriptionName = "test-subscription"; - if (await _adminClient.TopicExistsAsync(topicName)) - { - await _adminClient.DeleteTopicAsync(topicName); - } + var topicExistsBefore = (await _adminClient.TopicExistsAsync(topicName)).Value; + topicExistsBefore.Should().BeFalse("The topic should not exist before subscribing."); // Act await _messagingProvider.SubscribeAsync(topicName, subscriptionName, (msg, token) => Task.CompletedTask); @@ -90,10 +88,8 @@ public async Task AutoCreateTopic_ShouldNotCreateTopicOnSubscribe_WhenSettingDis // Arrange var topicName = $"test-topic-{Guid.NewGuid()}"; - if (await _adminClient.TopicExistsAsync(topicName)) - { - await _adminClient.DeleteTopicAsync(topicName); - } + var topicExistsBefore = (await _adminClient.TopicExistsAsync(topicName)).Value; + topicExistsBefore.Should().BeFalse("The topic should not exist before subscribing."); var messagingProvider = new AzureServiceBusMessagingProvider( _serviceBusClient, diff --git a/src/OpenDDD.Tests/Integration/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs b/src/OpenDDD.Tests/Integration/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs index 612246d..38c0635 100644 --- a/src/OpenDDD.Tests/Integration/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs +++ b/src/OpenDDD.Tests/Integration/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs @@ -131,6 +131,9 @@ public async Task AutoCreateTopic_ShouldNotCreateTopicOnSubscribe_WhenSettingDis var topicName = $"test-topic-{Guid.NewGuid()}"; var consumerGroup = "test-consumer-group"; + var metadata = _adminClient.GetMetadata(TimeSpan.FromSeconds(5)); + metadata.Topics.Any(t => t.Topic == topicName).Should().BeFalse("The topic should not exist before subscribing."); + var messagingProvider = new KafkaMessagingProvider( _adminClient, _producer, diff --git a/src/OpenDDD.Tests/Integration/Infrastructure/Events/RabbitMq/RabbitMqMessagingProviderTests.cs b/src/OpenDDD.Tests/Integration/Infrastructure/Events/RabbitMq/RabbitMqMessagingProviderTests.cs index 7be87c6..c7c9310 100644 --- a/src/OpenDDD.Tests/Integration/Infrastructure/Events/RabbitMq/RabbitMqMessagingProviderTests.cs +++ b/src/OpenDDD.Tests/Integration/Infrastructure/Events/RabbitMq/RabbitMqMessagingProviderTests.cs @@ -172,15 +172,15 @@ public async Task AutoCreateTopic_ShouldNotCreateTopicOnSubscribe_WhenSettingDis // Arrange var topicName = $"test-topic-{Guid.NewGuid()}"; var consumerGroup = "test-consumer-group"; + + var exchangeExistsBefore = await ExchangeExistsAsync(topicName, _cts.Token); + exchangeExistsBefore.Should().BeFalse("The exchange should not exist before subscribing."); var messagingProvider = new RabbitMqMessagingProvider( _connectionFactory, _consumerFactory, autoCreateTopics: false, // Auto-create disabled _logger); - - var exchangeExistsBefore = await ExchangeExistsAsync(topicName, _cts.Token); - exchangeExistsBefore.Should().BeFalse("The exchange should not exist before subscribing."); // Act & Assert var exception = await Assert.ThrowsAsync(async () => From a80a69bf95c4bebc0418c52b940b50620c88812f Mon Sep 17 00:00:00 2001 From: David Runemalm Date: Sat, 8 Mar 2025 09:46:10 +0700 Subject: [PATCH 101/109] Add a couple of test methods for publishing with auto-create true/false to messaging provider tests. --- .../AzureServiceBusMessagingProviderTests.cs | 47 ++++++++++++++++++ .../Kafka/KafkaMessagingProviderTests.cs | 49 +++++++++++++++++++ .../RabbitMqMessagingProviderTests.cs | 47 ++++++++++++++++++ 3 files changed, 143 insertions(+) diff --git a/src/OpenDDD.Tests/Integration/Infrastructure/Events/Azure/AzureServiceBusMessagingProviderTests.cs b/src/OpenDDD.Tests/Integration/Infrastructure/Events/Azure/AzureServiceBusMessagingProviderTests.cs index 894fd15..2a88694 100644 --- a/src/OpenDDD.Tests/Integration/Infrastructure/Events/Azure/AzureServiceBusMessagingProviderTests.cs +++ b/src/OpenDDD.Tests/Integration/Infrastructure/Events/Azure/AzureServiceBusMessagingProviderTests.cs @@ -108,6 +108,53 @@ public async Task AutoCreateTopic_ShouldNotCreateTopicOnSubscribe_WhenSettingDis exception.Message.Should().Be($"Topic '{topicName}' does not exist. Enable 'autoCreateTopics' to create topics automatically."); } + [Fact] + public async Task AutoCreateTopic_ShouldCreateTopicOnPublish_WhenSettingEnabled() + { + // Arrange + var topicName = $"test-topic-{Guid.NewGuid()}"; + + var topicExistsBefore = await _adminClient.TopicExistsAsync(topicName); + topicExistsBefore.Value.Should().BeFalse("The topic should not exist before publishing."); + + var messagingProvider = new AzureServiceBusMessagingProvider( + _serviceBusClient, + _adminClient, + autoCreateTopics: true, // Auto-create enabled + _logger); + + // Act + await messagingProvider.PublishAsync(topicName, "Test message", _cts.Token); + + // Assert + var topicExistsAfter = await _adminClient.TopicExistsAsync(topicName); + topicExistsAfter.Value.Should().BeTrue("Azure Service Bus should create the topic automatically when publishing."); + } + + [Fact] + public async Task AutoCreateTopic_ShouldNotCreateTopicOnPublish_WhenSettingDisabled() + { + // Arrange + var topicName = $"test-topic-{Guid.NewGuid()}"; + + var topicExistsBefore = await _adminClient.TopicExistsAsync(topicName); + topicExistsBefore.Value.Should().BeFalse("The topic should not exist before publishing."); + + var messagingProvider = new AzureServiceBusMessagingProvider( + _serviceBusClient, + _adminClient, + autoCreateTopics: false, // Auto-create disabled + _logger); + + // Act & Assert + var exception = await Assert.ThrowsAsync(async () => + { + await messagingProvider.PublishAsync(topicName, "Test message", _cts.Token); + }); + + exception.Message.Should().Contain($"Topic '{topicName}' does not exist. Enable 'autoCreateTopics' to create topics automatically."); + } + [Fact] public async Task AtLeastOnceGurantee_ShouldDeliverToLateSubscriber_WhenSubscribedBefore() { diff --git a/src/OpenDDD.Tests/Integration/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs b/src/OpenDDD.Tests/Integration/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs index 38c0635..3872434 100644 --- a/src/OpenDDD.Tests/Integration/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs +++ b/src/OpenDDD.Tests/Integration/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs @@ -151,6 +151,55 @@ await messagingProvider.SubscribeAsync(topicName, consumerGroup, async (msg, tok exception.Message.Should().Be($"Topic '{topicName}' does not exist. Enable 'autoCreateTopics' to create topics automatically."); } + [Fact] + public async Task AutoCreateTopic_ShouldCreateTopicOnPublish_WhenSettingEnabled() + { + // Arrange + var topicName = $"test-topic-{Guid.NewGuid()}"; + + var metadataBefore = _adminClient.GetMetadata(TimeSpan.FromSeconds(5)); + metadataBefore.Topics.Any(t => t.Topic == topicName).Should().BeFalse("The topic should not exist before publishing."); + + var messagingProvider = new KafkaMessagingProvider( + _adminClient, + _producer, + _consumerFactory, + autoCreateTopics: true, // Auto-create enabled + _logger); + + // Act + await messagingProvider.PublishAsync(topicName, "Test message", _cts.Token); + + // Assert + var metadataAfter = _adminClient.GetMetadata(TimeSpan.FromSeconds(5)); + metadataAfter.Topics.Any(t => t.Topic == topicName).Should().BeTrue("Kafka should create the topic automatically when publishing."); + } + + [Fact] + public async Task AutoCreateTopic_ShouldNotCreateTopicOnPublish_WhenSettingDisabled() + { + // Arrange + var topicName = $"test-topic-{Guid.NewGuid()}"; + + var metadataBefore = _adminClient.GetMetadata(TimeSpan.FromSeconds(5)); + metadataBefore.Topics.Any(t => t.Topic == topicName).Should().BeFalse("The topic should not exist before publishing."); + + var messagingProvider = new KafkaMessagingProvider( + _adminClient, + _producer, + _consumerFactory, + autoCreateTopics: false, // Auto-create disabled + _logger); + + // Act & Assert + var exception = await Assert.ThrowsAsync(async () => + { + await messagingProvider.PublishAsync(topicName, "Test message", _cts.Token); + }); + + exception.Message.Should().Contain($"Topic '{topicName}' does not exist. Enable 'autoCreateTopics' to create topics automatically."); + } + [Fact] public async Task AtLeastOnceGurantee_ShouldDeliverToLateSubscriber_WhenSubscribedBefore() { diff --git a/src/OpenDDD.Tests/Integration/Infrastructure/Events/RabbitMq/RabbitMqMessagingProviderTests.cs b/src/OpenDDD.Tests/Integration/Infrastructure/Events/RabbitMq/RabbitMqMessagingProviderTests.cs index c7c9310..caa3e56 100644 --- a/src/OpenDDD.Tests/Integration/Infrastructure/Events/RabbitMq/RabbitMqMessagingProviderTests.cs +++ b/src/OpenDDD.Tests/Integration/Infrastructure/Events/RabbitMq/RabbitMqMessagingProviderTests.cs @@ -192,6 +192,53 @@ await messagingProvider.SubscribeAsync(topicName, consumerGroup, async (msg, tok exception.Message.Should().Be($"Topic '{topicName}' does not exist. Enable 'autoCreateTopics' to create topics automatically."); } + [Fact] + public async Task AutoCreateTopic_ShouldCreateTopicOnPublish_WhenSettingEnabled() + { + // Arrange + var topicName = $"test-topic-{Guid.NewGuid()}"; + + var exchangeExistsBefore = await ExchangeExistsAsync(topicName, _cts.Token); + exchangeExistsBefore.Should().BeFalse("The exchange should not exist before publishing."); + + var messagingProvider = new RabbitMqMessagingProvider( + _connectionFactory, + _consumerFactory, + autoCreateTopics: true, // Auto-create enabled + _logger); + + // Act + await messagingProvider.PublishAsync(topicName, "Test message", _cts.Token); + + // Assert + var exchangeExistsAfter = await ExchangeExistsAsync(topicName, _cts.Token); + exchangeExistsAfter.Should().BeTrue("RabbitMQ should create the exchange automatically when publishing."); + } + + [Fact] + public async Task AutoCreateTopic_ShouldNotCreateTopicOnPublish_WhenSettingDisabled() + { + // Arrange + var topicName = $"test-topic-{Guid.NewGuid()}"; + + var exchangeExistsBefore = await ExchangeExistsAsync(topicName, _cts.Token); + exchangeExistsBefore.Should().BeFalse("The exchange should not exist before publishing."); + + var messagingProvider = new RabbitMqMessagingProvider( + _connectionFactory, + _consumerFactory, + autoCreateTopics: false, // Auto-create disabled + _logger); + + // Act & Assert + var exception = await Assert.ThrowsAsync(async () => + { + await messagingProvider.PublishAsync(topicName, "Test message", _cts.Token); + }); + + exception.Message.Should().Contain($"Topic '{topicName}' does not exist. Enable 'autoCreateTopics' to create topics automatically."); + } + [Fact] public async Task AtLeastOnceGurantee_ShouldDeliverToLateSubscriber_WhenSubscribedBefore() { From 126a26d70bed4956f3836a7407d8aeb3c485cd02 Mon Sep 17 00:00:00 2001 From: David Runemalm Date: Sat, 8 Mar 2025 10:02:32 +0700 Subject: [PATCH 102/109] Cache topic checks for ten minutes in messaging providers. --- .../Kafka/KafkaMessagingProviderTests.cs | 2 +- .../Azure/AzureServiceBusMessagingProvider.cs | 33 +++++++++++++------ .../Events/Kafka/KafkaMessagingProvider.cs | 11 +++++++ .../RabbitMq/RabbitMqMessagingProvider.cs | 24 ++++++++++---- 4 files changed, 53 insertions(+), 17 deletions(-) diff --git a/src/OpenDDD.Tests/Integration/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs b/src/OpenDDD.Tests/Integration/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs index 3872434..90cc136 100644 --- a/src/OpenDDD.Tests/Integration/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs +++ b/src/OpenDDD.Tests/Integration/Infrastructure/Events/Kafka/KafkaMessagingProviderTests.cs @@ -365,7 +365,7 @@ public async Task CompetingConsumers_ShouldDistributeMessages_WhenMultipleConsum var consumerGroup = "test-consumer-group"; var totalMessages = 100; var numConsumers = 2; - var variancePercentage = 0.2; + var variancePercentage = 0.3; var perConsumerMessageCount = new ConcurrentDictionary(); // Track messages per consumer var allMessagesProcessed = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); diff --git a/src/OpenDDD/Infrastructure/Events/Azure/AzureServiceBusMessagingProvider.cs b/src/OpenDDD/Infrastructure/Events/Azure/AzureServiceBusMessagingProvider.cs index ef755ab..ba7bd4f 100644 --- a/src/OpenDDD/Infrastructure/Events/Azure/AzureServiceBusMessagingProvider.cs +++ b/src/OpenDDD/Infrastructure/Events/Azure/AzureServiceBusMessagingProvider.cs @@ -13,6 +13,8 @@ public class AzureServiceBusMessagingProvider : IMessagingProvider, IAsyncDispos private readonly bool _autoCreateTopics; private readonly ILogger _logger; private readonly ConcurrentDictionary _subscriptions = new(); + private readonly ConcurrentDictionary _topicCache = new(); + private readonly TimeSpan _cacheExpiration = TimeSpan.FromSeconds(600); private bool _disposed; public AzureServiceBusMessagingProvider( @@ -105,18 +107,29 @@ public async Task PublishAsync(string topic, string message, CancellationToken c private async Task EnsureTopicExistsAsync(string topic, CancellationToken cancellationToken) { - if (!await _adminClient.TopicExistsAsync(topic, cancellationToken)) + if (_topicCache.TryGetValue(topic, out var lastChecked) && DateTime.UtcNow - lastChecked < _cacheExpiration) { - if (_autoCreateTopics) - { - await _adminClient.CreateTopicAsync(topic, cancellationToken); - _logger.LogInformation("Created topic: {Topic}", topic); - } - else - { - throw new InvalidOperationException($"Topic '{topic}' does not exist. Enable 'autoCreateTopics' to create topics automatically."); - } + _logger.LogDebug("Skipping topic check for '{Topic}' (cached result).", topic); + return; + } + + var topicExists = await _adminClient.TopicExistsAsync(topic, cancellationToken); + + if (topicExists) + { + _topicCache[topic] = DateTime.UtcNow; + return; + } + + if (_autoCreateTopics) + { + await _adminClient.CreateTopicAsync(topic, cancellationToken); + _logger.LogInformation("Created topic: {Topic}", topic); + _topicCache[topic] = DateTime.UtcNow; + return; } + + throw new InvalidOperationException($"Topic '{topic}' does not exist. Enable 'autoCreateTopics' to create topics automatically."); } private async Task CreateSubscriptionIfNotExistsAsync(string topic, string subscriptionName, CancellationToken cancellationToken) diff --git a/src/OpenDDD/Infrastructure/Events/Kafka/KafkaMessagingProvider.cs b/src/OpenDDD/Infrastructure/Events/Kafka/KafkaMessagingProvider.cs index 6297c0e..189c3d9 100644 --- a/src/OpenDDD/Infrastructure/Events/Kafka/KafkaMessagingProvider.cs +++ b/src/OpenDDD/Infrastructure/Events/Kafka/KafkaMessagingProvider.cs @@ -15,6 +15,8 @@ public class KafkaMessagingProvider : IMessagingProvider, IAsyncDisposable private readonly IKafkaConsumerFactory _consumerFactory; private readonly ILogger _logger; private readonly ConcurrentDictionary _subscriptions = new(); + private readonly ConcurrentDictionary _topicCache = new(); + private readonly TimeSpan _cacheExpiration = TimeSpan.FromSeconds(600); private readonly CancellationTokenSource _cts = new(); private bool _disposed; @@ -93,11 +95,18 @@ public async Task PublishAsync(string topic, string message, CancellationToken c private async Task EnsureTopicExistsAsync(string topic, CancellationToken cancellationToken) { + if (_topicCache.TryGetValue(topic, out var lastChecked) && DateTime.UtcNow - lastChecked < _cacheExpiration) + { + _logger.LogDebug("Skipping topic check for '{Topic}' (cached result).", topic); + return; + } + var metadata = _adminClient.GetMetadata(TimeSpan.FromSeconds(5)); if (metadata.Topics.Any(t => t.Topic == topic)) { _logger.LogDebug("Topic '{Topic}' already exists.", topic); + _topicCache[topic] = DateTime.UtcNow; return; } @@ -109,6 +118,8 @@ await _adminClient.CreateTopicsAsync(new[] new TopicSpecification { Name = topic, NumPartitions = 2, ReplicationFactor = 1 } }, null); + _topicCache[topic] = DateTime.UtcNow; + // Wait for the topic to be available for (int i = 0; i < 30; i++) { diff --git a/src/OpenDDD/Infrastructure/Events/RabbitMq/RabbitMqMessagingProvider.cs b/src/OpenDDD/Infrastructure/Events/RabbitMq/RabbitMqMessagingProvider.cs index 410b54d..fbb5eb4 100644 --- a/src/OpenDDD/Infrastructure/Events/RabbitMq/RabbitMqMessagingProvider.cs +++ b/src/OpenDDD/Infrastructure/Events/RabbitMq/RabbitMqMessagingProvider.cs @@ -17,6 +17,9 @@ public class RabbitMqMessagingProvider : IMessagingProvider, IAsyncDisposable private IConnection? _connection; private IChannel? _channel; private readonly ConcurrentDictionary _subscriptions = new(); + private readonly ConcurrentDictionary _topicCache = new(); + private readonly TimeSpan _cacheExpiration = TimeSpan.FromSeconds(600); + private bool _disposed; public RabbitMqMessagingProvider( IConnectionFactory factory, @@ -104,11 +107,17 @@ private async Task EnsureConnectedAsync(CancellationToken cancellationToken) private async Task EnsureTopicExistsAsync(string topic, CancellationToken cancellationToken) { - bool exchangeExists = await ExchangeExistsAsync(topic, cancellationToken); + if (_topicCache.TryGetValue(topic, out var lastChecked) && DateTime.UtcNow - lastChecked < _cacheExpiration) + { + _logger.LogDebug("Skipping exchange check for '{Topic}' (cached result).", topic); + return; + } + bool exchangeExists = await ExchangeExistsAsync(topic, cancellationToken); + if (exchangeExists) { - _logger.LogDebug("Exchange '{Topic}' already exists.", topic); + _topicCache[topic] = DateTime.UtcNow; return; } @@ -116,11 +125,11 @@ private async Task EnsureTopicExistsAsync(string topic, CancellationToken cancel { await _channel.ExchangeDeclareAsync(topic, ExchangeType.Fanout, durable: true, autoDelete: false, cancellationToken: cancellationToken); _logger.LogInformation("Auto-created exchange (topic): {Topic}", topic); + _topicCache[topic] = DateTime.UtcNow; + return; } - else - { - throw new InvalidOperationException($"Topic '{topic}' does not exist. Enable 'autoCreateTopics' to create topics automatically."); - } + + throw new InvalidOperationException($"Topic '{topic}' does not exist. Enable 'autoCreateTopics' to create topics automatically."); } private async Task ExchangeExistsAsync(string exchange, CancellationToken cancellationToken) @@ -147,6 +156,9 @@ private async Task ExchangeExistsAsync(string exchange, CancellationToken public async ValueTask DisposeAsync() { + if (_disposed) return; + _disposed = true; + _logger.LogDebug("Disposing RabbitMqMessagingProvider..."); foreach (var subscription in _subscriptions.Values) From 25306f31abac1117c9568bd8c7475e647a4d5fb1 Mon Sep 17 00:00:00 2001 From: David Runemalm Date: Sat, 8 Mar 2025 19:44:11 +0700 Subject: [PATCH 103/109] Update version to beta.2. --- Makefile | 4 ++-- src/OpenDDD/OpenDDD.csproj | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 1e239ab..d1e99dd 100644 --- a/Makefile +++ b/Makefile @@ -18,7 +18,7 @@ export $(shell sed 's/=.*//' env.make) HOME := $(shell echo ~) PWD := $(shell pwd) NETWORK := openddd-net -BUILD_VERSION := 3.0.0-beta.1 +BUILD_VERSION := 3.0.0-beta.2 NUGET_NAME := OpenDDD.NET ROOT_NAMESPACE := OpenDDD @@ -215,7 +215,7 @@ TEMPLATES_DIR := $(PWD)/templates TEMPLATES_CSPROJ := $(TEMPLATES_DIR)/templatepack.csproj TEMPLATES_OUT := $(TEMPLATES_DIR)/bin/templates TEMPLATES_NAME := OpenDDD.NET-Templates -TEMPLATES_VERSION := 3.0.0-alpha.1 +TEMPLATES_VERSION := 3.0.0-beta.2 TEMPLATES_NUPKG := $(TEMPLATES_OUT)/$(TEMPLATES_NAME).$(TEMPLATES_VERSION).nupkg .PHONY: templates-install diff --git a/src/OpenDDD/OpenDDD.csproj b/src/OpenDDD/OpenDDD.csproj index fd6eff6..ceca2ad 100644 --- a/src/OpenDDD/OpenDDD.csproj +++ b/src/OpenDDD/OpenDDD.csproj @@ -7,7 +7,7 @@ false true OpenDDD.NET - 3.0.0-alpha.3 + 3.0.0-beta.2 David Runemalm A framework for domain-driven design using C# and .NET. DDD;Domain-Driven Design;C#;.NET;Hexagonal Architecture From 1df95b399adbd0ca43760b377af7abef335c8f3e Mon Sep 17 00:00:00 2001 From: David Runemalm Date: Sat, 8 Mar 2025 19:45:11 +0700 Subject: [PATCH 104/109] Cleanup a bit. --- .../Events/Kafka/Factories/KafkaConsumer.cs | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/src/OpenDDD/Infrastructure/Events/Kafka/Factories/KafkaConsumer.cs b/src/OpenDDD/Infrastructure/Events/Kafka/Factories/KafkaConsumer.cs index 5c38814..39e1c83 100644 --- a/src/OpenDDD/Infrastructure/Events/Kafka/Factories/KafkaConsumer.cs +++ b/src/OpenDDD/Infrastructure/Events/Kafka/Factories/KafkaConsumer.cs @@ -86,7 +86,6 @@ public async Task StopProcessingAsync() _consumerTask = null; } - // Make sure all partitions have initial commit offset await CommitInitialOffsetsAsync(); _consumer.Close(); @@ -97,20 +96,17 @@ private async Task CommitInitialOffsetsAsync() { foreach (var partition in _consumer.Assignment) { - // Check if an offset is already committed var committedOffsets = _consumer.Committed(new[] { partition }, TimeSpan.FromSeconds(5)); var currentOffset = committedOffsets?.FirstOrDefault()?.Offset; if (currentOffset == null || currentOffset == Offset.Unset) { - // Query watermark offsets (LOW = earliest, HIGH = next available offset) var watermarkOffsets = _consumer.QueryWatermarkOffsets(partition, TimeSpan.FromSeconds(5)); if (watermarkOffsets.High == 0) // No messages ever written { _logger.LogDebug("Partition {Partition} has no messages. Sending placeholder message.", partition); - // Publish a placeholder message to the **specific partition** using var producer = new ProducerBuilder(new ProducerConfig { BootstrapServers = "localhost:9092" }).Build(); var deliveryResult = await producer.ProduceAsync( new TopicPartition(partition.Topic, partition.Partition), // Ensure correct partition @@ -120,16 +116,15 @@ private async Task CommitInitialOffsetsAsync() _logger.LogDebug("Sent __init__ message to partition {Partition}. Offset: {Offset}", partition, deliveryResult.Offset); - // Poll Kafka until watermark updates or timeout occurs var timeout = TimeSpan.FromSeconds(5); var startTime = DateTime.UtcNow; while (DateTime.UtcNow - startTime < timeout) { - await Task.Delay(100); // Small delay to avoid excessive polling + await Task.Delay(100); watermarkOffsets = _consumer.QueryWatermarkOffsets(partition, TimeSpan.FromSeconds(5)); - if (watermarkOffsets.High > 0) // Message registered + if (watermarkOffsets.High > 0) break; } @@ -138,7 +133,7 @@ private async Task CommitInitialOffsetsAsync() throw new TimeoutException($"Kafka did not register the __init__ message for partition {partition} within 5 seconds."); } - // Ensure the new high watermark is exactly 1 + // Ensure the new offset for the partition is exactly 1 var initialOffset = watermarkOffsets.High; if (initialOffset.Value != 1) { From 18e7aecf5ee34c3cda02f1c77229e92d71fe8a35 Mon Sep 17 00:00:00 2001 From: David Runemalm Date: Sat, 8 Mar 2025 19:45:39 +0700 Subject: [PATCH 105/109] Remove tests that shouldn't be in sample and/or template projects. --- .../EfCore/EfCoreConfigurationTests.cs | 84 ------------------- .../EfCore/EfCoreConfigurationTests.cs | 84 ------------------- 2 files changed, 168 deletions(-) delete mode 100644 samples/Bookstore/src/Bookstore/Tests/Infrastructure/Persistence/EfCore/EfCoreConfigurationTests.cs delete mode 100644 templates/Bookstore/src/Bookstore/Tests/Infrastructure/Persistence/EfCore/EfCoreConfigurationTests.cs diff --git a/samples/Bookstore/src/Bookstore/Tests/Infrastructure/Persistence/EfCore/EfCoreConfigurationTests.cs b/samples/Bookstore/src/Bookstore/Tests/Infrastructure/Persistence/EfCore/EfCoreConfigurationTests.cs deleted file mode 100644 index fddcb62..0000000 --- a/samples/Bookstore/src/Bookstore/Tests/Infrastructure/Persistence/EfCore/EfCoreConfigurationTests.cs +++ /dev/null @@ -1,84 +0,0 @@ -using Xunit; -using Microsoft.EntityFrameworkCore; -using Microsoft.Extensions.Options; -using OpenDDD.Infrastructure.Persistence.EfCore.UoW; -using OpenDDD.Infrastructure.Persistence.UoW; -using OpenDDD.API.Options; -using OpenDDD.Domain.Model; -using OpenDDD.Infrastructure.Persistence.EfCore.Base; -using OpenDDD.Infrastructure.Repository.EfCore; -using OpenDDD.Infrastructure.Persistence.DatabaseSession; -using OpenDDD.Infrastructure.Persistence.EfCore.DatabaseSession; -using OpenDDD.Infrastructure.Events; -using OpenDDD.Infrastructure.TransactionalOutbox; -using OpenDDD.Infrastructure.TransactionalOutbox.EfCore; -using Bookstore.Domain.Model; -using Bookstore.Infrastructure.Persistence.EfCore; - -namespace Bookstore.Tests.Infrastructure.Persistence.EfCore -{ - public class EfCoreConfigurationTests - { - private readonly IServiceProvider _serviceProvider; - - public EfCoreConfigurationTests() - { - var services = new ServiceCollection(); - - // Register logging - services.AddLogging(); - - // Manually configure OpenDDD options - var options = new OpenDddOptions(); - services.AddSingleton(Options.Create(options)); - services.AddSingleton(options); - - // Add an in-memory database - services.AddDbContext(opts => - opts.UseInMemoryDatabase("TestDatabase")); - services.AddScoped(sp => sp.GetRequiredService()); - - // Register EfCoreDatabaseSession as the IDatabaseSession - services.AddScoped(); - services.AddScoped(sp => sp.GetRequiredService()); - - // Register dependencies - services.AddScoped(); - services.AddScoped(typeof(IRepository), typeof(EfCoreRepository)); - - // Register publishers - services.AddScoped(); - services.AddScoped(); - - // Register IOutboxRepository (EF Core implementation) - services.AddScoped(); - - _serviceProvider = services.BuildServiceProvider(); - } - - [Fact] - public async Task CreateAndRetrieveOrder_WithLineItems_ShouldPersistCorrectly() - { - using var scope = _serviceProvider.CreateScope(); - var repository = scope.ServiceProvider.GetRequiredService>(); - - var ct = CancellationToken.None; - - // Arrange - Create and save an order with line items - var order = Order.Create(Guid.NewGuid()); - order.AddLineItem(Guid.NewGuid(), Money.USD(19.99m)); - order.AddLineItem(Guid.NewGuid(), Money.USD(29.99m)); - - await repository.SaveAsync(order, ct); - - // Act - Retrieve order - var retrievedOrder = await repository.GetAsync(order.Id, ct); - - // Assert - Order and line items should be persisted - Assert.NotNull(retrievedOrder); - Assert.Equal(order.Id, retrievedOrder.Id); - Assert.NotEmpty(retrievedOrder.LineItems); - Assert.Equal(2, retrievedOrder.LineItems.Count); - } - } -} diff --git a/templates/Bookstore/src/Bookstore/Tests/Infrastructure/Persistence/EfCore/EfCoreConfigurationTests.cs b/templates/Bookstore/src/Bookstore/Tests/Infrastructure/Persistence/EfCore/EfCoreConfigurationTests.cs deleted file mode 100644 index e3f2db6..0000000 --- a/templates/Bookstore/src/Bookstore/Tests/Infrastructure/Persistence/EfCore/EfCoreConfigurationTests.cs +++ /dev/null @@ -1,84 +0,0 @@ -using Xunit; -using Microsoft.EntityFrameworkCore; -using Microsoft.Extensions.Options; -using OpenDDD.Infrastructure.Persistence.EfCore.UoW; -using OpenDDD.Infrastructure.Persistence.UoW; -using OpenDDD.API.Options; -using OpenDDD.Domain.Model; -using OpenDDD.Infrastructure.Persistence.EfCore.Base; -using OpenDDD.Infrastructure.Repository.EfCore; -using OpenDDD.Infrastructure.Persistence.DatabaseSession; -using OpenDDD.Infrastructure.Persistence.EfCore.DatabaseSession; -using Bookstore.Domain.Model; -using Bookstore.Infrastructure.Persistence.EfCore; -using OpenDDD.Infrastructure.Events; -using OpenDDD.Infrastructure.TransactionalOutbox; -using OpenDDD.Infrastructure.TransactionalOutbox.EfCore; - -namespace Bookstore.Tests.Infrastructure.Persistence.EfCore -{ - public class EfCoreConfigurationTests - { - private readonly IServiceProvider _serviceProvider; - - public EfCoreConfigurationTests() - { - var services = new ServiceCollection(); - - // Register logging - services.AddLogging(); - - // Manually configure OpenDDD options - var options = new OpenDddOptions(); - services.AddSingleton(Options.Create(options)); - services.AddSingleton(options); - - // Add an in-memory database - services.AddDbContext(opts => - opts.UseInMemoryDatabase("TestDatabase")); - services.AddScoped(sp => sp.GetRequiredService()); - - // Register EfCoreDatabaseSession as the IDatabaseSession - services.AddScoped(); - services.AddScoped(sp => sp.GetRequiredService()); - - // Register dependencies - services.AddScoped(); - services.AddScoped(typeof(IRepository), typeof(EfCoreRepository)); - - // Register publishers - services.AddScoped(); - services.AddScoped(); - - // Register IOutboxRepository (EF Core implementation) - services.AddScoped(); - - _serviceProvider = services.BuildServiceProvider(); - } - - [Fact] - public async Task CreateAndRetrieveOrder_WithLineItems_ShouldPersistCorrectly() - { - using var scope = _serviceProvider.CreateScope(); - var repository = scope.ServiceProvider.GetRequiredService>(); - - var ct = CancellationToken.None; - - // Arrange - Create and save an order with line items - var order = Order.Create(Guid.NewGuid()); - order.AddLineItem(Guid.NewGuid(), Money.USD(19.99m)); - order.AddLineItem(Guid.NewGuid(), Money.USD(29.99m)); - - await repository.SaveAsync(order, ct); - - // Act - Retrieve order - var retrievedOrder = await repository.GetAsync(order.Id, ct); - - // Assert - Order and line items should be persisted - Assert.NotNull(retrievedOrder); - Assert.Equal(order.Id, retrievedOrder.Id); - Assert.NotEmpty(retrievedOrder.LineItems); - Assert.Equal(2, retrievedOrder.LineItems.Count); - } - } -} From 96b58e77d9a92ac5f11116e70043730cfcf03add Mon Sep 17 00:00:00 2001 From: David Runemalm Date: Sat, 8 Mar 2025 19:46:55 +0700 Subject: [PATCH 106/109] Fix appsettings key in sample and template projects. --- samples/Bookstore/src/Bookstore/appsettings.json | 4 ++-- templates/Bookstore/src/Bookstore/appsettings.json | 10 ++++++---- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/samples/Bookstore/src/Bookstore/appsettings.json b/samples/Bookstore/src/Bookstore/appsettings.json index 14b80e6..ebe6847 100644 --- a/samples/Bookstore/src/Bookstore/appsettings.json +++ b/samples/Bookstore/src/Bookstore/appsettings.json @@ -13,8 +13,8 @@ "DatabaseProvider": "InMemory", "MessagingProvider": "InMemory", "Events": { - "DomainEventTopicTemplate": "Bookstore.Domain.{EventName}", - "IntegrationEventTopicTemplate": "Bookstore.Interchange.{EventName}", + "DomainEventTopic": "Bookstore.Domain.{EventName}", + "IntegrationEventTopic": "Bookstore.Interchange.{EventName}", "ListenerGroup": "Default" }, "SQLite": { diff --git a/templates/Bookstore/src/Bookstore/appsettings.json b/templates/Bookstore/src/Bookstore/appsettings.json index cf4c806..ebe6847 100644 --- a/templates/Bookstore/src/Bookstore/appsettings.json +++ b/templates/Bookstore/src/Bookstore/appsettings.json @@ -13,8 +13,8 @@ "DatabaseProvider": "InMemory", "MessagingProvider": "InMemory", "Events": { - "DomainEventTopicTemplate": "Bookstore.Domain.{EventName}", - "IntegrationEventTopicTemplate": "Bookstore.Interchange.{EventName}", + "DomainEventTopic": "Bookstore.Domain.{EventName}", + "IntegrationEventTopic": "Bookstore.Interchange.{EventName}", "ListenerGroup": "Default" }, "SQLite": { @@ -32,10 +32,12 @@ "Port": 5672, "Username": "guest", "Password": "guest", - "VirtualHost": "/" + "VirtualHost": "/", + "AutoCreateTopics": true }, "Kafka": { - "BootstrapServers": "localhost:9092" + "BootstrapServers": "localhost:9092", + "AutoCreateTopics": true }, "AutoRegister": { "Actions": true, From 2d9050a9296f65e0c1895a017448be026fe3250c Mon Sep 17 00:00:00 2001 From: David Runemalm Date: Sat, 8 Mar 2025 19:48:23 +0700 Subject: [PATCH 107/109] Use beta.2 in template project. --- templates/Bookstore/src/Bookstore/Bookstore.csproj | 2 +- templates/templatepack.csproj | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/Bookstore/src/Bookstore/Bookstore.csproj b/templates/Bookstore/src/Bookstore/Bookstore.csproj index 2830eba..e2613e9 100644 --- a/templates/Bookstore/src/Bookstore/Bookstore.csproj +++ b/templates/Bookstore/src/Bookstore/Bookstore.csproj @@ -13,7 +13,7 @@ all runtime; build; native; contentfiles; analyzers; buildtransitive - + diff --git a/templates/templatepack.csproj b/templates/templatepack.csproj index 4f485c6..53f69dd 100644 --- a/templates/templatepack.csproj +++ b/templates/templatepack.csproj @@ -13,7 +13,7 @@ OpenDDD.NET-Templates - 3.0.0-alpha.1 + 3.0.0-beta.2 David Runemalm Project templates for OpenDDD.NET dotnet-new;templates;openddd.net;ddd;hexagonal From 28f1372703387ad66906ceac8e4d7789a3fb3ab1 Mon Sep 17 00:00:00 2001 From: David Runemalm Date: Thu, 13 Mar 2025 14:27:29 +0700 Subject: [PATCH 108/109] Add release notes. --- README.md | 5 +++++ docs/releases.rst | 5 +++++ samples/Bookstore/Makefile | 26 +++++++++++++++++++++++--- samples/Bookstore/env.make.sample | 1 + 4 files changed, 34 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index b07dbf0..4d4db18 100644 --- a/README.md +++ b/README.md @@ -91,6 +91,11 @@ Explore the project in the repository: [Bookstore Sample Project](https://github ## Release History +**3.0.0-beta.2 (2025-03-13)** + +- **Integration Test Coverage**: Added full integration tests for repositories and messaging providers. +- **Reliability Improvements**: Fixed issues discovered through test coverage. + **3.0.0-beta.1 (2025-02-17)** - **Beta Release**: OpenDDD.NET has moved from alpha to `beta`, indicating improved stability. diff --git a/docs/releases.rst b/docs/releases.rst index a6c86f5..2a79989 100644 --- a/docs/releases.rst +++ b/docs/releases.rst @@ -6,6 +6,11 @@ Version History ############### +**3.0.0-beta.2 (2025-03-13)** + +- **Integration Test Coverage**: Added full integration tests for repositories and messaging providers. +- **Fix Issues**: Fixed issues discovered through test coverage. + **3.0.0-beta.1 (2025-02-17)** - **Beta Release**: OpenDDD.NET has moved from alpha to **beta**, indicating improved stability. diff --git a/samples/Bookstore/Makefile b/samples/Bookstore/Makefile index ef72cbb..96cfb4a 100644 --- a/samples/Bookstore/Makefile +++ b/samples/Bookstore/Makefile @@ -152,13 +152,23 @@ apply-migrations: ##@Migrations Apply all pending migrations to the database --project $(SRC)/Bookstore ########################################################################## -# AZURE +# AZURE SERVICE BUS ########################################################################## .PHONY: azure-create-resource-group -azure-create-resource-group: ##@Azure Create the Azure Resource Group +azure-create-resource-group: ##@Azure Create the Azure Resource Group az group create --name $(AZURE_RESOURCE_GROUP) --location $(AZURE_REGION) +.PHONY: azure-create-service-principal +azure-create-service-principal: ##@Azure Create an Azure Service Principal for GitHub Actions + @echo "Creating Azure Service Principal..." + az ad sp create-for-rbac \ + --name "github-actions-opendddnet" \ + --role "Contributor" \ + --scopes /subscriptions/$(AZURE_SUBSCRIPTION_ID)/resourceGroups/$(AZURE_RESOURCE_GROUP) \ + --sdk-auth + @echo "✅ Copy the output above and add it as 'AZURE_CREDENTIALS' in GitHub Secrets." + .PHONY: azure-create-servicebus-namespace azure-create-servicebus-namespace: ##@Azure Create the Azure Service Bus namespace az servicebus namespace create --name $(AZURE_SERVICEBUS_NAMESPACE) --resource-group $(AZURE_RESOURCE_GROUP) --location $(AZURE_REGION) --sku Standard @@ -172,6 +182,16 @@ azure-get-servicebus-connection: ##@Azure Get the Service Bus connection string --query primaryConnectionString \ --output tsv +.PHONY: azure-delete-servicebus-namespace +azure-delete-servicebus-namespace: ##@Azure Delete the Azure Service Bus namespace + az servicebus namespace delete --resource-group $(AZURE_RESOURCE_GROUP) --name $(AZURE_SERVICEBUS_NAMESPACE) + +########################################################################## +# AZURE SQL SERVER +########################################################################## + + + ########################################################################## # RABBITMQ ########################################################################## @@ -269,7 +289,7 @@ endif @docker exec -it $(KAFKA_CONTAINER) kafka-console-producer.sh --broker-list $(KAFKA_BROKER) --topic $(NAME) ########################################################################## -# POSTGRESQL +# POSTGRES ########################################################################## .PHONY: postgres-start diff --git a/samples/Bookstore/env.make.sample b/samples/Bookstore/env.make.sample index 36968ae..0c44dc5 100644 --- a/samples/Bookstore/env.make.sample +++ b/samples/Bookstore/env.make.sample @@ -1,3 +1,4 @@ +AZURE_SUBSCRIPTION_ID="YOUR_SUB_ID" AZURE_RESOURCE_GROUP=bookstore AZURE_REGION=northeurope AZURE_SERVICEBUS_NAMESPACE=opendddnet-bookstore-sample From 1cc47840e8cfa5d4ee1a66bf3623b516a2765d66 Mon Sep 17 00:00:00 2001 From: David Runemalm Date: Thu, 13 Mar 2025 14:28:43 +0700 Subject: [PATCH 109/109] Bump version in docs conf.py --- docs/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index 755ca56..e6d0c84 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -14,7 +14,7 @@ version = "3.0" # The full version, including alpha/beta/rc tags -release = '3.0.0-beta.1' +release = '3.0.0-beta.2' # -- General configuration --------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration