It turns out I was missing some critical code since Umbraco does not handle the response independently.

This is what the middleware looks like

        namespace Providers;

    using Microsoft.AspNetCore.Authentication.Cookies;
    using Microsoft.AspNetCore.Authentication.OpenIdConnect;
    using Microsoft.Extensions.Configuration;
    using Microsoft.Extensions.DependencyInjection;
    using Microsoft.IdentityModel.Protocols.OpenIdConnect;

    using Umbraco.Cms.Core.DependencyInjection;
    using Umbraco.Cms.Core.Security;
    using Umbraco.Cms.eTransit.Core.Models;
    using Umbraco.Cms.Web.BackOffice.Security;
    using Umbraco.Extensions;

    public static class DuendeExternalLoginProvider
    {
        public static async Task<IUmbracoBuilder> AddDuendeAuthenticationAsync(this IUmbracoBuilder builder)
        {
            var settings = builder.Config.GetSection("DuendeSettings").Get<DuendeSettings>();
            builder.Services.ConfigureOptions<DuendeBackOfficeExternalLoginProviderOptions>();

            //Identity Server 6 Integration
            builder.AddBackOfficeExternalLogins(loginsBuilder =>
                loginsBuilder.AddBackOfficeLogin(build =>
                build.AddOpenIdConnect(DuendeBackOfficeExternalLoginProviderOptions.SchemeName, "Identity Server", options =>
                {
                    options.SignInScheme = CookieAuthenticationDefaults.AuthenticationScheme;
                    options.Authority = settings.Authority;
                    options.ClientId = settings.ClientId;
                    options.ClientSecret = settings.ClientSecret;
                    options.CallbackPath = settings.CallbackUri;
                    options.ResponseType = OpenIdConnectResponseType.Code;
                    options.AuthenticationMethod = OpenIdConnectRedirectBehavior.RedirectGet;
                    options.TokenValidationParameters.NameClaimType = "name";
                    options.TokenValidationParameters.RoleClaimType = "role";
                    options.RequireHttpsMetadata = true;
                    //#if DEBUG
                    //                options.RequireHttpsMetadata = false; // dev only
                    //#endif
                    options.MapInboundClaims = true;
                    options.SaveTokens = true;
                    options.Scope.Add("api1");
                    options.Scope.Add("openid");
                    options.Scope.Add("profile");
                    options.Scope.Add("email");
                    options.Scope.Add("offline_access");
                    options.GetClaimsFromUserInfoEndpoint = true;
                    options.TokenValidationParameters.SaveSigninToken = true;
                    options.Events.OnTicketReceived = async context =>
                    {
                        var userManager = context.HttpContext.RequestServices.GetService<IBackOfficeUserManager>();
                        var signInManager = context.HttpContext.RequestServices.GetService<IBackOfficeSignInManager>();

                        var claims = context?.Principal?.Claims.ToList();

                        if (claims is null) throw new MissingFieldException(nameof(claims));
                        if (userManager is null || signInManager is null) throw new Exception("services not resolved from DI");

                        var email = claims.SingleOrDefault(x => x.Type == "email")?.Value ?? "";
                        var user = await userManager.FindByEmailAsync(email);

                        if (user is not null)
                            await signInManager.SignInAsync(user, false);
                        else
                            await signInManager.SignOutAsync();

                        await Task.FromResult(0);
                    };
                })));
            await Task.FromResult(0);

            return builder;
        }
    }

and was applied here

            namespace Umbraco.Cms.eTransit.Core.Components;

        using Umbraco.Cms.Core.Composing;
        using Umbraco.Cms.Core.DependencyInjection;
        using Umbraco.Cms.eTransit.Core.Providers;

        public class DuendeComponent : IComposer
        {
            public void Compose(IUmbracoBuilder builder)
            {
                builder.AddDuendeAuthenticationAsync().GetAwaiter().GetResult();
            }
        }