I think I got a fix for this. The code below checks if the request is for an item in the media folder and if not it applies the appBuilder.UseStaticFiles().

    public void Configure(IApplicationBuilder app, IWebHostEnvironment env)
    {
        if (env.IsDevelopment())
        {
            app.UseDeveloperExceptionPage();
        }
        var rewriteOptions = new RewriteOptions()
            .AddIISUrlRewrite(env.ContentRootFileProvider, "IISUrlRewrite.xml");

        app.UseRewriter(rewriteOptions);

        // This line is needed for the rewrites to take effect.
        app.UseWhen(context => !IsDynamicImageRequest(context), appBuilder =>
        {
            appBuilder.UseStaticFiles();
        });

        app.UseUmbraco()
            .WithMiddleware(u =>
            {
                u.UseBackOffice();
                u.UseWebsite();
            })
            .WithEndpoints(u =>
            {
                u.UseInstallerEndpoints();
                u.UseBackOfficeEndpoints();
                u.UseWebsiteEndpoints();
                u.EndpointRouteBuilder.MapControllerRoute("sitemap", "sitemap.xml", new { controller = "Sitemap", action = "Sitemap" });
            });
    }

    private bool IsDynamicImageRequest(HttpContext context)
    {
        // Implement logic to determine if the request is for a dynamic image
        // For example, checking if the path starts with "/media/" and has query parameters like "?width="
        return context.Request.Path.StartsWithSegments("/media") && context.Request.QueryString.HasValue;
    }