Here is my shot at it:

public class SitemapHandler : IHttpHandler
{
    public bool IsReusable => true;

    public void ProcessRequest(HttpContext context)
    {
        GetSitemapXml(context);
    }

    private static readonly string CACHE_KEY = Guid.NewGuid().ToString();

    public static void ClearCache()
    {
        HttpContext.Current.Cache.Remove(CACHE_KEY);
    }

    private void GetSitemapXml(HttpContext context)
    {
        var baseUrl = context.Request.Url.AbsoluteUri.ToLower().Replace("/sitemap.xml", "");

        var factory = DependencyResolver.Current.GetService<IUmbracoContextFactory>();
        using (factory.EnsureUmbracoContext())
        {
            var uHelper = DependencyResolver.Current.GetService<UmbracoHelper>();
            IPublishedContent siteRoot = uHelper.ContentAtRoot().First();

            var cultures = siteRoot.Cultures;

            HttpResponse response = context.Response;
            XDocument xdoc = null;

            if (context.Cache[CACHE_KEY] == null || !(context.Cache[CACHE_KEY] is XDocument))
            {
                xdoc = new XDocument();
                XNamespace ns = "http://www.sitemaps.org/schemas/sitemap/0.9";
                XNamespace xhtml = "http://www.w3.org/1999/xhtml";

                XElement root = new XElement("urlset",
                    new XAttribute("xmlns", ns),
                    new XAttribute(XNamespace.Xmlns + "xhtml", xhtml));

                xdoc.Declaration = new XDeclaration("1.0", "utf-8", "yes");
                xdoc.Add(root);

                var urls = siteRoot.Descendants().Where(d => d.TemplateId > 0).Where(content =>
                    {
                        if (content is INavigationBase naviBase)
                        {
                            return !naviBase.HideInSitemap;
                        }

                        return true;
                    }).SelectMany(content =>
                        cultures.Select(culture =>
                            new
                            {
                                Content = content,
                                Culture = culture.Key,
                                Url = $"{baseUrl}{content.Url(culture.Key)}",
                            })).GroupBy(f => f.Url)
                    .Select(group => group.First())
                    .Where(f => !f.Url.Contains("#"));

                foreach (var url in urls)
                {
                    root.Add(new XElement("url", new XElement("loc", url.Url),
                        new XElement("lastmod", url.Content.UpdateDate.ToString("yyyy-MM-ddTHH:mm:sszzz")),
                        new XElement("changefreq", "weekly")
                    ));
                }

                context.Cache.Insert(CACHE_KEY, xdoc, null, DateTime.Now.AddDays(1), Cache.NoSlidingExpiration);
            }
            else
            {
                xdoc = context.Cache[CACHE_KEY] as XDocument;
            }

            response.Clear();
            response.ContentType = "text/xml";

            using (StreamWriter streamWriter = new StreamWriter(response.OutputStream, Encoding.UTF8))
            {
                XmlTextWriter xmlWriter = new XmlTextWriter(streamWriter);
                xdoc.WriteTo(xmlWriter);
            }

            response.End();
        }
    }
}