I'm also struggling with this, in Umbraco 8 I was using the style_formats custom config but have so far failed to get it to work in Umbraco 9 :(

This is an example of what I had in 8

   <customConfig>
       <config key="style_formats">
                [
                {
                "title":"Headings",
                "items":[
                {
                "title":"Top Heading",
                "block":"h1"
                },
                {
                "title":"Heading h2",
                "block": "h2"
                },
                {
                "title":"Sub Head h3",
                "block":"h3"
                },
                {
                "title":"Heading h4",
                "block":"h4"
                },
                {
                "title":"Heading h5",
                "block":"h5"
                },
                {
                "title":"Heading h6",
                "block":"h6"
                }
                ]}
        ]
        </config>
    </customConfig>

I tried the same thing in 9 but it throws an error in the backoffice, An error occurred Cannot create instance of type 'System.String' because it is missing a public parameterless constructor.

"CustomConfig": {
  "style_formats": [
    {
      "title": "Headings",
      "items": [
        {
          "title": "Top Heading",
          "block": "h1"
        },
        {
          "title": "Heading h2",
          "block": "h2"
        },
        {
          "title": "Sub Head h3",
          "block": "h3"
        },
        {
          "title": "Heading h4",
          "block": "h4"
        },
        {
          "title": "Heading h5",
          "block": "h5"
        },
        {
          "title": "Heading h6",
          "block": "h6"
        }
      ]
    }
  ]
}

I have now set mine to do set the style_formats automatically while allowing me to use a nice json file to maintain it in. Instructions in the link below

https://umbraco.themediawizards.co.uk/the-grimoire/formatting-for-richtexteditor-config/

I am using this simple extension method to read a json file with the rte style formats and update the app configuration accordingly:

using System.Text.Json;

namespace Website
{
    /// <summary>
    /// Provides extension method to update <see cref="IConfiguration"/> values.
    /// </summary>
    public static class IConfigurationExtensions
    {
        /// <summary>
        /// Update Umbraco RTE style formats from json file
        /// </summary>
        /// <param name="config">The <see cref="IConfiguration"/> object of the app.</param>
        /// <param name="pathToUmbracoRtfStyleFormatsJsonFile">The file with the formats json.</param>
        /// <exception cref="ArgumentNullException"></exception>
        public static async void UpdateUmbracoRteStyleFormats(
            this IConfiguration config,
            string pathToUmbracoRtfStyleFormatsJsonFile
            )
        {
            const string UmbracoRteStyleFormatsKey =
                "Umbraco:CMS:RichTextEditor:CustomConfig:style_formats";

            if (config is null)
            {
                throw new ArgumentNullException(nameof(config));
            }

            if (pathToUmbracoRtfStyleFormatsJsonFile is null)
            {
                throw new ArgumentNullException(nameof(pathToUmbracoRtfStyleFormatsJsonFile));
            }

            // get the rte style config
            using var reader = new StreamReader(pathToUmbracoRtfStyleFormatsJsonFile);
            var rteStyleFormatsJson = await JsonDocument.ParseAsync(reader.BaseStream);

            // serialize into string
            var rteStyleFormatsAsString = JsonSerializer.Serialize(rteStyleFormatsJson);

            // update config
            config[UmbracoRteStyleFormatsKey] = rteStyleFormatsAsString;
        }
    }
}

Example rte style formats json file "rteStyleFormats.json" in application root:

[
  {
    "title": "Headings",
    "items": [
      {
        "title": "Heading 1 (h1)",
        "block": "h1"
      },
      {
        "title": "Heading 2 (h2)",
        "block": "h2"
      },
      {
        "title": "Heading 3 (h3)",
        "block": "h3"
      }
    ]
  },
  {
    "title": "Paragraphs",
    "items": [
      {
        "title": "Introduction (large)",
        "classes": "intro lg",
        "block": "p"
      },
      {
        "title": "Introduction (medium)",
        "classes": "intro",
        "block": "p"
      },
      {
        "title": "Introduction (small)",
        "classes": "intro sm",
        "block": "p"
      }
    ]
  },
  {
    "title": "Quotes",
    "items": [
      {
        "title": "Quote (large)",
        "classes": "quote lg",
        "block": "p"
      },
      {
        "title": "Quote (medium)",
        "classes": "quote",
        "block": "p"
      },
      {
        "title": "Quote (Small)",
        "classes": "quote sm",
        "block": "p"
      }
    ]
  }
]

Calling the extension method from startup to process file "rteStyleFormats.json" in application root:

public class Startup
{
    private readonly IWebHostEnvironment _env;
    private readonly IConfiguration _config;

    /// <summary>
    /// Initializes a new instance of the <see cref="Startup" /> class.
    /// </summary>
    /// <param name="webHostEnvironment">The web hosting environment.</param>
    /// <param name="config">The configuration.</param>
    /// <remarks>
    /// Only a few services are possible to be injected here https://github.com/dotnet/aspnetcore/issues/9337.
    /// </remarks>
    public Startup(IWebHostEnvironment webHostEnvironment, IConfiguration config)
    {
        _env = webHostEnvironment ?? throw new ArgumentNullException(nameof(webHostEnvironment));
        _config = config ?? throw new ArgumentNullException(nameof(config));

        _config.UpdateUmbracoRteStyleFormats("rteStyleFormats.json");
    }
    ....
}

is working on a reply...