Turns out there's a handy Components.Replace function as part of the Composition class which allows you to swap out components. An example of how to use this would be:

using Umbraco.Core.Composing;
using Umbraco.Core;

namespace MyNamespace
{
    public class TestComposer : IUserComposer
    {
        public void Compose(Composition composition)
        {
            composition.Components().Replace<Umbraco.Core.Compose.AuditEventsComponent, TestComponent>();
        }
    }

    public class TestComponent : IComponent
    {
        public void Initialize()
        {
        }

        //...All the stuff I want to do instead of the AuditEventsComponent!

        public void Terminate()
        {
        }
    }
}

Thanks to Rowan for the help on Slack!