I believe I figured out a workaround. I created a custom property value converter that does essentially nothing but is for the color picker.

using Umbraco.Core.Models.PublishedContent;
using Umbraco.Core.PropertyEditors;

namespace MySite.Web
{
    [PropertyValueType(typeof(string))]
    public class ColorPickerPropertyValueConverter : IPropertyValueConverter
    {
        public bool IsConverter(PublishedPropertyType propertyType)
        {
            return propertyType.PropertyEditorAlias == "Umbraco.ColorPickerAlias";
        }

        public object ConvertDataToSource(PublishedPropertyType propertyType, object source, bool preview)
        {
            return (source ?? string.Empty).ToString();
        }

        public object ConvertSourceToObject(PublishedPropertyType propertyType, object source, bool preview)
        {
            return (source ?? string.Empty).ToString();
        }

        public object ConvertSourceToXPath(PublishedPropertyType propertyType, object source, bool preview)
        {
            return (source ?? string.Empty).ToString();
        }
    }
}

For now, this works, but I'm open to better solutions.