I've found the cause of the problem. The standard UmbracoRoleProvider wants to retrieve the nodeId of the user that needs to be updated. The provider checks the cmsMember tabel to find this user. Evidently the user is not found as a custom membership provider is used where members are not stored in the cmsMembers table.

I "solved" this by a workaround to sync members of my custom membership provider with the default umbraco membership provider. I choose to create the user in the umbraco membership provider just before the role is assigned. (It might be better to sync all users in application startup.)

This is the code for the workaround:

    public class CustomMembershipProvider : MembershipProvider { 
    private MembersMembershipProvider _umbracoMembershipProvider;

    public CustomMembershipProvider()
    {
        _umbracoMembershipProvider = new MembersMembershipProvider();
    }

    public override void Initialize(string name, NameValueCollection config)
    {
        if (config == null) { throw new ArgumentNullException("config"); }
        _umbracoMembershipProvider.Initialize(name, config);

        if (string.IsNullOrEmpty(name)) { name = "CustomMemberShipProvider"; }

        base.Initialize(name, config);
    }


    public override MembershipUser GetUser(object providerUserKey, bool userIsOnline)
    {
        var user = GetAllCustomMembershipUsers.FirstOrDefault(u => u.ProviderUserKey!=null && u.ProviderUserKey.Equals(providerUserKey));
        if (user == null) return null;

        var umbracoUser = _umbracoMembershipProvider.GetUser(user.UserName, userIsOnline);
        if (umbracoUser != null) return user;

        MembershipCreateStatus createStatus;
        _umbracoMembershipProvider.CreateUser(user.UserName, Guid.NewGuid().ToString(), user.Email, string.Empty, string.Empty, false, Guid.NewGuid(), out createStatus);

        return user;
    }
    private List<MembershipUser> GetAllCustomMembershipUsers() {
        //Get all users;
    }
}