I have found a way to solve this :-) There are also other ways to do this, but this is one solution.

I have created an ApiController in umbraco with this code:

using System;
using System.Security.Principal;
using System.Text;
using System.Threading;
using System.Web.Http;
using System.Web.Http.Controllers;
using System.Web.Http.Filters;
using Umbraco.Web.Mvc;
using Umbraco.Web.WebApi;
using Umbraco.Web;
using Umbraco.Web.Security;

namespace Demo.Controllers
{
    [MyBasicAuthenticationFilter]   
    [PluginController("demo")]
    public class DemoApiController : UmbracoApiController
    {
        [HttpGet]
        public string test()
        {
            return "Hello World!";
        }

    }

    ///
    /// Generic Basic Authentication filter that checks for basic authentication
    /// headers and challenges for authentication if no authentication is provided
    /// Sets the Thread Principle with a GenericAuthenticationPrincipal.
    /// 
    /// You can override the OnAuthorize method for custom auth logic that
    /// might be application specific.    
    ///
    ///Always remember that Basic Authentication passes username and passwords
    /// from client to server in plain text, so make sure SSL is used with basic auth
    /// to encode the Authorization header on all requests (not just the login).
    ///
    [AttributeUsage(AttributeTargets.Class | AttributeTargets.Method, AllowMultiple = false)]
    public class BasicAuthenticationFilter : AuthorizationFilterAttribute
    {
        bool Active = true;

        public BasicAuthenticationFilter()
        { }

        ///
        /// Overriden constructor to allow explicit disabling of this
        /// filter's behavior. Pass false to disable (same as no filter
        /// but declarative)
        ///
        ///
        public BasicAuthenticationFilter(bool active)
        {
            Active = active;
        }


        ///
        /// Override to Web API filter method to handle Basic Auth check
        ///
        ///
        public override void OnAuthorization(HttpActionContext actionContext)
        {
            if (Active)
            {
                var identity = ParseAuthorizationHeader(actionContext);
                if (identity == null)
                {
                    Challenge(actionContext);
                    return;
                }


                if (!OnAuthorizeUser(identity.Name, identity.Password, actionContext))
                {
                    Challenge(actionContext);
                    return;
                }

                var principal = new GenericPrincipal(identity, null);

                Thread.CurrentPrincipal = principal;

                // inside of ASP.NET this is required
                //if (HttpContext.Current != null)
                //    HttpContext.Current.User = principal;

                base.OnAuthorization(actionContext);
            }
        }

        ///
        /// Base implementation for user authentication - you probably will
        /// want to override this method for application specific logic.
        /// 
        /// The base implementation merely checks for username and password
        /// present and set the Thread principal.
        /// 
        /// Override this method if you want to customize Authentication
        /// and store user data as needed in a Thread Principle or other
        /// Request specific storage.
        ///
        ///
        ///
        ///
        ///
        protected virtual bool OnAuthorizeUser(string username, string password, HttpActionContext actionContext)
        {
            if (string.IsNullOrEmpty(username) || string.IsNullOrEmpty(password))
                return false;

            return true;
        }

        ///
        /// Parses the Authorization header and creates user credentials
        ///
        ///
        protected virtual BasicAuthenticationIdentity ParseAuthorizationHeader(HttpActionContext actionContext)
        {
            string authHeader = null;
            var auth = actionContext.Request.Headers.Authorization;
            if (auth != null && auth.Scheme == "Basic")
                authHeader = auth.Parameter;

            if (string.IsNullOrEmpty(authHeader))
                return null;

            authHeader = Encoding.Default.GetString(Convert.FromBase64String(authHeader));

            var tokens = authHeader.Split(':');
            if (tokens.Length < 2)
                return null;

            return new BasicAuthenticationIdentity(tokens[0], tokens[1]);
        }


        ///
        /// Send the Authentication Challenge request
        ///
        ///
        ///
        void Challenge(HttpActionContext actionContext)
        {
            var host = actionContext.Request.RequestUri.DnsSafeHost;
            //actionContext.Response = actionContext.Request.CreateResponse(HttpStatusCode.Unauthorized);
            actionContext.Response = actionContext.Request.CreateUserNoAccessResponse();
            actionContext.Response.Headers.Add("WWW-Authenticate", string.Format("Basic realm=\"{0}\"", host));
        }

    }

    public class BasicAuthenticationIdentity : GenericIdentity
    {
        public BasicAuthenticationIdentity(string name, string password)
            : base(name, "Basic")
        {
            this.Password = password;
        }

        /// 
        /// Basic Auth Password for custom authentication
        /// 
        public string Password { get; set; }
    }

    public class MyBasicAuthenticationFilter : BasicAuthenticationFilter
    {

        public MyBasicAuthenticationFilter()
        { }

        public MyBasicAuthenticationFilter(bool active)
            : base(active)
        { }


        protected override bool OnAuthorizeUser(string username, string password, HttpActionContext actionContext)
        {
            MembershipHelper Members = new MembershipHelper(UmbracoContext.Current);

            if (Members.Login(username, password))
            {
// Here you can also check against a specific member type or member group

                Members.Logout();
                return true;
            }
            return false;
        }
    }
}

If you will call the umbraco api from an external application (e.g. other umbraco application, console application, php application or others) you must response an authentication header to the api url.

This shows how I call the api with authentication header from another umbraco application:

using System;
using System.Net;
using System.Text;
using System.Web.Http;
using Umbraco.Web.Mvc;
using Umbraco.Web.WebApi;

namespace Demo_External.Controllers
{
    [PluginController("demoExternal")]
    public class ExternalApiController : UmbracoApiController
    {
        [Umbraco.Web.WebApi.UmbracoAuthorize]
        [HttpGet]
        public string CallUmbraco()
        {
            WebClient wc = new WebClient();
            wc.Headers.Add("Authorization", "Basic " +
                Convert.ToBase64String(
                Encoding.ASCII.GetBytes("test:12345!")));
            string a = wc.DownloadString("http://localhost:52125/Umbraco/demo/DemoApi/test");

            return "Hello from external application! Here is the result: " + a;           
        }

    }
}

To make it work you must create a member with username "test" and password "12345!" in the umbraco application, which provides the web api.