xmltooling/soap/impl/CURLSOAPTransport.cpp

   1 /*
   2  *  Copyright 2001-2007 Internet2
   3  *
   4  * Licensed under the Apache License, Version 2.0 (the "License");
   5  * you may not use this file except in compliance with the License.
   6  * You may obtain a copy of the License at
   7  *
   8  *     http://www.apache.org/licenses/LICENSE-2.0
   9  *
  10  * Unless required by applicable law or agreed to in writing, software
  11  * distributed under the License is distributed on an "AS IS" BASIS,
  12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13  * See the License for the specific language governing permissions and
  14  * limitations under the License.
  15  */
  16
  17 /**
  18  * CURLSOAPTransport.cpp
  19  *
  20  * libcurl-based SOAPTransport implementation
  21  */
  22
  23 #include "internal.h"
  24 #include "exceptions.h"
  25 #include "security/OpenSSLTrustEngine.h"
  26 #include "signature/OpenSSLCredentialResolver.h"
  27 #include "soap/HTTPSOAPTransport.h"
  28 #include "soap/OpenSSLSOAPTransport.h"
  29 #include "util/NDC.h"
  30 #include "util/Threads.h"
  31
  32 #include <list>
  33 #include <curl/curl.h>
  34 #include <log4cpp/Category.hh>
  35 #include <openssl/x509_vfy.h>
  36
  37 using namespace xmlsignature;
  38 using namespace xmltooling;
  39 using namespace log4cpp;
  40 using namespace std;
  41
  42 namespace xmltooling {
  43
  44     // Manages cache of socket connections via CURL handles.
  45     class XMLTOOL_DLLLOCAL CURLPool
  46     {
  47     public:
  48         CURLPool() : m_size(256), m_lock(Mutex::create()),
  49             m_log(Category::getInstance(XMLTOOLING_LOGCAT".SOAPTransport.CURLPool")) {}
  50         ~CURLPool();
  51
  52         CURL* get(const string& to, const char* endpoint);
  53         void put(const string& to, const char* endpoint, CURL* handle);
  54
  55     private:
  56         typedef map<string,vector<CURL*> > poolmap_t;
  57         poolmap_t m_bindingMap;
  58         list< vector<CURL*>* > m_pools;
  59         long m_size;
  60         Mutex* m_lock;
  61         Category& m_log;
  62     };
  63
  64     static XMLTOOL_DLLLOCAL CURLPool* g_CURLPool = NULL;
  65
  66     class XMLTOOL_DLLLOCAL CURLSOAPTransport : public HTTPSOAPTransport, public OpenSSLSOAPTransport
  67     {
  68     public:
  69         CURLSOAPTransport(const KeyInfoSource& peer, const char* endpoint)
  70                 : m_peer(peer), m_endpoint(endpoint), m_handle(NULL), m_headers(NULL),
  71 #ifndef XMLTOOLING_NO_XMLSEC
  72                     m_credResolver(NULL), m_trustEngine(NULL), m_mandatory(false), m_keyResolver(NULL),
  73 #endif
  74                     m_ssl_callback(NULL), m_ssl_userptr(NULL), m_secure(false) {
  75             m_handle = g_CURLPool->get(peer.getName(), endpoint);
  76             curl_easy_setopt(m_handle,CURLOPT_URL,endpoint);
  77             curl_easy_setopt(m_handle,CURLOPT_CONNECTTIMEOUT,15);
  78             curl_easy_setopt(m_handle,CURLOPT_TIMEOUT,30);
  79             curl_easy_setopt(m_handle,CURLOPT_HTTPAUTH,0);
  80             curl_easy_setopt(m_handle,CURLOPT_USERPWD,NULL);
  81             curl_easy_setopt(m_handle,CURLOPT_HEADERDATA,this);
  82             m_headers=curl_slist_append(m_headers,"Content-Type: text/xml");
  83             m_headers=curl_slist_append(m_headers,"Transport-Encoding: chunked");
  84         }
  85
  86         virtual ~CURLSOAPTransport() {
  87             curl_slist_free_all(m_headers);
  88             curl_easy_setopt(m_handle,CURLOPT_ERRORBUFFER,NULL);
  89             curl_easy_setopt(m_handle,CURLOPT_PRIVATE,m_secure ? "secure" : NULL); // Save off security "state".
  90             g_CURLPool->put(m_peer.getName(), m_endpoint.c_str(), m_handle);
  91         }
  92
  93         bool setConnectTimeout(long timeout) const {
  94             return (curl_easy_setopt(m_handle,CURLOPT_CONNECTTIMEOUT,timeout)==CURLE_OK);
  95         }
  96
  97         bool setTimeout(long timeout) const {
  98             return (curl_easy_setopt(m_handle,CURLOPT_TIMEOUT,timeout)==CURLE_OK);
  99         }
 100
 101         bool setAuth(transport_auth_t authType, const char* username=NULL, const char* password=NULL) const;
 102
 103 #ifndef XMLTOOLING_NO_XMLSEC
 104         bool setCredentialResolver(const CredentialResolver* credResolver) const {
 105             const OpenSSLCredentialResolver* down = dynamic_cast<const OpenSSLCredentialResolver*>(credResolver);
 106             if (!down) {
 107                 m_credResolver = NULL;
 108                 return (credResolver==NULL);
 109             }
 110             m_credResolver = down;
 111             return true;
 112         }
 113
 114         bool setTrustEngine(const X509TrustEngine* trustEngine, bool mandatory=true, const KeyResolver* keyResolver=NULL) const {
 115             const OpenSSLTrustEngine* down = dynamic_cast<const OpenSSLTrustEngine*>(trustEngine);
 116             if (!down) {
 117                 m_trustEngine = NULL;
 118                 m_keyResolver = NULL;
 119                 return (trustEngine==NULL);
 120             }
 121             m_trustEngine = down;
 122             m_keyResolver = keyResolver;
 123             m_mandatory = mandatory;
 124             return true;
 125         }
 126
 127 #endif
 128
 129         void send(istream& in);
 130
 131         istream& receive() {
 132             return m_stream;
 133         }
 134
 135         bool isSecure() const {
 136             return m_secure;
 137         }
 138
 139         void setSecure(bool secure) {
 140             m_secure = secure;
 141         }
 142
 143         string getContentType() const;
 144
 145         bool setRequestHeader(const char* name, const char* val) const {
 146             string temp(name);
 147             temp=temp + ": " + val;
 148             m_headers=curl_slist_append(m_headers,temp.c_str());
 149             return true;
 150         }
 151
 152         const vector<string>& getResponseHeader(const char* val) const;
 153
 154         bool setSSLCallback(ssl_ctx_callback_fn fn, void* userptr=NULL) const {
 155             m_ssl_callback=fn;
 156             m_ssl_userptr=userptr;
 157             return true;
 158         }
 159
 160     private:
 161         // per-call state
 162         const KeyInfoSource& m_peer;
 163         string m_endpoint;
 164         CURL* m_handle;
 165         stringstream m_stream;
 166         mutable struct curl_slist* m_headers;
 167         map<string,vector<string> > m_response_headers;
 168 #ifndef XMLTOOLING_NO_XMLSEC
 169         mutable const OpenSSLCredentialResolver* m_credResolver;
 170         mutable const OpenSSLTrustEngine* m_trustEngine;
 171         mutable bool m_mandatory;
 172         mutable const KeyResolver* m_keyResolver;
 173 #endif
 174         mutable ssl_ctx_callback_fn m_ssl_callback;
 175         mutable void* m_ssl_userptr;
 176         bool m_secure;
 177
 178         friend size_t XMLTOOL_DLLLOCAL curl_header_hook(void* ptr, size_t size, size_t nmemb, void* stream);
 179         friend CURLcode XMLTOOL_DLLLOCAL xml_ssl_ctx_callback(CURL* curl, SSL_CTX* ssl_ctx, void* userptr);
 180         friend int XMLTOOL_DLLLOCAL verify_callback(X509_STORE_CTX* x509_ctx, void* arg);
 181     };
 182
 183     // libcurl callback functions
 184     size_t XMLTOOL_DLLLOCAL curl_header_hook(void* ptr, size_t size, size_t nmemb, void* stream);
 185     size_t XMLTOOL_DLLLOCAL curl_write_hook(void* ptr, size_t size, size_t nmemb, void* stream);
 186     size_t XMLTOOL_DLLLOCAL curl_read_hook( void *ptr, size_t size, size_t nmemb, void *stream);
 187     int XMLTOOL_DLLLOCAL curl_debug_hook(CURL* handle, curl_infotype type, char* data, size_t len, void* ptr);
 188     CURLcode XMLTOOL_DLLLOCAL xml_ssl_ctx_callback(CURL* curl, SSL_CTX* ssl_ctx, void* userptr);
 189 #ifndef XMLTOOLING_NO_XMLSEC
 190     int XMLTOOL_DLLLOCAL verify_callback(X509_STORE_CTX* x509_ctx, void* arg);
 191 #endif
 192
 193     SOAPTransport* CURLSOAPTransportFactory(const pair<const KeyInfoSource*,const char*>& dest)
 194     {
 195         return new CURLSOAPTransport(*dest.first, dest.second);
 196     }
 197 };
 198
 199 void xmltooling::registerSOAPTransports()
 200 {
 201     XMLToolingConfig& conf=XMLToolingConfig::getConfig();
 202     conf.SOAPTransportManager.registerFactory("http", CURLSOAPTransportFactory);
 203     conf.SOAPTransportManager.registerFactory("https", CURLSOAPTransportFactory);
 204 }
 205
 206 void xmltooling::initSOAPTransports()
 207 {
 208     g_CURLPool=new CURLPool();
 209 }
 210
 211 void xmltooling::termSOAPTransports()
 212 {
 213     delete g_CURLPool;
 214     g_CURLPool = NULL;
 215 }
 216
 217 CURLPool::~CURLPool()
 218 {
 219     for (poolmap_t::iterator i=m_bindingMap.begin(); i!=m_bindingMap.end(); i++) {
 220         for (vector<CURL*>::iterator j=i->second.begin(); j!=i->second.end(); j++)
 221             curl_easy_cleanup(*j);
 222     }
 223     delete m_lock;
 224 }
 225
 226 CURL* CURLPool::get(const string& to, const char* endpoint)
 227 {
 228 #ifdef _DEBUG
 229     xmltooling::NDC("get");
 230 #endif
 231     m_log.debug("getting connection handle to %s", endpoint);
 232     m_lock->lock();
 233     poolmap_t::iterator i=m_bindingMap.find(to + "|" + endpoint);
 234
 235     if (i!=m_bindingMap.end()) {
 236         // Move this pool to the front of the list.
 237         m_pools.remove(&(i->second));
 238         m_pools.push_front(&(i->second));
 239
 240         // If a free connection exists, return it.
 241         if (!(i->second.empty())) {
 242             CURL* handle=i->second.back();
 243             i->second.pop_back();
 244             m_size--;
 245             m_lock->unlock();
 246             m_log.debug("returning existing connection handle from pool");
 247             return handle;
 248         }
 249     }
 250
 251     m_lock->unlock();
 252     m_log.debug("nothing free in pool, returning new connection handle");
 253
 254     // Create a new connection and set non-varying options.
 255     CURL* handle=curl_easy_init();
 256     if (!handle)
 257         return NULL;
 258     curl_easy_setopt(handle,CURLOPT_NOPROGRESS,1);
 259     curl_easy_setopt(handle,CURLOPT_NOSIGNAL,1);
 260     curl_easy_setopt(handle,CURLOPT_FAILONERROR,1);
 261     curl_easy_setopt(handle,CURLOPT_SSLVERSION,3);
 262     curl_easy_setopt(handle,CURLOPT_SSL_VERIFYHOST,2);
 263     curl_easy_setopt(handle,CURLOPT_HEADERFUNCTION,&curl_header_hook);
 264     curl_easy_setopt(handle,CURLOPT_READFUNCTION,&curl_read_hook);
 265     curl_easy_setopt(handle,CURLOPT_WRITEFUNCTION,&curl_write_hook);
 266     curl_easy_setopt(handle,CURLOPT_DEBUGFUNCTION,&curl_debug_hook);
 267
 268     return handle;
 269 }
 270
 271 void CURLPool::put(const string& to, const char* endpoint, CURL* handle)
 272 {
 273     string key = to + "|" + endpoint;
 274     m_lock->lock();
 275     poolmap_t::iterator i=m_bindingMap.find(key);
 276     if (i==m_bindingMap.end())
 277         m_pools.push_front(&(m_bindingMap.insert(poolmap_t::value_type(key,vector<CURL*>(1,handle))).first->second));
 278     else
 279         i->second.push_back(handle);
 280
 281     CURL* killit=NULL;
 282     if (++m_size > 256) {
 283         // Kick a handle out from the back of the bus.
 284         while (true) {
 285             vector<CURL*>* corpse=m_pools.back();
 286             if (!corpse->empty()) {
 287                 killit=corpse->back();
 288                 corpse->pop_back();
 289                 m_size--;
 290                 break;
 291             }
 292
 293             // Move an empty pool up to the front so we don't keep hitting it.
 294             m_pools.pop_back();
 295             m_pools.push_front(corpse);
 296         }
 297     }
 298     m_lock->unlock();
 299     if (killit) {
 300         curl_easy_cleanup(killit);
 301 #ifdef _DEBUG
 302         xmltooling::NDC("put");
 303 #endif
 304         m_log.info("conn_pool_max limit reached, dropping an old connection");
 305     }
 306 }
 307
 308 bool CURLSOAPTransport::setAuth(transport_auth_t authType, const char* username, const char* password) const
 309 {
 310     if (authType==transport_auth_none) {
 311         if (curl_easy_setopt(m_handle,CURLOPT_HTTPAUTH,0)!=CURLE_OK)
 312             return false;
 313         return (curl_easy_setopt(m_handle,CURLOPT_USERPWD,NULL)==CURLE_OK);
 314     }
 315     long flag=0;
 316     switch (authType) {
 317         case transport_auth_basic:    flag = CURLAUTH_BASIC; break;
 318         case transport_auth_digest:   flag = CURLAUTH_DIGEST; break;
 319         case transport_auth_ntlm:     flag = CURLAUTH_NTLM; break;
 320         case transport_auth_gss:      flag = CURLAUTH_GSSNEGOTIATE; break;
 321         default:            return false;
 322     }
 323     if (curl_easy_setopt(m_handle,CURLOPT_HTTPAUTH,flag)!=CURLE_OK)
 324         return false;
 325     string creds = string(username ? username : "") + ':' + (password ? password : "");
 326     return (curl_easy_setopt(m_handle,CURLOPT_USERPWD,creds.c_str())==CURLE_OK);
 327 }
 328
 329 const vector<string>& CURLSOAPTransport::getResponseHeader(const char* name) const
 330 {
 331     static vector<string> emptyVector;
 332
 333     map<string,vector<string> >::const_iterator i=m_response_headers.find(name);
 334     if (i!=m_response_headers.end())
 335         return i->second;
 336
 337     for (map<string,vector<string> >::const_iterator j=m_response_headers.begin(); j!=m_response_headers.end(); j++) {
 338 #ifdef HAVE_STRCASECMP
 339         if (!strcasecmp(j->first.c_str(), name))
 340 #else
 341         if (!stricmp(j->first.c_str(), name))
 342 #endif
 343             return j->second;
 344     }
 345
 346     return emptyVector;
 347 }
 348
 349 string CURLSOAPTransport::getContentType() const
 350 {
 351     char* content_type=NULL;
 352     curl_easy_getinfo(m_handle,CURLINFO_CONTENT_TYPE,&content_type);
 353     return content_type ? content_type : "";
 354 }
 355
 356 void CURLSOAPTransport::send(istream& in)
 357 {
 358 #ifdef _DEBUG
 359     xmltooling::NDC ndc("send");
 360 #endif
 361     Category& log=Category::getInstance(XMLTOOLING_LOGCAT".SOAPTransport");
 362     Category& log_curl=Category::getInstance(XMLTOOLING_LOGCAT".libcurl");
 363
 364     // By this time, the handle has been prepared with the URL to use and the
 365     // caller should have executed any set functions to manipulate it.
 366
 367     // Setup standard per-call curl properties.
 368     curl_easy_setopt(m_handle,CURLOPT_POST,1);
 369     curl_easy_setopt(m_handle,CURLOPT_READDATA,&in);
 370     curl_easy_setopt(m_handle,CURLOPT_FILE,&m_stream);
 371     curl_easy_setopt(m_handle,CURLOPT_DEBUGDATA,&log_curl);
 372
 373     char curl_errorbuf[CURL_ERROR_SIZE];
 374     curl_errorbuf[0]=0;
 375     curl_easy_setopt(m_handle,CURLOPT_ERRORBUFFER,curl_errorbuf);
 376     if (log_curl.isDebugEnabled())
 377         curl_easy_setopt(m_handle,CURLOPT_VERBOSE,1);
 378
 379     // Set request headers (possibly appended by hooks).
 380     curl_easy_setopt(m_handle,CURLOPT_HTTPHEADER,m_headers);
 381
 382     if (m_ssl_callback || m_credResolver || m_trustEngine) {
 383         curl_easy_setopt(m_handle,CURLOPT_SSL_CTX_FUNCTION,xml_ssl_ctx_callback);
 384         curl_easy_setopt(m_handle,CURLOPT_SSL_CTX_DATA,this);
 385
 386         // Restore security "state". Necessary because the callback only runs
 387         // when handshakes occur. Even new TCP connections won't execute it.
 388         char* priv=NULL;
 389         curl_easy_getinfo(m_handle,CURLINFO_PRIVATE,&priv);
 390         if (priv)
 391             m_secure=true;
 392     }
 393     else {
 394         curl_easy_setopt(m_handle,CURLOPT_SSL_CTX_FUNCTION,NULL);
 395         curl_easy_setopt(m_handle,CURLOPT_SSL_CTX_DATA,NULL);
 396     }
 397
 398     // Verification of the peer is via TrustEngine only.
 399     curl_easy_setopt(m_handle,CURLOPT_SSL_VERIFYPEER,0);
 400
 401     // Make the call.
 402     log.info("sending SOAP message to %s", m_endpoint.c_str());
 403     if (curl_easy_perform(m_handle) != CURLE_OK) {
 404         log.error("failed communicating with SOAP endpoint: %s",
 405             (curl_errorbuf[0] ? curl_errorbuf : "no further information available"));
 406         throw IOException(
 407             string("CURLSOAPTransport::send() failed while contacting SOAP responder: ") +
 408                 (curl_errorbuf[0] ? curl_errorbuf : "no further information available"));
 409     }
 410 }
 411
 412 // callback to buffer headers from server
 413 size_t xmltooling::curl_header_hook(void* ptr, size_t size, size_t nmemb, void* stream)
 414 {
 415     // only handle single-byte data
 416     if (size!=1)
 417         return 0;
 418     CURLSOAPTransport* ctx = reinterpret_cast<CURLSOAPTransport*>(stream);
 419     char* buf = (char*)malloc(nmemb + 1);
 420     if (buf) {
 421         memset(buf,0,nmemb + 1);
 422         memcpy(buf,ptr,nmemb);
 423         char* sep=(char*)strchr(buf,':');
 424         if (sep) {
 425             *(sep++)=0;
 426             while (*sep==' ')
 427                 *(sep++)=0;
 428             char* white=buf+nmemb-1;
 429             while (isspace(*white))
 430                 *(white--)=0;
 431             ctx->m_response_headers[buf].push_back(sep);
 432         }
 433         free(buf);
 434         return nmemb;
 435     }
 436     return 0;
 437 }
 438
 439 // callback to send data to server
 440 size_t xmltooling::curl_read_hook(void* ptr, size_t size, size_t nmemb, void* stream)
 441 {
 442     // *stream is actually an istream object
 443     istream& buf=*(reinterpret_cast<istream*>(stream));
 444     buf.read(reinterpret_cast<char*>(ptr),size*nmemb);
 445     return buf.gcount();
 446 }
 447
 448 // callback to buffer data from server
 449 size_t xmltooling::curl_write_hook(void* ptr, size_t size, size_t nmemb, void* stream)
 450 {
 451     size_t len = size*nmemb;
 452     reinterpret_cast<stringstream*>(stream)->write(reinterpret_cast<const char*>(ptr),len);
 453     return len;
 454 }
 455
 456 // callback for curl debug data
 457 int xmltooling::curl_debug_hook(CURL* handle, curl_infotype type, char* data, size_t len, void* ptr)
 458 {
 459     // *ptr is actually a logging object
 460     if (!ptr) return 0;
 461     CategoryStream log=reinterpret_cast<Category*>(ptr)->debugStream();
 462     for (char* ch=data; len && (isprint(*ch) || isspace(*ch)); len--)
 463         log << *ch++;
 464     log << CategoryStream::ENDLINE;
 465     return 0;
 466 }
 467
 468 #ifndef XMLTOOLING_NO_XMLSEC
 469 int xmltooling::verify_callback(X509_STORE_CTX* x509_ctx, void* arg)
 470 {
 471     Category::getInstance("OpenSSL").debug("invoking X509 verify callback");
 472 #if (OPENSSL_VERSION_NUMBER >= 0x00907000L)
 473     CURLSOAPTransport* ctx = reinterpret_cast<CURLSOAPTransport*>(arg);
 474 #else
 475     // Yes, this sucks. I'd use TLS, but there's no really obvious spot to put the thread key
 476     // and global variables suck too. We can't access the X509_STORE_CTX depth directly because
 477     // OpenSSL only copies it into the context if it's >=0, and the unsigned pointer may be
 478     // negative in the SSL structure's int member.
 479     CURLSOAPTransport* ctx = reinterpret_cast<CURLSOAPTransport*>(
 480         SSL_get_verify_depth(
 481             reinterpret_cast<SSL*>(X509_STORE_CTX_get_ex_data(x509_ctx,SSL_get_ex_data_X509_STORE_CTX_idx()))
 482             )
 483         );
 484 #endif
 485
 486      // Bypass name check (handled for us by curl).
 487     if (!ctx->m_trustEngine->validate(x509_ctx->cert,x509_ctx->untrusted,ctx->m_peer,false,ctx->m_keyResolver)) {
 488         x509_ctx->error=X509_V_ERR_APPLICATION_VERIFICATION;     // generic error, check log for plugin specifics
 489         ctx->setSecure(false);
 490         return ctx->m_mandatory ? 0 : 1;
 491     }
 492
 493     // Signal success. Hopefully it doesn't matter what's actually in the structure now.
 494     ctx->setSecure(true);
 495     return 1;
 496 }
 497 #endif
 498
 499 // callback to invoke a caller-defined SSL callback
 500 CURLcode xmltooling::xml_ssl_ctx_callback(CURL* curl, SSL_CTX* ssl_ctx, void* userptr)
 501 {
 502     CURLSOAPTransport* conf = reinterpret_cast<CURLSOAPTransport*>(userptr);
 503
 504 #ifndef XMLTOOLING_NO_XMLSEC
 505     if (conf->m_credResolver)
 506         conf->m_credResolver->attach(ssl_ctx);
 507
 508     if (conf->m_trustEngine) {
 509         SSL_CTX_set_verify(ssl_ctx,SSL_VERIFY_PEER,NULL);
 510 #if (OPENSSL_VERSION_NUMBER >= 0x00907000L)
 511         // With 0.9.7, we can pass a callback argument directly.
 512         SSL_CTX_set_cert_verify_callback(ssl_ctx,verify_callback,userptr);
 513 #else
 514         // With 0.9.6, there's no argument, so we're going to use a really embarrassing hack and
 515         // stuff the argument in the depth property where it will get copied to the context object
 516         // that's handed to the callback.
 517         SSL_CTX_set_cert_verify_callback(ssl_ctx,reinterpret_cast<int (*)()>(verify_callback),NULL);
 518         SSL_CTX_set_verify_depth(ssl_ctx,reinterpret_cast<int>(userptr));
 519 #endif
 520     }
 521 #endif
 522
 523     if (!conf->m_ssl_callback(conf, ssl_ctx, conf->m_ssl_userptr))
 524         return CURLE_SSL_CERTPROBLEM;
 525
 526     return CURLE_OK;
 527 }