xmltooling/soap/impl/CURLSOAPTransport.cpp

   1 /*
   2  *  Copyright 2001-2007 Internet2
   3  *
   4  * Licensed under the Apache License, Version 2.0 (the "License");
   5  * you may not use this file except in compliance with the License.
   6  * You may obtain a copy of the License at
   7  *
   8  *     http://www.apache.org/licenses/LICENSE-2.0
   9  *
  10  * Unless required by applicable law or agreed to in writing, software
  11  * distributed under the License is distributed on an "AS IS" BASIS,
  12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13  * See the License for the specific language governing permissions and
  14  * limitations under the License.
  15  */
  16
  17 /**
  18  * CURLSOAPTransport.cpp
  19  *
  20  * libcurl-based SOAPTransport implementation
  21  */
  22
  23 #include "internal.h"
  24 #include "exceptions.h"
  25 #include "security/OpenSSLTrustEngine.h"
  26 #include "security/OpenSSLCredentialResolver.h"
  27 #include "soap/HTTPSOAPTransport.h"
  28 #include "soap/OpenSSLSOAPTransport.h"
  29 #include "util/NDC.h"
  30 #include "util/Threads.h"
  31
  32 #include <list>
  33 #include <curl/curl.h>
  34 #include <log4cpp/Category.hh>
  35 #include <openssl/x509_vfy.h>
  36
  37 using namespace xmltooling;
  38 using namespace log4cpp;
  39 using namespace std;
  40
  41 namespace xmltooling {
  42
  43     // Manages cache of socket connections via CURL handles.
  44     class XMLTOOL_DLLLOCAL CURLPool
  45     {
  46     public:
  47         CURLPool() : m_size(0), m_lock(Mutex::create()),
  48             m_log(Category::getInstance(XMLTOOLING_LOGCAT".SOAPTransport.CURLPool")) {}
  49         ~CURLPool();
  50
  51         CURL* get(const string& to, const char* endpoint);
  52         void put(const string& to, const char* endpoint, CURL* handle);
  53
  54     private:
  55         typedef map<string,vector<CURL*> > poolmap_t;
  56         poolmap_t m_bindingMap;
  57         list< vector<CURL*>* > m_pools;
  58         long m_size;
  59         Mutex* m_lock;
  60         Category& m_log;
  61     };
  62
  63     static XMLTOOL_DLLLOCAL CURLPool* g_CURLPool = NULL;
  64
  65     class XMLTOOL_DLLLOCAL CURLSOAPTransport : public HTTPSOAPTransport, public OpenSSLSOAPTransport
  66     {
  67     public:
  68         CURLSOAPTransport(const KeyInfoSource& peer, const char* endpoint)
  69                 : m_peer(peer), m_endpoint(endpoint), m_handle(NULL), m_headers(NULL),
  70 #ifndef XMLTOOLING_NO_XMLSEC
  71                     m_credResolver(NULL), m_trustEngine(NULL), m_mandatory(false), m_keyResolver(NULL),
  72 #endif
  73                     m_ssl_callback(NULL), m_ssl_userptr(NULL), m_chunked(true), m_secure(false) {
  74             m_handle = g_CURLPool->get(peer.getName(), endpoint);
  75             curl_easy_setopt(m_handle,CURLOPT_URL,endpoint);
  76             curl_easy_setopt(m_handle,CURLOPT_CONNECTTIMEOUT,15);
  77             curl_easy_setopt(m_handle,CURLOPT_TIMEOUT,30);
  78             curl_easy_setopt(m_handle,CURLOPT_HTTPAUTH,0);
  79             curl_easy_setopt(m_handle,CURLOPT_USERPWD,NULL);
  80             curl_easy_setopt(m_handle,CURLOPT_HEADERDATA,this);
  81             m_headers=curl_slist_append(m_headers,"Content-Type: text/xml");
  82         }
  83
  84         virtual ~CURLSOAPTransport() {
  85             curl_slist_free_all(m_headers);
  86             curl_easy_setopt(m_handle,CURLOPT_ERRORBUFFER,NULL);
  87             curl_easy_setopt(m_handle,CURLOPT_PRIVATE,m_secure ? "secure" : NULL); // Save off security "state".
  88             g_CURLPool->put(m_peer.getName(), m_endpoint.c_str(), m_handle);
  89         }
  90
  91         bool isConfidential() const {
  92             return m_endpoint.find("https")==0;
  93         }
  94
  95         bool setConnectTimeout(long timeout) {
  96             return (curl_easy_setopt(m_handle,CURLOPT_CONNECTTIMEOUT,timeout)==CURLE_OK);
  97         }
  98
  99         bool setTimeout(long timeout) {
 100             return (curl_easy_setopt(m_handle,CURLOPT_TIMEOUT,timeout)==CURLE_OK);
 101         }
 102
 103         bool setAuth(transport_auth_t authType, const char* username=NULL, const char* password=NULL);
 104
 105 #ifndef XMLTOOLING_NO_XMLSEC
 106         bool setCredentialResolver(const CredentialResolver* credResolver) {
 107             const OpenSSLCredentialResolver* down = dynamic_cast<const OpenSSLCredentialResolver*>(credResolver);
 108             if (!down) {
 109                 m_credResolver = NULL;
 110                 return (credResolver==NULL);
 111             }
 112             m_credResolver = down;
 113             return true;
 114         }
 115
 116         bool setTrustEngine(const X509TrustEngine* trustEngine, bool mandatory=true, const KeyResolver* keyResolver=NULL) {
 117             const OpenSSLTrustEngine* down = dynamic_cast<const OpenSSLTrustEngine*>(trustEngine);
 118             if (!down) {
 119                 m_trustEngine = NULL;
 120                 m_keyResolver = NULL;
 121                 return (trustEngine==NULL);
 122             }
 123             m_trustEngine = down;
 124             m_keyResolver = keyResolver;
 125             m_mandatory = mandatory;
 126             return true;
 127         }
 128
 129 #endif
 130
 131         bool useChunkedEncoding(bool chunked=true) {
 132             m_chunked = chunked;
 133             return true;
 134         }
 135
 136         void send(istream& in);
 137
 138         istream& receive() {
 139             return m_stream;
 140         }
 141
 142         bool isSecure() const {
 143             return m_secure;
 144         }
 145
 146         void setSecure(bool secure) {
 147             m_secure = secure;
 148         }
 149
 150         string getContentType() const;
 151
 152         bool setRequestHeader(const char* name, const char* val) {
 153             string temp(name);
 154             temp=temp + ": " + val;
 155             m_headers=curl_slist_append(m_headers,temp.c_str());
 156             return true;
 157         }
 158
 159         const vector<string>& getResponseHeader(const char* val) const;
 160
 161         bool setSSLCallback(ssl_ctx_callback_fn fn, void* userptr=NULL) {
 162             m_ssl_callback=fn;
 163             m_ssl_userptr=userptr;
 164             return true;
 165         }
 166
 167     private:
 168         // per-call state
 169         const KeyInfoSource& m_peer;
 170         string m_endpoint;
 171         CURL* m_handle;
 172         stringstream m_stream;
 173         struct curl_slist* m_headers;
 174         map<string,vector<string> > m_response_headers;
 175 #ifndef XMLTOOLING_NO_XMLSEC
 176         const OpenSSLCredentialResolver* m_credResolver;
 177         const OpenSSLTrustEngine* m_trustEngine;
 178         bool m_mandatory;
 179         const KeyResolver* m_keyResolver;
 180 #endif
 181         ssl_ctx_callback_fn m_ssl_callback;
 182         void* m_ssl_userptr;
 183         bool m_chunked;
 184         bool m_secure;
 185
 186         friend size_t XMLTOOL_DLLLOCAL curl_header_hook(void* ptr, size_t size, size_t nmemb, void* stream);
 187         friend CURLcode XMLTOOL_DLLLOCAL xml_ssl_ctx_callback(CURL* curl, SSL_CTX* ssl_ctx, void* userptr);
 188         friend int XMLTOOL_DLLLOCAL verify_callback(X509_STORE_CTX* x509_ctx, void* arg);
 189     };
 190
 191     // libcurl callback functions
 192     size_t XMLTOOL_DLLLOCAL curl_header_hook(void* ptr, size_t size, size_t nmemb, void* stream);
 193     size_t XMLTOOL_DLLLOCAL curl_write_hook(void* ptr, size_t size, size_t nmemb, void* stream);
 194     size_t XMLTOOL_DLLLOCAL curl_read_hook( void *ptr, size_t size, size_t nmemb, void *stream);
 195     int XMLTOOL_DLLLOCAL curl_debug_hook(CURL* handle, curl_infotype type, char* data, size_t len, void* ptr);
 196     CURLcode XMLTOOL_DLLLOCAL xml_ssl_ctx_callback(CURL* curl, SSL_CTX* ssl_ctx, void* userptr);
 197 #ifndef XMLTOOLING_NO_XMLSEC
 198     int XMLTOOL_DLLLOCAL verify_callback(X509_STORE_CTX* x509_ctx, void* arg);
 199 #endif
 200
 201     SOAPTransport* CURLSOAPTransportFactory(const pair<const KeyInfoSource*,const char*>& dest)
 202     {
 203         return new CURLSOAPTransport(*dest.first, dest.second);
 204     }
 205 };
 206
 207 void xmltooling::registerSOAPTransports()
 208 {
 209     XMLToolingConfig& conf=XMLToolingConfig::getConfig();
 210     conf.SOAPTransportManager.registerFactory("http", CURLSOAPTransportFactory);
 211     conf.SOAPTransportManager.registerFactory("https", CURLSOAPTransportFactory);
 212 }
 213
 214 void xmltooling::initSOAPTransports()
 215 {
 216     g_CURLPool=new CURLPool();
 217 }
 218
 219 void xmltooling::termSOAPTransports()
 220 {
 221     delete g_CURLPool;
 222     g_CURLPool = NULL;
 223 }
 224
 225 CURLPool::~CURLPool()
 226 {
 227     for (poolmap_t::iterator i=m_bindingMap.begin(); i!=m_bindingMap.end(); i++) {
 228         for (vector<CURL*>::iterator j=i->second.begin(); j!=i->second.end(); j++)
 229             curl_easy_cleanup(*j);
 230     }
 231     delete m_lock;
 232 }
 233
 234 CURL* CURLPool::get(const string& to, const char* endpoint)
 235 {
 236 #ifdef _DEBUG
 237     xmltooling::NDC("get");
 238 #endif
 239     m_log.debug("getting connection handle to %s", endpoint);
 240     m_lock->lock();
 241     poolmap_t::iterator i=m_bindingMap.find(to + "|" + endpoint);
 242
 243     if (i!=m_bindingMap.end()) {
 244         // Move this pool to the front of the list.
 245         m_pools.remove(&(i->second));
 246         m_pools.push_front(&(i->second));
 247
 248         // If a free connection exists, return it.
 249         if (!(i->second.empty())) {
 250             CURL* handle=i->second.back();
 251             i->second.pop_back();
 252             m_size--;
 253             m_lock->unlock();
 254             m_log.debug("returning existing connection handle from pool");
 255             return handle;
 256         }
 257     }
 258
 259     m_lock->unlock();
 260     m_log.debug("nothing free in pool, returning new connection handle");
 261
 262     // Create a new connection and set non-varying options.
 263     CURL* handle=curl_easy_init();
 264     if (!handle)
 265         return NULL;
 266     curl_easy_setopt(handle,CURLOPT_NOPROGRESS,1);
 267     curl_easy_setopt(handle,CURLOPT_NOSIGNAL,1);
 268     curl_easy_setopt(handle,CURLOPT_FAILONERROR,1);
 269     curl_easy_setopt(handle,CURLOPT_SSLVERSION,3);
 270     // Verification of the peer is via TrustEngine only.
 271     curl_easy_setopt(handle,CURLOPT_SSL_VERIFYPEER,0);
 272     curl_easy_setopt(handle,CURLOPT_SSL_VERIFYHOST,2);
 273     curl_easy_setopt(handle,CURLOPT_HEADERFUNCTION,&curl_header_hook);
 274     curl_easy_setopt(handle,CURLOPT_WRITEFUNCTION,&curl_write_hook);
 275     curl_easy_setopt(handle,CURLOPT_DEBUGFUNCTION,&curl_debug_hook);
 276
 277     return handle;
 278 }
 279
 280 void CURLPool::put(const string& to, const char* endpoint, CURL* handle)
 281 {
 282     string key = to + "|" + endpoint;
 283     m_lock->lock();
 284     poolmap_t::iterator i=m_bindingMap.find(key);
 285     if (i==m_bindingMap.end())
 286         m_pools.push_front(&(m_bindingMap.insert(poolmap_t::value_type(key,vector<CURL*>(1,handle))).first->second));
 287     else
 288         i->second.push_back(handle);
 289
 290     CURL* killit=NULL;
 291     if (++m_size > 256) {
 292         // Kick a handle out from the back of the bus.
 293         while (true) {
 294             vector<CURL*>* corpse=m_pools.back();
 295             if (!corpse->empty()) {
 296                 killit=corpse->back();
 297                 corpse->pop_back();
 298                 m_size--;
 299                 break;
 300             }
 301
 302             // Move an empty pool up to the front so we don't keep hitting it.
 303             m_pools.pop_back();
 304             m_pools.push_front(corpse);
 305         }
 306     }
 307     m_lock->unlock();
 308     if (killit) {
 309         curl_easy_cleanup(killit);
 310 #ifdef _DEBUG
 311         xmltooling::NDC("put");
 312 #endif
 313         m_log.info("conn_pool_max limit reached, dropping an old connection");
 314     }
 315 }
 316
 317 bool CURLSOAPTransport::setAuth(transport_auth_t authType, const char* username, const char* password)
 318 {
 319     if (authType==transport_auth_none) {
 320         if (curl_easy_setopt(m_handle,CURLOPT_HTTPAUTH,0)!=CURLE_OK)
 321             return false;
 322         return (curl_easy_setopt(m_handle,CURLOPT_USERPWD,NULL)==CURLE_OK);
 323     }
 324     long flag=0;
 325     switch (authType) {
 326         case transport_auth_basic:    flag = CURLAUTH_BASIC; break;
 327         case transport_auth_digest:   flag = CURLAUTH_DIGEST; break;
 328         case transport_auth_ntlm:     flag = CURLAUTH_NTLM; break;
 329         case transport_auth_gss:      flag = CURLAUTH_GSSNEGOTIATE; break;
 330         default:            return false;
 331     }
 332     if (curl_easy_setopt(m_handle,CURLOPT_HTTPAUTH,flag)!=CURLE_OK)
 333         return false;
 334     string creds = string(username ? username : "") + ':' + (password ? password : "");
 335     return (curl_easy_setopt(m_handle,CURLOPT_USERPWD,creds.c_str())==CURLE_OK);
 336 }
 337
 338 const vector<string>& CURLSOAPTransport::getResponseHeader(const char* name) const
 339 {
 340     static vector<string> emptyVector;
 341
 342     map<string,vector<string> >::const_iterator i=m_response_headers.find(name);
 343     if (i!=m_response_headers.end())
 344         return i->second;
 345
 346     for (map<string,vector<string> >::const_iterator j=m_response_headers.begin(); j!=m_response_headers.end(); j++) {
 347 #ifdef HAVE_STRCASECMP
 348         if (!strcasecmp(j->first.c_str(), name))
 349 #else
 350         if (!stricmp(j->first.c_str(), name))
 351 #endif
 352             return j->second;
 353     }
 354
 355     return emptyVector;
 356 }
 357
 358 string CURLSOAPTransport::getContentType() const
 359 {
 360     char* content_type=NULL;
 361     curl_easy_getinfo(m_handle,CURLINFO_CONTENT_TYPE,&content_type);
 362     return content_type ? content_type : "";
 363 }
 364
 365 void CURLSOAPTransport::send(istream& in)
 366 {
 367 #ifdef _DEBUG
 368     xmltooling::NDC ndc("send");
 369 #endif
 370     Category& log=Category::getInstance(XMLTOOLING_LOGCAT".SOAPTransport");
 371     Category& log_curl=Category::getInstance(XMLTOOLING_LOGCAT".libcurl");
 372
 373     string msg;
 374
 375     // By this time, the handle has been prepared with the URL to use and the
 376     // caller should have executed any set functions to manipulate it.
 377
 378     // Setup standard per-call curl properties.
 379     curl_easy_setopt(m_handle,CURLOPT_DEBUGDATA,&log_curl);
 380     curl_easy_setopt(m_handle,CURLOPT_FILE,&m_stream);
 381     curl_easy_setopt(m_handle,CURLOPT_POST,1);
 382     if (m_chunked) {
 383         m_headers=curl_slist_append(m_headers,"Transfer-Encoding: chunked");
 384         curl_easy_setopt(m_handle,CURLOPT_READFUNCTION,&curl_read_hook);
 385         curl_easy_setopt(m_handle,CURLOPT_READDATA,&in);
 386     }
 387     else {
 388         char buf[1024];
 389         while (in) {
 390             in.read(buf,1024);
 391             msg.append(buf,in.gcount());
 392         }
 393         curl_easy_setopt(m_handle,CURLOPT_READFUNCTION,NULL);
 394         curl_easy_setopt(m_handle,CURLOPT_POSTFIELDS,msg.c_str());
 395         curl_easy_setopt(m_handle,CURLOPT_POSTFIELDSIZE,msg.length());
 396     }
 397
 398     char curl_errorbuf[CURL_ERROR_SIZE];
 399     curl_errorbuf[0]=0;
 400     curl_easy_setopt(m_handle,CURLOPT_ERRORBUFFER,curl_errorbuf);
 401     if (log_curl.isDebugEnabled())
 402         curl_easy_setopt(m_handle,CURLOPT_VERBOSE,1);
 403
 404     // Set request headers.
 405     curl_easy_setopt(m_handle,CURLOPT_HTTPHEADER,m_headers);
 406
 407     if (m_ssl_callback || m_credResolver || m_trustEngine) {
 408         curl_easy_setopt(m_handle,CURLOPT_SSL_CTX_FUNCTION,xml_ssl_ctx_callback);
 409         curl_easy_setopt(m_handle,CURLOPT_SSL_CTX_DATA,this);
 410
 411         // Restore security "state". Necessary because the callback only runs
 412         // when handshakes occur. Even new TCP connections won't execute it.
 413         char* priv=NULL;
 414         curl_easy_getinfo(m_handle,CURLINFO_PRIVATE,&priv);
 415         if (priv)
 416             m_secure=true;
 417     }
 418     else {
 419         curl_easy_setopt(m_handle,CURLOPT_SSL_CTX_FUNCTION,NULL);
 420         curl_easy_setopt(m_handle,CURLOPT_SSL_CTX_DATA,NULL);
 421     }
 422
 423     // Make the call.
 424     log.debug("sending SOAP message to %s", m_endpoint.c_str());
 425     if (curl_easy_perform(m_handle) != CURLE_OK) {
 426         throw IOException(
 427             string("CURLSOAPTransport failed while contacting SOAP responder: ") +
 428                 (curl_errorbuf[0] ? curl_errorbuf : "no further information available"));
 429     }
 430 }
 431
 432 // callback to buffer headers from server
 433 size_t xmltooling::curl_header_hook(void* ptr, size_t size, size_t nmemb, void* stream)
 434 {
 435     // only handle single-byte data
 436     if (size!=1)
 437         return 0;
 438     CURLSOAPTransport* ctx = reinterpret_cast<CURLSOAPTransport*>(stream);
 439     char* buf = (char*)malloc(nmemb + 1);
 440     if (buf) {
 441         memset(buf,0,nmemb + 1);
 442         memcpy(buf,ptr,nmemb);
 443         char* sep=(char*)strchr(buf,':');
 444         if (sep) {
 445             *(sep++)=0;
 446             while (*sep==' ')
 447                 *(sep++)=0;
 448             char* white=buf+nmemb-1;
 449             while (isspace(*white))
 450                 *(white--)=0;
 451             ctx->m_response_headers[buf].push_back(sep);
 452         }
 453         free(buf);
 454         return nmemb;
 455     }
 456     return 0;
 457 }
 458
 459 // callback to send data to server
 460 size_t xmltooling::curl_read_hook(void* ptr, size_t size, size_t nmemb, void* stream)
 461 {
 462     // *stream is actually an istream object
 463     istream& buf=*(reinterpret_cast<istream*>(stream));
 464     buf.read(reinterpret_cast<char*>(ptr),size*nmemb);
 465     return buf.gcount();
 466 }
 467
 468 // callback to buffer data from server
 469 size_t xmltooling::curl_write_hook(void* ptr, size_t size, size_t nmemb, void* stream)
 470 {
 471     size_t len = size*nmemb;
 472     reinterpret_cast<stringstream*>(stream)->write(reinterpret_cast<const char*>(ptr),len);
 473     return len;
 474 }
 475
 476 // callback for curl debug data
 477 int xmltooling::curl_debug_hook(CURL* handle, curl_infotype type, char* data, size_t len, void* ptr)
 478 {
 479     // *ptr is actually a logging object
 480     if (!ptr) return 0;
 481     CategoryStream log=reinterpret_cast<Category*>(ptr)->debugStream();
 482     for (char* ch=data; len && (isprint(*ch) || isspace(*ch)); len--)
 483         log << *ch++;
 484     log << CategoryStream::ENDLINE;
 485     return 0;
 486 }
 487
 488 #ifndef XMLTOOLING_NO_XMLSEC
 489 int xmltooling::verify_callback(X509_STORE_CTX* x509_ctx, void* arg)
 490 {
 491     Category& log = Category::getInstance("OpenSSL");
 492     log.debug("invoking X509 verify callback");
 493 #if (OPENSSL_VERSION_NUMBER >= 0x00907000L)
 494     CURLSOAPTransport* ctx = reinterpret_cast<CURLSOAPTransport*>(arg);
 495 #else
 496     // Yes, this sucks. I'd use TLS, but there's no really obvious spot to put the thread key
 497     // and global variables suck too. We can't access the X509_STORE_CTX depth directly because
 498     // OpenSSL only copies it into the context if it's >=0, and the unsigned pointer may be
 499     // negative in the SSL structure's int member.
 500     CURLSOAPTransport* ctx = reinterpret_cast<CURLSOAPTransport*>(
 501         SSL_get_verify_depth(
 502             reinterpret_cast<SSL*>(X509_STORE_CTX_get_ex_data(x509_ctx,SSL_get_ex_data_X509_STORE_CTX_idx()))
 503             )
 504         );
 505 #endif
 506
 507      // Bypass name check (handled for us by curl).
 508     if (!ctx->m_trustEngine->validate(x509_ctx->cert,x509_ctx->untrusted,ctx->m_peer,false,ctx->m_keyResolver)) {
 509         log.error("supplied TrustEngine failed to validate SSL/TLS server certificate");
 510         x509_ctx->error=X509_V_ERR_APPLICATION_VERIFICATION;     // generic error, check log for plugin specifics
 511         ctx->setSecure(false);
 512         return ctx->m_mandatory ? 0 : 1;
 513     }
 514
 515     // Signal success. Hopefully it doesn't matter what's actually in the structure now.
 516     ctx->setSecure(true);
 517     return 1;
 518 }
 519 #endif
 520
 521 // callback to invoke a caller-defined SSL callback
 522 CURLcode xmltooling::xml_ssl_ctx_callback(CURL* curl, SSL_CTX* ssl_ctx, void* userptr)
 523 {
 524     CURLSOAPTransport* conf = reinterpret_cast<CURLSOAPTransport*>(userptr);
 525
 526 #ifndef XMLTOOLING_NO_XMLSEC
 527     if (conf->m_credResolver)
 528         conf->m_credResolver->attach(ssl_ctx);
 529
 530     if (conf->m_trustEngine) {
 531         SSL_CTX_set_verify(ssl_ctx,SSL_VERIFY_PEER,NULL);
 532 #if (OPENSSL_VERSION_NUMBER >= 0x00907000L)
 533         // With 0.9.7, we can pass a callback argument directly.
 534         SSL_CTX_set_cert_verify_callback(ssl_ctx,verify_callback,userptr);
 535 #else
 536         // With 0.9.6, there's no argument, so we're going to use a really embarrassing hack and
 537         // stuff the argument in the depth property where it will get copied to the context object
 538         // that's handed to the callback.
 539         SSL_CTX_set_cert_verify_callback(ssl_ctx,reinterpret_cast<int (*)()>(verify_callback),NULL);
 540         SSL_CTX_set_verify_depth(ssl_ctx,reinterpret_cast<int>(userptr));
 541 #endif
 542     }
 543 #endif
 544
 545     if (conf->m_ssl_callback && !conf->m_ssl_callback(conf, ssl_ctx, conf->m_ssl_userptr))
 546         return CURLE_SSL_CERTPROBLEM;
 547
 548     return CURLE_OK;
 549 }