2 * Copyright 2001-2009 Internet2
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
18 * CURLSOAPTransport.cpp
20 * libcurl-based SOAPTransport implementation
24 #include "exceptions.h"
26 #include "security/CredentialCriteria.h"
27 #include "security/OpenSSLTrustEngine.h"
28 #include "security/OpenSSLCredential.h"
29 #include "soap/HTTPSOAPTransport.h"
30 #include "soap/OpenSSLSOAPTransport.h"
32 #include "util/Threads.h"
35 #include <curl/curl.h>
36 #include <openssl/x509_vfy.h>
38 using namespace xmltooling::logging;
39 using namespace xmltooling;
42 namespace xmltooling {
44 // Manages cache of socket connections via CURL handles.
45 class XMLTOOL_DLLLOCAL CURLPool
48 CURLPool() : m_size(0), m_lock(Mutex::create()),
49 m_log(Category::getInstance(XMLTOOLING_LOGCAT".SOAPTransport.CURL")) {}
52 CURL* get(const SOAPTransport::Address& addr);
53 void put(const char* from, const char* to, const char* endpoint, CURL* handle);
56 typedef map<string,vector<CURL*> > poolmap_t;
57 poolmap_t m_bindingMap;
58 list< vector<CURL*>* > m_pools;
64 static XMLTOOL_DLLLOCAL CURLPool* g_CURLPool = NULL;
66 class XMLTOOL_DLLLOCAL CURLSOAPTransport : public HTTPSOAPTransport, public OpenSSLSOAPTransport
69 CURLSOAPTransport(const Address& addr)
70 : m_sender(addr.m_from ? addr.m_from : ""), m_peerName(addr.m_to ? addr.m_to : ""), m_endpoint(addr.m_endpoint),
71 m_handle(NULL), m_headers(NULL),
72 #ifndef XMLTOOLING_NO_XMLSEC
73 m_cred(NULL), m_trustEngine(NULL), m_peerResolver(NULL), m_mandatory(false),
75 m_ssl_callback(NULL), m_ssl_userptr(NULL), m_chunked(true), m_authenticated(false), m_cacheTag(NULL) {
76 m_handle = g_CURLPool->get(addr);
77 curl_easy_setopt(m_handle,CURLOPT_URL,addr.m_endpoint);
78 curl_easy_setopt(m_handle,CURLOPT_CONNECTTIMEOUT,15);
79 curl_easy_setopt(m_handle,CURLOPT_TIMEOUT,30);
80 curl_easy_setopt(m_handle,CURLOPT_HTTPAUTH,0);
81 curl_easy_setopt(m_handle,CURLOPT_USERPWD,NULL);
82 curl_easy_setopt(m_handle,CURLOPT_SSL_VERIFYHOST,2);
83 curl_easy_setopt(m_handle,CURLOPT_HEADERDATA,this);
84 m_headers=curl_slist_append(m_headers,"Content-Type: text/xml");
87 virtual ~CURLSOAPTransport() {
88 curl_slist_free_all(m_headers);
89 curl_easy_setopt(m_handle,CURLOPT_ERRORBUFFER,NULL);
90 curl_easy_setopt(m_handle,CURLOPT_PRIVATE,m_authenticated ? "secure" : NULL); // Save off security "state".
91 g_CURLPool->put(m_sender.c_str(), m_peerName.c_str(), m_endpoint.c_str(), m_handle);
94 bool isConfidential() const {
95 return m_endpoint.find("https")==0;
98 bool setConnectTimeout(long timeout) {
99 return (curl_easy_setopt(m_handle,CURLOPT_CONNECTTIMEOUT,timeout)==CURLE_OK);
102 bool setTimeout(long timeout) {
103 return (curl_easy_setopt(m_handle,CURLOPT_TIMEOUT,timeout)==CURLE_OK);
106 bool setAuth(transport_auth_t authType, const char* username=NULL, const char* password=NULL);
108 bool setVerifyHost(bool verify) {
109 return (curl_easy_setopt(m_handle,CURLOPT_SSL_VERIFYHOST,verify ? 2 : 0)==CURLE_OK);
112 #ifndef XMLTOOLING_NO_XMLSEC
113 bool setCredential(const Credential* cred=NULL) {
114 const OpenSSLCredential* down = dynamic_cast<const OpenSSLCredential*>(cred);
124 const X509TrustEngine* trustEngine=NULL,
125 const CredentialResolver* peerResolver=NULL,
126 CredentialCriteria* criteria=NULL,
129 const OpenSSLTrustEngine* down = dynamic_cast<const OpenSSLTrustEngine*>(trustEngine);
131 m_trustEngine = NULL;
132 m_peerResolver = NULL;
134 return (trustEngine==NULL);
136 m_trustEngine = down;
137 m_peerResolver = peerResolver;
138 m_criteria = criteria;
139 m_mandatory = mandatory;
145 bool useChunkedEncoding(bool chunked=true) {
150 bool setCacheTag(string* cacheTag) {
151 m_cacheTag = cacheTag;
155 bool setProviderOption(const char* provider, const char* option, const char* value) {
156 if (!provider || strcmp(provider, "CURL"))
158 // For libcurl, the option is an enum and the value type depends on the option.
159 CURLoption opt = static_cast<CURLoption>(strtol(option, NULL, 10));
160 if (opt < CURLOPTTYPE_OBJECTPOINT)
161 return (curl_easy_setopt(m_handle, opt, strtol(value, NULL, 10)) == CURLE_OK);
162 #ifdef CURLOPTTYPE_OFF_T
163 else if (opt < CURLOPTTYPE_OFF_T) {
165 m_saved_options.push_back(value);
166 return (curl_easy_setopt(m_handle, opt, value ? m_saved_options.back().c_str() : NULL) == CURLE_OK);
168 # ifdef HAVE_CURL_OFF_T
169 else if (sizeof(curl_off_t) == sizeof(long))
170 return (curl_easy_setopt(m_handle, opt, strtol(value, NULL, 10)) == CURLE_OK);
172 else if (sizeof(off_t) == sizeof(long))
173 return (curl_easy_setopt(m_handle, opt, strtol(value, NULL, 10)) == CURLE_OK);
179 m_saved_options.push_back(value);
180 return (curl_easy_setopt(m_handle, opt, value ? m_saved_options.back().c_str() : NULL) == CURLE_OK);
185 void send(istream& in) {
189 void send(istream* in=NULL);
195 bool isAuthenticated() const {
196 return m_authenticated;
199 void setAuthenticated(bool auth) {
200 m_authenticated = auth;
203 string getContentType() const;
204 long getStatusCode() const;
206 bool setRequestHeader(const char* name, const char* val) {
208 temp=temp + ": " + val;
209 m_headers=curl_slist_append(m_headers,temp.c_str());
213 const vector<string>& getResponseHeader(const char* val) const;
215 bool setSSLCallback(ssl_ctx_callback_fn fn, void* userptr=NULL) {
217 m_ssl_userptr=userptr;
223 string m_sender,m_peerName,m_endpoint,m_simplecreds;
225 stringstream m_stream;
226 struct curl_slist* m_headers;
227 map<string,vector<string> > m_response_headers;
228 vector<string> m_saved_options;
229 #ifndef XMLTOOLING_NO_XMLSEC
230 const OpenSSLCredential* m_cred;
231 const OpenSSLTrustEngine* m_trustEngine;
232 const CredentialResolver* m_peerResolver;
233 CredentialCriteria* m_criteria;
236 ssl_ctx_callback_fn m_ssl_callback;
239 bool m_authenticated;
242 friend size_t XMLTOOL_DLLLOCAL curl_header_hook(void* ptr, size_t size, size_t nmemb, void* stream);
243 friend CURLcode XMLTOOL_DLLLOCAL xml_ssl_ctx_callback(CURL* curl, SSL_CTX* ssl_ctx, void* userptr);
244 friend int XMLTOOL_DLLLOCAL verify_callback(X509_STORE_CTX* x509_ctx, void* arg);
247 // libcurl callback functions
248 size_t XMLTOOL_DLLLOCAL curl_header_hook(void* ptr, size_t size, size_t nmemb, void* stream);
249 size_t XMLTOOL_DLLLOCAL curl_write_hook(void* ptr, size_t size, size_t nmemb, void* stream);
250 size_t XMLTOOL_DLLLOCAL curl_read_hook( void *ptr, size_t size, size_t nmemb, void *stream);
251 int XMLTOOL_DLLLOCAL curl_debug_hook(CURL* handle, curl_infotype type, char* data, size_t len, void* ptr);
252 CURLcode XMLTOOL_DLLLOCAL xml_ssl_ctx_callback(CURL* curl, SSL_CTX* ssl_ctx, void* userptr);
253 #ifndef XMLTOOLING_NO_XMLSEC
254 int XMLTOOL_DLLLOCAL verify_callback(X509_STORE_CTX* x509_ctx, void* arg);
257 SOAPTransport* CURLSOAPTransportFactory(const SOAPTransport::Address& addr)
259 return new CURLSOAPTransport(addr);
263 void xmltooling::registerSOAPTransports()
265 XMLToolingConfig& conf=XMLToolingConfig::getConfig();
266 conf.SOAPTransportManager.registerFactory("http", CURLSOAPTransportFactory);
267 conf.SOAPTransportManager.registerFactory("https", CURLSOAPTransportFactory);
270 void xmltooling::initSOAPTransports()
272 g_CURLPool=new CURLPool();
275 void xmltooling::termSOAPTransports()
281 OpenSSLSOAPTransport::OpenSSLSOAPTransport()
285 OpenSSLSOAPTransport::~OpenSSLSOAPTransport()
289 CURLPool::~CURLPool()
291 for (poolmap_t::iterator i=m_bindingMap.begin(); i!=m_bindingMap.end(); i++) {
292 for (vector<CURL*>::iterator j=i->second.begin(); j!=i->second.end(); j++)
293 curl_easy_cleanup(*j);
298 CURL* CURLPool::get(const SOAPTransport::Address& addr)
301 xmltooling::NDC("get");
303 m_log.debug("getting connection handle to %s", addr.m_endpoint);
304 string key(addr.m_endpoint);
306 key = key + '|' + addr.m_from;
308 key = key + '|' + addr.m_to;
310 poolmap_t::iterator i=m_bindingMap.find(key);
312 if (i!=m_bindingMap.end()) {
313 // Move this pool to the front of the list.
314 m_pools.remove(&(i->second));
315 m_pools.push_front(&(i->second));
317 // If a free connection exists, return it.
318 if (!(i->second.empty())) {
319 CURL* handle=i->second.back();
320 i->second.pop_back();
323 m_log.debug("returning existing connection handle from pool");
329 m_log.debug("nothing free in pool, returning new connection handle");
331 // Create a new connection and set non-varying options.
332 CURL* handle=curl_easy_init();
335 curl_easy_setopt(handle,CURLOPT_NOPROGRESS,1);
336 curl_easy_setopt(handle,CURLOPT_NOSIGNAL,1);
337 curl_easy_setopt(handle,CURLOPT_FAILONERROR,1);
338 curl_easy_setopt(handle,CURLOPT_SSL_CIPHER_LIST,"ALL:!aNULL:!LOW:!EXPORT:!SSLv2");
339 // Verification of the peer is via TrustEngine only.
340 curl_easy_setopt(handle,CURLOPT_SSL_VERIFYPEER,0);
341 curl_easy_setopt(handle,CURLOPT_CAINFO,NULL);
342 curl_easy_setopt(handle,CURLOPT_HEADERFUNCTION,&curl_header_hook);
343 curl_easy_setopt(handle,CURLOPT_WRITEFUNCTION,&curl_write_hook);
344 curl_easy_setopt(handle,CURLOPT_DEBUGFUNCTION,&curl_debug_hook);
349 void CURLPool::put(const char* from, const char* to, const char* endpoint, CURL* handle)
351 string key(endpoint);
353 key = key + '|' + from;
355 key = key + '|' + to;
357 poolmap_t::iterator i=m_bindingMap.find(key);
358 if (i==m_bindingMap.end())
359 m_pools.push_front(&(m_bindingMap.insert(poolmap_t::value_type(key,vector<CURL*>(1,handle))).first->second));
361 i->second.push_back(handle);
364 if (++m_size > 256) {
365 // Kick a handle out from the back of the bus.
367 vector<CURL*>* corpse=m_pools.back();
368 if (!corpse->empty()) {
369 killit=corpse->back();
375 // Move an empty pool up to the front so we don't keep hitting it.
377 m_pools.push_front(corpse);
382 curl_easy_cleanup(killit);
384 xmltooling::NDC("put");
386 m_log.info("conn_pool_max limit reached, dropping an old connection");
390 bool CURLSOAPTransport::setAuth(transport_auth_t authType, const char* username, const char* password)
392 if (authType==transport_auth_none) {
393 if (curl_easy_setopt(m_handle,CURLOPT_HTTPAUTH,0)!=CURLE_OK)
395 return (curl_easy_setopt(m_handle,CURLOPT_USERPWD,NULL)==CURLE_OK);
399 case transport_auth_basic: flag = CURLAUTH_BASIC; break;
400 case transport_auth_digest: flag = CURLAUTH_DIGEST; break;
401 case transport_auth_ntlm: flag = CURLAUTH_NTLM; break;
402 case transport_auth_gss: flag = CURLAUTH_GSSNEGOTIATE; break;
403 default: return false;
405 if (curl_easy_setopt(m_handle,CURLOPT_HTTPAUTH,flag)!=CURLE_OK)
407 m_simplecreds = string(username ? username : "") + ':' + (password ? password : "");
408 return (curl_easy_setopt(m_handle,CURLOPT_USERPWD,m_simplecreds.c_str())==CURLE_OK);
411 const vector<string>& CURLSOAPTransport::getResponseHeader(const char* name) const
413 static vector<string> emptyVector;
415 map<string,vector<string> >::const_iterator i=m_response_headers.find(name);
416 if (i!=m_response_headers.end())
419 for (map<string,vector<string> >::const_iterator j=m_response_headers.begin(); j!=m_response_headers.end(); j++) {
420 #ifdef HAVE_STRCASECMP
421 if (!strcasecmp(j->first.c_str(), name))
423 if (!stricmp(j->first.c_str(), name))
431 string CURLSOAPTransport::getContentType() const
433 char* content_type=NULL;
434 curl_easy_getinfo(m_handle,CURLINFO_CONTENT_TYPE,&content_type);
435 return content_type ? content_type : "";
438 long CURLSOAPTransport::getStatusCode() const
441 if (curl_easy_getinfo(m_handle, CURLINFO_RESPONSE_CODE, &code) != CURLE_OK)
446 void CURLSOAPTransport::send(istream* in)
449 xmltooling::NDC ndc("send");
451 Category& log=Category::getInstance(XMLTOOLING_LOGCAT".SOAPTransport.CURL");
452 Category& log_curl=Category::getInstance(XMLTOOLING_LOGCAT".libcurl");
454 // For this implementation, it's sufficient to check for https as a sign of transport security.
455 if (m_mandatory && !isConfidential())
456 throw IOException("Blocking unprotected HTTP request, transport authentication by server required.");
460 // By this time, the handle has been prepared with the URL to use and the
461 // caller should have executed any set functions to manipulate it.
463 // Setup standard per-call curl properties.
464 curl_easy_setopt(m_handle,CURLOPT_DEBUGDATA,&log_curl);
465 curl_easy_setopt(m_handle,CURLOPT_FILE,&m_stream);
466 if (m_chunked && in) {
467 curl_easy_setopt(m_handle,CURLOPT_POST,1);
468 m_headers=curl_slist_append(m_headers,"Transfer-Encoding: chunked");
469 curl_easy_setopt(m_handle,CURLOPT_READFUNCTION,&curl_read_hook);
470 curl_easy_setopt(m_handle,CURLOPT_READDATA,in);
476 msg.append(buf,in->gcount());
478 curl_easy_setopt(m_handle,CURLOPT_POST,1);
479 curl_easy_setopt(m_handle,CURLOPT_READFUNCTION,NULL);
480 curl_easy_setopt(m_handle,CURLOPT_POSTFIELDS,msg.c_str());
481 curl_easy_setopt(m_handle,CURLOPT_POSTFIELDSIZE,msg.length());
484 curl_easy_setopt(m_handle,CURLOPT_HTTPGET,1);
487 char curl_errorbuf[CURL_ERROR_SIZE];
489 curl_easy_setopt(m_handle,CURLOPT_ERRORBUFFER,curl_errorbuf);
490 if (log_curl.isDebugEnabled())
491 curl_easy_setopt(m_handle,CURLOPT_VERBOSE,1);
493 // Check for cache tag.
494 if (m_cacheTag && !m_cacheTag->empty()) {
495 string hdr("If-None-Match: ");
497 m_headers = curl_slist_append(m_headers, hdr.c_str());
500 // Set request headers.
501 curl_easy_setopt(m_handle,CURLOPT_HTTPHEADER,m_headers);
503 #ifndef XMLTOOLING_NO_XMLSEC
504 if (m_ssl_callback || m_cred || m_trustEngine) {
506 if (m_ssl_callback) {
508 curl_easy_setopt(m_handle,CURLOPT_SSL_CTX_FUNCTION,xml_ssl_ctx_callback);
509 curl_easy_setopt(m_handle,CURLOPT_SSL_CTX_DATA,this);
511 // Restore security "state". Necessary because the callback only runs
512 // when handshakes occur. Even new TCP connections won't execute it.
514 curl_easy_getinfo(m_handle,CURLINFO_PRIVATE,&priv);
516 m_authenticated=true;
519 curl_easy_setopt(m_handle,CURLOPT_SSL_CTX_FUNCTION,NULL);
520 curl_easy_setopt(m_handle,CURLOPT_SSL_CTX_DATA,NULL);
524 log.debug("sending SOAP message to %s", m_endpoint.c_str());
525 if (curl_easy_perform(m_handle) != CURLE_OK) {
527 string("CURLSOAPTransport failed while contacting SOAP endpoint (") + m_endpoint + "): " +
528 (curl_errorbuf[0] ? curl_errorbuf : "no further information available"));
531 // Check for outgoing cache tag.
533 const vector<string>& tags = getResponseHeader("ETag");
535 *m_cacheTag = tags.front();
539 // callback to buffer headers from server
540 size_t xmltooling::curl_header_hook(void* ptr, size_t size, size_t nmemb, void* stream)
542 // only handle single-byte data
545 CURLSOAPTransport* ctx = reinterpret_cast<CURLSOAPTransport*>(stream);
546 char* buf = (char*)malloc(nmemb + 1);
548 memset(buf,0,nmemb + 1);
549 memcpy(buf,ptr,nmemb);
550 char* sep=(char*)strchr(buf,':');
555 char* white=buf+nmemb-1;
556 while (isspace(*white))
558 ctx->m_response_headers[buf].push_back(sep);
566 // callback to send data to server
567 size_t xmltooling::curl_read_hook(void* ptr, size_t size, size_t nmemb, void* stream)
569 // stream is actually an istream pointer
570 istream* buf=reinterpret_cast<istream*>(stream);
571 buf->read(reinterpret_cast<char*>(ptr),size*nmemb);
572 return buf->gcount();
575 // callback to buffer data from server
576 size_t xmltooling::curl_write_hook(void* ptr, size_t size, size_t nmemb, void* stream)
578 size_t len = size*nmemb;
579 reinterpret_cast<stringstream*>(stream)->write(reinterpret_cast<const char*>(ptr),len);
583 // callback for curl debug data
584 int xmltooling::curl_debug_hook(CURL* handle, curl_infotype type, char* data, size_t len, void* ptr)
586 // *ptr is actually a logging object
588 CategoryStream log=reinterpret_cast<Category*>(ptr)->debugStream();
589 for (unsigned char* ch=(unsigned char*)data; len && (isprint(*ch) || isspace(*ch)); len--)
594 #ifndef XMLTOOLING_NO_XMLSEC
595 int xmltooling::verify_callback(X509_STORE_CTX* x509_ctx, void* arg)
597 Category& log=Category::getInstance(XMLTOOLING_LOGCAT".SOAPTransport.CURL");
598 log.debug("invoking custom X.509 verify callback");
599 #if (OPENSSL_VERSION_NUMBER >= 0x00907000L)
600 CURLSOAPTransport* ctx = reinterpret_cast<CURLSOAPTransport*>(arg);
602 // Yes, this sucks. I'd use TLS, but there's no really obvious spot to put the thread key
603 // and global variables suck too. We can't access the X509_STORE_CTX depth directly because
604 // OpenSSL only copies it into the context if it's >=0, and the unsigned pointer may be
605 // negative in the SSL structure's int member.
606 CURLSOAPTransport* ctx = reinterpret_cast<CURLSOAPTransport*>(
607 SSL_get_verify_depth(
608 reinterpret_cast<SSL*>(X509_STORE_CTX_get_ex_data(x509_ctx,SSL_get_ex_data_X509_STORE_CTX_idx()))
614 if (ctx->m_criteria) {
615 ctx->m_criteria->setUsage(Credential::TLS_CREDENTIAL);
616 // Bypass name check (handled for us by curl).
617 ctx->m_criteria->setPeerName(NULL);
618 success = ctx->m_trustEngine->validate(x509_ctx->cert,x509_ctx->untrusted,*(ctx->m_peerResolver),ctx->m_criteria);
621 // Bypass name check (handled for us by curl).
622 CredentialCriteria cc;
623 cc.setUsage(Credential::TLS_CREDENTIAL);
624 success = ctx->m_trustEngine->validate(x509_ctx->cert,x509_ctx->untrusted,*(ctx->m_peerResolver),&cc);
628 log.error("supplied TrustEngine failed to validate SSL/TLS server certificate");
629 x509_ctx->error=X509_V_ERR_APPLICATION_VERIFICATION; // generic error, check log for plugin specifics
630 ctx->setAuthenticated(false);
631 return ctx->m_mandatory ? 0 : 1;
634 // Signal success. Hopefully it doesn't matter what's actually in the structure now.
635 ctx->setAuthenticated(true);
640 // callback to invoke a caller-defined SSL callback
641 CURLcode xmltooling::xml_ssl_ctx_callback(CURL* curl, SSL_CTX* ssl_ctx, void* userptr)
643 CURLSOAPTransport* conf = reinterpret_cast<CURLSOAPTransport*>(userptr);
645 // Manually disable SSLv2 so we're not dependent on libcurl to do it.
646 // Also disable the ticket option where implemented, since this breaks a variety
647 // of servers. Newer libcurl also does this for us.
648 #ifdef SSL_OP_NO_TICKET
649 SSL_CTX_set_options(ssl_ctx, SSL_OP_ALL|SSL_OP_NO_SSLv2|SSL_OP_NO_TICKET);
651 SSL_CTX_set_options(ssl_ctx, SSL_OP_ALL|SSL_OP_NO_SSLv2);
654 #ifndef XMLTOOLING_NO_XMLSEC
656 conf->m_cred->attach(ssl_ctx);
658 if (conf->m_trustEngine) {
659 SSL_CTX_set_verify(ssl_ctx,SSL_VERIFY_PEER,NULL);
660 #if (OPENSSL_VERSION_NUMBER >= 0x00907000L)
661 // With 0.9.7, we can pass a callback argument directly.
662 SSL_CTX_set_cert_verify_callback(ssl_ctx,verify_callback,userptr);
664 // With 0.9.6, there's no argument, so we're going to use a really embarrassing hack and
665 // stuff the argument in the depth property where it will get copied to the context object
666 // that's handed to the callback.
667 SSL_CTX_set_cert_verify_callback(ssl_ctx,reinterpret_cast<int (*)()>(verify_callback),NULL);
668 SSL_CTX_set_verify_depth(ssl_ctx,reinterpret_cast<int>(userptr));
673 if (conf->m_ssl_callback && !conf->m_ssl_callback(conf, ssl_ctx, conf->m_ssl_userptr))
674 return CURLE_SSL_CERTPROBLEM;