34359b4a15e8a1cd14cb649d74ca85455d9353aa
[shibboleth/cpp-xmltooling.git] / xmltooling / soap / impl / CURLSOAPTransport.cpp
1 /*
2  *  Copyright 2001-2007 Internet2
3  * 
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *     http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16
17 /**
18  * CURLSOAPTransport.cpp
19  * 
20  * libcurl-based SOAPTransport implementation
21  */
22
23 #include "internal.h"
24 #include "exceptions.h"
25 #include "security/OpenSSLTrustEngine.h"
26 #include "signature/OpenSSLCredentialResolver.h"
27 #include "soap/HTTPSOAPTransport.h"
28 #include "soap/OpenSSLSOAPTransport.h"
29 #include "util/NDC.h"
30 #include "util/Threads.h"
31
32 #include <list>
33 #include <curl/curl.h>
34 #include <log4cpp/Category.hh>
35 #include <openssl/x509_vfy.h>
36
37 using namespace xmlsignature;
38 using namespace xmltooling;
39 using namespace log4cpp;
40 using namespace std;
41
42 namespace xmltooling {
43
44     // Manages cache of socket connections via CURL handles.
45     class XMLTOOL_DLLLOCAL CURLPool
46     {
47     public:
48         CURLPool() : m_size(0), m_lock(Mutex::create()),
49             m_log(Category::getInstance(XMLTOOLING_LOGCAT".SOAPTransport.CURLPool")) {}
50         ~CURLPool();
51         
52         CURL* get(const string& to, const char* endpoint);
53         void put(const string& to, const char* endpoint, CURL* handle);
54     
55     private:    
56         typedef map<string,vector<CURL*> > poolmap_t;
57         poolmap_t m_bindingMap;
58         list< vector<CURL*>* > m_pools;
59         long m_size;
60         Mutex* m_lock;
61         Category& m_log;
62     };
63     
64     static XMLTOOL_DLLLOCAL CURLPool* g_CURLPool = NULL;
65     
66     class XMLTOOL_DLLLOCAL CURLSOAPTransport : public HTTPSOAPTransport, public OpenSSLSOAPTransport
67     {
68     public:
69         CURLSOAPTransport(const KeyInfoSource& peer, const char* endpoint)
70                 : m_peer(peer), m_endpoint(endpoint), m_handle(NULL), m_headers(NULL),
71 #ifndef XMLTOOLING_NO_XMLSEC
72                     m_credResolver(NULL), m_trustEngine(NULL), m_mandatory(false), m_keyResolver(NULL),
73 #endif
74                     m_ssl_callback(NULL), m_ssl_userptr(NULL), m_chunked(true), m_secure(false) {
75             m_handle = g_CURLPool->get(peer.getName(), endpoint);
76             curl_easy_setopt(m_handle,CURLOPT_URL,endpoint);
77             curl_easy_setopt(m_handle,CURLOPT_CONNECTTIMEOUT,15);
78             curl_easy_setopt(m_handle,CURLOPT_TIMEOUT,30);
79             curl_easy_setopt(m_handle,CURLOPT_HTTPAUTH,0);
80             curl_easy_setopt(m_handle,CURLOPT_USERPWD,NULL);
81             curl_easy_setopt(m_handle,CURLOPT_HEADERDATA,this);
82             m_headers=curl_slist_append(m_headers,"Content-Type: text/xml");
83         }
84         
85         virtual ~CURLSOAPTransport() {
86             curl_slist_free_all(m_headers);
87             curl_easy_setopt(m_handle,CURLOPT_ERRORBUFFER,NULL);
88             curl_easy_setopt(m_handle,CURLOPT_PRIVATE,m_secure ? "secure" : NULL); // Save off security "state".
89             g_CURLPool->put(m_peer.getName(), m_endpoint.c_str(), m_handle);
90         }
91
92         bool isConfidential() const {
93             return m_endpoint.find("https")==0;
94         }
95
96         bool setConnectTimeout(long timeout) {
97             return (curl_easy_setopt(m_handle,CURLOPT_CONNECTTIMEOUT,timeout)==CURLE_OK);
98         }
99         
100         bool setTimeout(long timeout) {
101             return (curl_easy_setopt(m_handle,CURLOPT_TIMEOUT,timeout)==CURLE_OK);
102         }
103         
104         bool setAuth(transport_auth_t authType, const char* username=NULL, const char* password=NULL);
105         
106 #ifndef XMLTOOLING_NO_XMLSEC
107         bool setCredentialResolver(const CredentialResolver* credResolver) {
108             const OpenSSLCredentialResolver* down = dynamic_cast<const OpenSSLCredentialResolver*>(credResolver);
109             if (!down) {
110                 m_credResolver = NULL;
111                 return (credResolver==NULL);
112             }
113             m_credResolver = down;
114             return true;
115         }
116         
117         bool setTrustEngine(const X509TrustEngine* trustEngine, bool mandatory=true, const KeyResolver* keyResolver=NULL) {
118             const OpenSSLTrustEngine* down = dynamic_cast<const OpenSSLTrustEngine*>(trustEngine);
119             if (!down) {
120                 m_trustEngine = NULL;
121                 m_keyResolver = NULL;
122                 return (trustEngine==NULL);
123             }
124             m_trustEngine = down;
125             m_keyResolver = keyResolver;
126             m_mandatory = mandatory;
127             return true;
128         }
129         
130 #endif
131         
132         bool useChunkedEncoding(bool chunked=true) {
133             m_chunked = chunked;
134             return true;
135         }
136
137         void send(istream& in);
138         
139         istream& receive() {
140             return m_stream;
141         }
142         
143         bool isSecure() const {
144             return m_secure;
145         }
146
147         void setSecure(bool secure) {
148             m_secure = secure;
149         }
150
151         string getContentType() const;
152         
153         bool setRequestHeader(const char* name, const char* val) {
154             string temp(name);
155             temp=temp + ": " + val;
156             m_headers=curl_slist_append(m_headers,temp.c_str());
157             return true;
158         }
159         
160         const vector<string>& getResponseHeader(const char* val) const;
161         
162         bool setSSLCallback(ssl_ctx_callback_fn fn, void* userptr=NULL) {
163             m_ssl_callback=fn;
164             m_ssl_userptr=userptr;
165             return true;
166         }
167
168     private:        
169         // per-call state
170         const KeyInfoSource& m_peer;
171         string m_endpoint;
172         CURL* m_handle;
173         stringstream m_stream;
174         struct curl_slist* m_headers;
175         map<string,vector<string> > m_response_headers;
176 #ifndef XMLTOOLING_NO_XMLSEC
177         const OpenSSLCredentialResolver* m_credResolver;
178         const OpenSSLTrustEngine* m_trustEngine;
179         bool m_mandatory;
180         const KeyResolver* m_keyResolver;
181 #endif
182         ssl_ctx_callback_fn m_ssl_callback;
183         void* m_ssl_userptr;
184         bool m_chunked;
185         bool m_secure;
186         
187         friend size_t XMLTOOL_DLLLOCAL curl_header_hook(void* ptr, size_t size, size_t nmemb, void* stream);
188         friend CURLcode XMLTOOL_DLLLOCAL xml_ssl_ctx_callback(CURL* curl, SSL_CTX* ssl_ctx, void* userptr);
189         friend int XMLTOOL_DLLLOCAL verify_callback(X509_STORE_CTX* x509_ctx, void* arg);
190     };
191
192     // libcurl callback functions
193     size_t XMLTOOL_DLLLOCAL curl_header_hook(void* ptr, size_t size, size_t nmemb, void* stream);
194     size_t XMLTOOL_DLLLOCAL curl_write_hook(void* ptr, size_t size, size_t nmemb, void* stream);
195     size_t XMLTOOL_DLLLOCAL curl_read_hook( void *ptr, size_t size, size_t nmemb, void *stream);
196     int XMLTOOL_DLLLOCAL curl_debug_hook(CURL* handle, curl_infotype type, char* data, size_t len, void* ptr);
197     CURLcode XMLTOOL_DLLLOCAL xml_ssl_ctx_callback(CURL* curl, SSL_CTX* ssl_ctx, void* userptr);
198 #ifndef XMLTOOLING_NO_XMLSEC
199     int XMLTOOL_DLLLOCAL verify_callback(X509_STORE_CTX* x509_ctx, void* arg);
200 #endif
201
202     SOAPTransport* CURLSOAPTransportFactory(const pair<const KeyInfoSource*,const char*>& dest)
203     {
204         return new CURLSOAPTransport(*dest.first, dest.second);
205     }
206 };
207
208 void xmltooling::registerSOAPTransports()
209 {
210     XMLToolingConfig& conf=XMLToolingConfig::getConfig();
211     conf.SOAPTransportManager.registerFactory("http", CURLSOAPTransportFactory);
212     conf.SOAPTransportManager.registerFactory("https", CURLSOAPTransportFactory);
213 }
214
215 void xmltooling::initSOAPTransports()
216 {
217     g_CURLPool=new CURLPool();
218 }
219
220 void xmltooling::termSOAPTransports()
221 {
222     delete g_CURLPool;
223     g_CURLPool = NULL;
224 }
225
226 CURLPool::~CURLPool()
227 {
228     for (poolmap_t::iterator i=m_bindingMap.begin(); i!=m_bindingMap.end(); i++) {
229         for (vector<CURL*>::iterator j=i->second.begin(); j!=i->second.end(); j++)
230             curl_easy_cleanup(*j);
231     }
232     delete m_lock;
233 }
234
235 CURL* CURLPool::get(const string& to, const char* endpoint)
236 {
237 #ifdef _DEBUG
238     xmltooling::NDC("get");
239 #endif
240     m_log.debug("getting connection handle to %s", endpoint);
241     m_lock->lock();
242     poolmap_t::iterator i=m_bindingMap.find(to + "|" + endpoint);
243     
244     if (i!=m_bindingMap.end()) {
245         // Move this pool to the front of the list.
246         m_pools.remove(&(i->second));
247         m_pools.push_front(&(i->second));
248         
249         // If a free connection exists, return it.
250         if (!(i->second.empty())) {
251             CURL* handle=i->second.back();
252             i->second.pop_back();
253             m_size--;
254             m_lock->unlock();
255             m_log.debug("returning existing connection handle from pool");
256             return handle;
257         }
258     }
259     
260     m_lock->unlock();
261     m_log.debug("nothing free in pool, returning new connection handle");
262     
263     // Create a new connection and set non-varying options.
264     CURL* handle=curl_easy_init();
265     if (!handle)
266         return NULL;
267     curl_easy_setopt(handle,CURLOPT_NOPROGRESS,1);
268     curl_easy_setopt(handle,CURLOPT_NOSIGNAL,1);
269     curl_easy_setopt(handle,CURLOPT_FAILONERROR,1);
270     curl_easy_setopt(handle,CURLOPT_SSLVERSION,3);
271     // Verification of the peer is via TrustEngine only.
272     curl_easy_setopt(handle,CURLOPT_SSL_VERIFYPEER,0);
273     curl_easy_setopt(handle,CURLOPT_SSL_VERIFYHOST,2);
274     curl_easy_setopt(handle,CURLOPT_HEADERFUNCTION,&curl_header_hook);
275     curl_easy_setopt(handle,CURLOPT_WRITEFUNCTION,&curl_write_hook);
276     curl_easy_setopt(handle,CURLOPT_DEBUGFUNCTION,&curl_debug_hook);
277
278     return handle;
279 }
280
281 void CURLPool::put(const string& to, const char* endpoint, CURL* handle)
282 {
283     string key = to + "|" + endpoint;
284     m_lock->lock();
285     poolmap_t::iterator i=m_bindingMap.find(key);
286     if (i==m_bindingMap.end())
287         m_pools.push_front(&(m_bindingMap.insert(poolmap_t::value_type(key,vector<CURL*>(1,handle))).first->second));
288     else
289         i->second.push_back(handle);
290     
291     CURL* killit=NULL;
292     if (++m_size > 256) {
293         // Kick a handle out from the back of the bus.
294         while (true) {
295             vector<CURL*>* corpse=m_pools.back();
296             if (!corpse->empty()) {
297                 killit=corpse->back();
298                 corpse->pop_back();
299                 m_size--;
300                 break;
301             }
302             
303             // Move an empty pool up to the front so we don't keep hitting it.
304             m_pools.pop_back();
305             m_pools.push_front(corpse);
306         }
307     }
308     m_lock->unlock();
309     if (killit) {
310         curl_easy_cleanup(killit);
311 #ifdef _DEBUG
312         xmltooling::NDC("put");
313 #endif
314         m_log.info("conn_pool_max limit reached, dropping an old connection");
315     }
316 }
317
318 bool CURLSOAPTransport::setAuth(transport_auth_t authType, const char* username, const char* password)
319 {
320     if (authType==transport_auth_none) {
321         if (curl_easy_setopt(m_handle,CURLOPT_HTTPAUTH,0)!=CURLE_OK)
322             return false;
323         return (curl_easy_setopt(m_handle,CURLOPT_USERPWD,NULL)==CURLE_OK);
324     }
325     long flag=0;
326     switch (authType) {
327         case transport_auth_basic:    flag = CURLAUTH_BASIC; break;
328         case transport_auth_digest:   flag = CURLAUTH_DIGEST; break;
329         case transport_auth_ntlm:     flag = CURLAUTH_NTLM; break;
330         case transport_auth_gss:      flag = CURLAUTH_GSSNEGOTIATE; break;
331         default:            return false;
332     }
333     if (curl_easy_setopt(m_handle,CURLOPT_HTTPAUTH,flag)!=CURLE_OK)
334         return false;
335     string creds = string(username ? username : "") + ':' + (password ? password : "");
336     return (curl_easy_setopt(m_handle,CURLOPT_USERPWD,creds.c_str())==CURLE_OK);
337 }
338
339 const vector<string>& CURLSOAPTransport::getResponseHeader(const char* name) const
340 {
341     static vector<string> emptyVector;
342
343     map<string,vector<string> >::const_iterator i=m_response_headers.find(name);
344     if (i!=m_response_headers.end())
345         return i->second;
346     
347     for (map<string,vector<string> >::const_iterator j=m_response_headers.begin(); j!=m_response_headers.end(); j++) {
348 #ifdef HAVE_STRCASECMP
349         if (!strcasecmp(j->first.c_str(), name))
350 #else
351         if (!stricmp(j->first.c_str(), name))
352 #endif
353             return j->second;
354     }
355     
356     return emptyVector;
357 }
358
359 string CURLSOAPTransport::getContentType() const
360 {
361     char* content_type=NULL;
362     curl_easy_getinfo(m_handle,CURLINFO_CONTENT_TYPE,&content_type);
363     return content_type ? content_type : "";
364 }
365
366 void CURLSOAPTransport::send(istream& in)
367 {
368 #ifdef _DEBUG
369     xmltooling::NDC ndc("send");
370 #endif
371     Category& log=Category::getInstance(XMLTOOLING_LOGCAT".SOAPTransport");
372     Category& log_curl=Category::getInstance(XMLTOOLING_LOGCAT".libcurl");
373
374     string msg;
375
376     // By this time, the handle has been prepared with the URL to use and the
377     // caller should have executed any set functions to manipulate it.
378
379     // Setup standard per-call curl properties.
380     curl_easy_setopt(m_handle,CURLOPT_DEBUGDATA,&log_curl);
381     curl_easy_setopt(m_handle,CURLOPT_FILE,&m_stream);
382     curl_easy_setopt(m_handle,CURLOPT_POST,1);
383     if (m_chunked) {
384         m_headers=curl_slist_append(m_headers,"Transfer-Encoding: chunked");
385         curl_easy_setopt(m_handle,CURLOPT_READFUNCTION,&curl_read_hook);
386         curl_easy_setopt(m_handle,CURLOPT_READDATA,&in);
387     }
388     else {
389         char buf[1024];
390         while (in) {
391             in.read(buf,1024);
392             msg.append(buf,in.gcount());
393         }
394         curl_easy_setopt(m_handle,CURLOPT_READFUNCTION,NULL);
395         curl_easy_setopt(m_handle,CURLOPT_POSTFIELDS,msg.c_str());
396         curl_easy_setopt(m_handle,CURLOPT_POSTFIELDSIZE,msg.length());
397     }
398
399     char curl_errorbuf[CURL_ERROR_SIZE];
400     curl_errorbuf[0]=0;
401     curl_easy_setopt(m_handle,CURLOPT_ERRORBUFFER,curl_errorbuf);
402     if (log_curl.isDebugEnabled())
403         curl_easy_setopt(m_handle,CURLOPT_VERBOSE,1);
404
405     // Set request headers.
406     curl_easy_setopt(m_handle,CURLOPT_HTTPHEADER,m_headers);
407
408     if (m_ssl_callback || m_credResolver || m_trustEngine) {
409         curl_easy_setopt(m_handle,CURLOPT_SSL_CTX_FUNCTION,xml_ssl_ctx_callback);
410         curl_easy_setopt(m_handle,CURLOPT_SSL_CTX_DATA,this);
411
412         // Restore security "state". Necessary because the callback only runs
413         // when handshakes occur. Even new TCP connections won't execute it.
414         char* priv=NULL;
415         curl_easy_getinfo(m_handle,CURLINFO_PRIVATE,&priv);
416         if (priv)
417             m_secure=true;
418     }
419     else {
420         curl_easy_setopt(m_handle,CURLOPT_SSL_CTX_FUNCTION,NULL);
421         curl_easy_setopt(m_handle,CURLOPT_SSL_CTX_DATA,NULL);
422     }
423     
424     // Make the call.
425     log.debug("sending SOAP message to %s", m_endpoint.c_str());
426     if (curl_easy_perform(m_handle) != CURLE_OK) {
427         throw IOException(
428             string("CURLSOAPTransport failed while contacting SOAP responder: ") +
429                 (curl_errorbuf[0] ? curl_errorbuf : "no further information available"));
430     }
431 }
432
433 // callback to buffer headers from server
434 size_t xmltooling::curl_header_hook(void* ptr, size_t size, size_t nmemb, void* stream)
435 {
436     // only handle single-byte data
437     if (size!=1)
438         return 0;
439     CURLSOAPTransport* ctx = reinterpret_cast<CURLSOAPTransport*>(stream);
440     char* buf = (char*)malloc(nmemb + 1);
441     if (buf) {
442         memset(buf,0,nmemb + 1);
443         memcpy(buf,ptr,nmemb);
444         char* sep=(char*)strchr(buf,':');
445         if (sep) {
446             *(sep++)=0;
447             while (*sep==' ')
448                 *(sep++)=0;
449             char* white=buf+nmemb-1;
450             while (isspace(*white))
451                 *(white--)=0;
452             ctx->m_response_headers[buf].push_back(sep);
453         }
454         free(buf);
455         return nmemb;
456     }
457     return 0;
458 }
459
460 // callback to send data to server
461 size_t xmltooling::curl_read_hook(void* ptr, size_t size, size_t nmemb, void* stream)
462 {
463     // *stream is actually an istream object
464     istream& buf=*(reinterpret_cast<istream*>(stream));
465     buf.read(reinterpret_cast<char*>(ptr),size*nmemb);
466     return buf.gcount();
467 }
468
469 // callback to buffer data from server
470 size_t xmltooling::curl_write_hook(void* ptr, size_t size, size_t nmemb, void* stream)
471 {
472     size_t len = size*nmemb;
473     reinterpret_cast<stringstream*>(stream)->write(reinterpret_cast<const char*>(ptr),len);
474     return len;
475 }
476
477 // callback for curl debug data
478 int xmltooling::curl_debug_hook(CURL* handle, curl_infotype type, char* data, size_t len, void* ptr)
479 {
480     // *ptr is actually a logging object
481     if (!ptr) return 0;
482     CategoryStream log=reinterpret_cast<Category*>(ptr)->debugStream();
483     for (char* ch=data; len && (isprint(*ch) || isspace(*ch)); len--)
484         log << *ch++;
485     log << CategoryStream::ENDLINE;
486     return 0;
487 }
488
489 #ifndef XMLTOOLING_NO_XMLSEC
490 int xmltooling::verify_callback(X509_STORE_CTX* x509_ctx, void* arg)
491 {
492     Category& log = Category::getInstance("OpenSSL");
493     log.debug("invoking X509 verify callback");
494 #if (OPENSSL_VERSION_NUMBER >= 0x00907000L)
495     CURLSOAPTransport* ctx = reinterpret_cast<CURLSOAPTransport*>(arg);
496 #else
497     // Yes, this sucks. I'd use TLS, but there's no really obvious spot to put the thread key
498     // and global variables suck too. We can't access the X509_STORE_CTX depth directly because
499     // OpenSSL only copies it into the context if it's >=0, and the unsigned pointer may be
500     // negative in the SSL structure's int member.
501     CURLSOAPTransport* ctx = reinterpret_cast<CURLSOAPTransport*>(
502         SSL_get_verify_depth(
503             reinterpret_cast<SSL*>(X509_STORE_CTX_get_ex_data(x509_ctx,SSL_get_ex_data_X509_STORE_CTX_idx()))
504             )
505         );
506 #endif
507
508      // Bypass name check (handled for us by curl).
509     if (!ctx->m_trustEngine->validate(x509_ctx->cert,x509_ctx->untrusted,ctx->m_peer,false,ctx->m_keyResolver)) {
510         log.error("supplied TrustEngine failed to validate SSL/TLS server certificate");
511         x509_ctx->error=X509_V_ERR_APPLICATION_VERIFICATION;     // generic error, check log for plugin specifics
512         ctx->setSecure(false);
513         return ctx->m_mandatory ? 0 : 1;
514     }
515     
516     // Signal success. Hopefully it doesn't matter what's actually in the structure now.
517     ctx->setSecure(true);
518     return 1;
519 }
520 #endif
521
522 // callback to invoke a caller-defined SSL callback
523 CURLcode xmltooling::xml_ssl_ctx_callback(CURL* curl, SSL_CTX* ssl_ctx, void* userptr)
524 {
525     CURLSOAPTransport* conf = reinterpret_cast<CURLSOAPTransport*>(userptr);
526
527 #ifndef XMLTOOLING_NO_XMLSEC
528     if (conf->m_credResolver)
529         conf->m_credResolver->attach(ssl_ctx);
530
531     if (conf->m_trustEngine) {
532         SSL_CTX_set_verify(ssl_ctx,SSL_VERIFY_PEER,NULL);
533 #if (OPENSSL_VERSION_NUMBER >= 0x00907000L)
534         // With 0.9.7, we can pass a callback argument directly.
535         SSL_CTX_set_cert_verify_callback(ssl_ctx,verify_callback,userptr);
536 #else
537         // With 0.9.6, there's no argument, so we're going to use a really embarrassing hack and
538         // stuff the argument in the depth property where it will get copied to the context object
539         // that's handed to the callback.
540         SSL_CTX_set_cert_verify_callback(ssl_ctx,reinterpret_cast<int (*)()>(verify_callback),NULL);
541         SSL_CTX_set_verify_depth(ssl_ctx,reinterpret_cast<int>(userptr));
542 #endif
543     }
544 #endif
545         
546     if (conf->m_ssl_callback && !conf->m_ssl_callback(conf, ssl_ctx, conf->m_ssl_userptr))
547         return CURLE_SSL_CERTPROBLEM;
548         
549     return CURLE_OK;
550 }