f1a6022bb9039053ae559f22357465338e4f19a3
[shibboleth/cpp-xmltooling.git] / xmltooling / soap / impl / CURLSOAPTransport.cpp
1 /*
2  *  Copyright 2001-2007 Internet2
3  * 
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *     http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16
17 /**
18  * CURLSOAPTransport.cpp
19  * 
20  * libcurl-based SOAPTransport implementation
21  */
22
23 #include "internal.h"
24 #include "exceptions.h"
25 #include "security/CredentialCriteria.h"
26 #include "security/OpenSSLTrustEngine.h"
27 #include "security/OpenSSLCredential.h"
28 #include "soap/HTTPSOAPTransport.h"
29 #include "soap/OpenSSLSOAPTransport.h"
30 #include "util/NDC.h"
31 #include "util/Threads.h"
32
33 #include <list>
34 #include <curl/curl.h>
35 #include <log4cpp/Category.hh>
36 #include <openssl/x509_vfy.h>
37
38 using namespace xmltooling;
39 using namespace log4cpp;
40 using namespace std;
41
42 namespace xmltooling {
43
44     // Manages cache of socket connections via CURL handles.
45     class XMLTOOL_DLLLOCAL CURLPool
46     {
47     public:
48         CURLPool() : m_size(0), m_lock(Mutex::create()),
49             m_log(Category::getInstance(XMLTOOLING_LOGCAT".SOAPTransport.CURLPool")) {}
50         ~CURLPool();
51         
52         CURL* get(const char* to, const char* endpoint);
53         void put(const char* to, const char* endpoint, CURL* handle);
54     
55     private:    
56         typedef map<string,vector<CURL*> > poolmap_t;
57         poolmap_t m_bindingMap;
58         list< vector<CURL*>* > m_pools;
59         long m_size;
60         Mutex* m_lock;
61         Category& m_log;
62     };
63     
64     static XMLTOOL_DLLLOCAL CURLPool* g_CURLPool = NULL;
65     
66     class XMLTOOL_DLLLOCAL CURLSOAPTransport : public HTTPSOAPTransport, public OpenSSLSOAPTransport
67     {
68     public:
69         CURLSOAPTransport(const char* peerName, const char* endpoint)
70                 : m_peerName(peerName ? peerName : ""), m_endpoint(endpoint), m_handle(NULL), m_headers(NULL),
71 #ifndef XMLTOOLING_NO_XMLSEC
72                     m_cred(NULL), m_trustEngine(NULL), m_peerResolver(NULL), m_mandatory(false),
73 #endif
74                     m_ssl_callback(NULL), m_ssl_userptr(NULL), m_chunked(true), m_secure(false) {
75             m_handle = g_CURLPool->get(peerName, endpoint);
76             curl_easy_setopt(m_handle,CURLOPT_URL,endpoint);
77             curl_easy_setopt(m_handle,CURLOPT_CONNECTTIMEOUT,15);
78             curl_easy_setopt(m_handle,CURLOPT_TIMEOUT,30);
79             curl_easy_setopt(m_handle,CURLOPT_HTTPAUTH,0);
80             curl_easy_setopt(m_handle,CURLOPT_USERPWD,NULL);
81             curl_easy_setopt(m_handle,CURLOPT_SSL_VERIFYHOST,2);
82             curl_easy_setopt(m_handle,CURLOPT_HEADERDATA,this);
83             m_headers=curl_slist_append(m_headers,"Content-Type: text/xml");
84         }
85         
86         virtual ~CURLSOAPTransport() {
87             curl_slist_free_all(m_headers);
88             curl_easy_setopt(m_handle,CURLOPT_ERRORBUFFER,NULL);
89             curl_easy_setopt(m_handle,CURLOPT_PRIVATE,m_secure ? "secure" : NULL); // Save off security "state".
90             g_CURLPool->put(m_peerName.c_str(), m_endpoint.c_str(), m_handle);
91         }
92
93         bool isConfidential() const {
94             return m_endpoint.find("https")==0;
95         }
96
97         bool setConnectTimeout(long timeout) {
98             return (curl_easy_setopt(m_handle,CURLOPT_CONNECTTIMEOUT,timeout)==CURLE_OK);
99         }
100         
101         bool setTimeout(long timeout) {
102             return (curl_easy_setopt(m_handle,CURLOPT_TIMEOUT,timeout)==CURLE_OK);
103         }
104         
105         bool setAuth(transport_auth_t authType, const char* username=NULL, const char* password=NULL);
106         
107         bool setVerifyHost(bool verify) {
108             return (curl_easy_setopt(m_handle,CURLOPT_SSL_VERIFYHOST,verify ? 2 : 0)==CURLE_OK);
109         }
110         
111 #ifndef XMLTOOLING_NO_XMLSEC
112         bool setCredential(const Credential* cred=NULL) {
113             const OpenSSLCredential* down = dynamic_cast<const OpenSSLCredential*>(cred);
114             if (!down) {
115                 m_cred = NULL;
116                 return (cred==NULL);
117             }
118             m_cred = down;
119             return true;
120         }
121         
122         bool setTrustEngine(
123             const X509TrustEngine* trustEngine=NULL,
124             const CredentialResolver* peerResolver=NULL,
125             CredentialCriteria* criteria=NULL,
126             bool mandatory=true
127             ) {
128             const OpenSSLTrustEngine* down = dynamic_cast<const OpenSSLTrustEngine*>(trustEngine);
129             if (!down) {
130                 m_trustEngine = NULL;
131                 m_peerResolver = NULL;
132                 m_criteria = NULL;
133                 return (trustEngine==NULL);
134             }
135             m_trustEngine = down;
136             m_peerResolver = peerResolver;
137             m_criteria = criteria;
138             m_mandatory = mandatory;
139             return true;
140         }
141         
142 #endif
143         
144         bool useChunkedEncoding(bool chunked=true) {
145             m_chunked = chunked;
146             return true;
147         }
148
149         void send(istream& in);
150         
151         istream& receive() {
152             return m_stream;
153         }
154         
155         bool isSecure() const {
156             return m_secure;
157         }
158
159         void setSecure(bool secure) {
160             m_secure = secure;
161         }
162
163         string getContentType() const;
164         
165         bool setRequestHeader(const char* name, const char* val) {
166             string temp(name);
167             temp=temp + ": " + val;
168             m_headers=curl_slist_append(m_headers,temp.c_str());
169             return true;
170         }
171         
172         const vector<string>& getResponseHeader(const char* val) const;
173         
174         bool setSSLCallback(ssl_ctx_callback_fn fn, void* userptr=NULL) {
175             m_ssl_callback=fn;
176             m_ssl_userptr=userptr;
177             return true;
178         }
179
180     private:        
181         // per-call state
182         string m_peerName,m_endpoint,m_simplecreds;
183         CURL* m_handle;
184         stringstream m_stream;
185         struct curl_slist* m_headers;
186         map<string,vector<string> > m_response_headers;
187 #ifndef XMLTOOLING_NO_XMLSEC
188         const OpenSSLCredential* m_cred;
189         const OpenSSLTrustEngine* m_trustEngine;
190         const CredentialResolver* m_peerResolver;
191         CredentialCriteria* m_criteria;
192         bool m_mandatory;
193 #endif
194         ssl_ctx_callback_fn m_ssl_callback;
195         void* m_ssl_userptr;
196         bool m_chunked;
197         bool m_secure;
198         
199         friend size_t XMLTOOL_DLLLOCAL curl_header_hook(void* ptr, size_t size, size_t nmemb, void* stream);
200         friend CURLcode XMLTOOL_DLLLOCAL xml_ssl_ctx_callback(CURL* curl, SSL_CTX* ssl_ctx, void* userptr);
201         friend int XMLTOOL_DLLLOCAL verify_callback(X509_STORE_CTX* x509_ctx, void* arg);
202     };
203
204     // libcurl callback functions
205     size_t XMLTOOL_DLLLOCAL curl_header_hook(void* ptr, size_t size, size_t nmemb, void* stream);
206     size_t XMLTOOL_DLLLOCAL curl_write_hook(void* ptr, size_t size, size_t nmemb, void* stream);
207     size_t XMLTOOL_DLLLOCAL curl_read_hook( void *ptr, size_t size, size_t nmemb, void *stream);
208     int XMLTOOL_DLLLOCAL curl_debug_hook(CURL* handle, curl_infotype type, char* data, size_t len, void* ptr);
209     CURLcode XMLTOOL_DLLLOCAL xml_ssl_ctx_callback(CURL* curl, SSL_CTX* ssl_ctx, void* userptr);
210 #ifndef XMLTOOLING_NO_XMLSEC
211     int XMLTOOL_DLLLOCAL verify_callback(X509_STORE_CTX* x509_ctx, void* arg);
212 #endif
213
214     SOAPTransport* CURLSOAPTransportFactory(const pair<const char*,const char*>& dest)
215     {
216         return new CURLSOAPTransport(dest.first, dest.second);
217     }
218 };
219
220 void xmltooling::registerSOAPTransports()
221 {
222     XMLToolingConfig& conf=XMLToolingConfig::getConfig();
223     conf.SOAPTransportManager.registerFactory("http", CURLSOAPTransportFactory);
224     conf.SOAPTransportManager.registerFactory("https", CURLSOAPTransportFactory);
225 }
226
227 void xmltooling::initSOAPTransports()
228 {
229     g_CURLPool=new CURLPool();
230 }
231
232 void xmltooling::termSOAPTransports()
233 {
234     delete g_CURLPool;
235     g_CURLPool = NULL;
236 }
237
238 CURLPool::~CURLPool()
239 {
240     for (poolmap_t::iterator i=m_bindingMap.begin(); i!=m_bindingMap.end(); i++) {
241         for (vector<CURL*>::iterator j=i->second.begin(); j!=i->second.end(); j++)
242             curl_easy_cleanup(*j);
243     }
244     delete m_lock;
245 }
246
247 CURL* CURLPool::get(const char* to, const char* endpoint)
248 {
249 #ifdef _DEBUG
250     xmltooling::NDC("get");
251 #endif
252     m_log.debug("getting connection handle to %s", endpoint);
253     m_lock->lock();
254     poolmap_t::iterator i=m_bindingMap.find(string(to) + "|" + endpoint);
255     
256     if (i!=m_bindingMap.end()) {
257         // Move this pool to the front of the list.
258         m_pools.remove(&(i->second));
259         m_pools.push_front(&(i->second));
260         
261         // If a free connection exists, return it.
262         if (!(i->second.empty())) {
263             CURL* handle=i->second.back();
264             i->second.pop_back();
265             m_size--;
266             m_lock->unlock();
267             m_log.debug("returning existing connection handle from pool");
268             return handle;
269         }
270     }
271     
272     m_lock->unlock();
273     m_log.debug("nothing free in pool, returning new connection handle");
274     
275     // Create a new connection and set non-varying options.
276     CURL* handle=curl_easy_init();
277     if (!handle)
278         return NULL;
279     curl_easy_setopt(handle,CURLOPT_NOPROGRESS,1);
280     curl_easy_setopt(handle,CURLOPT_NOSIGNAL,1);
281     curl_easy_setopt(handle,CURLOPT_FAILONERROR,1);
282     curl_easy_setopt(handle,CURLOPT_SSLVERSION,3);
283     // Verification of the peer is via TrustEngine only.
284     curl_easy_setopt(handle,CURLOPT_SSL_VERIFYPEER,0);
285     curl_easy_setopt(handle,CURLOPT_HEADERFUNCTION,&curl_header_hook);
286     curl_easy_setopt(handle,CURLOPT_WRITEFUNCTION,&curl_write_hook);
287     curl_easy_setopt(handle,CURLOPT_DEBUGFUNCTION,&curl_debug_hook);
288
289     return handle;
290 }
291
292 void CURLPool::put(const char* to, const char* endpoint, CURL* handle)
293 {
294     string key = string(to) + "|" + endpoint;
295     m_lock->lock();
296     poolmap_t::iterator i=m_bindingMap.find(key);
297     if (i==m_bindingMap.end())
298         m_pools.push_front(&(m_bindingMap.insert(poolmap_t::value_type(key,vector<CURL*>(1,handle))).first->second));
299     else
300         i->second.push_back(handle);
301     
302     CURL* killit=NULL;
303     if (++m_size > 256) {
304         // Kick a handle out from the back of the bus.
305         while (true) {
306             vector<CURL*>* corpse=m_pools.back();
307             if (!corpse->empty()) {
308                 killit=corpse->back();
309                 corpse->pop_back();
310                 m_size--;
311                 break;
312             }
313             
314             // Move an empty pool up to the front so we don't keep hitting it.
315             m_pools.pop_back();
316             m_pools.push_front(corpse);
317         }
318     }
319     m_lock->unlock();
320     if (killit) {
321         curl_easy_cleanup(killit);
322 #ifdef _DEBUG
323         xmltooling::NDC("put");
324 #endif
325         m_log.info("conn_pool_max limit reached, dropping an old connection");
326     }
327 }
328
329 bool CURLSOAPTransport::setAuth(transport_auth_t authType, const char* username, const char* password)
330 {
331     if (authType==transport_auth_none) {
332         if (curl_easy_setopt(m_handle,CURLOPT_HTTPAUTH,0)!=CURLE_OK)
333             return false;
334         return (curl_easy_setopt(m_handle,CURLOPT_USERPWD,NULL)==CURLE_OK);
335     }
336     long flag=0;
337     switch (authType) {
338         case transport_auth_basic:    flag = CURLAUTH_BASIC; break;
339         case transport_auth_digest:   flag = CURLAUTH_DIGEST; break;
340         case transport_auth_ntlm:     flag = CURLAUTH_NTLM; break;
341         case transport_auth_gss:      flag = CURLAUTH_GSSNEGOTIATE; break;
342         default:            return false;
343     }
344     if (curl_easy_setopt(m_handle,CURLOPT_HTTPAUTH,flag)!=CURLE_OK)
345         return false;
346     m_simplecreds = string(username ? username : "") + ':' + (password ? password : "");
347     return (curl_easy_setopt(m_handle,CURLOPT_USERPWD,m_simplecreds.c_str())==CURLE_OK);
348 }
349
350 const vector<string>& CURLSOAPTransport::getResponseHeader(const char* name) const
351 {
352     static vector<string> emptyVector;
353
354     map<string,vector<string> >::const_iterator i=m_response_headers.find(name);
355     if (i!=m_response_headers.end())
356         return i->second;
357     
358     for (map<string,vector<string> >::const_iterator j=m_response_headers.begin(); j!=m_response_headers.end(); j++) {
359 #ifdef HAVE_STRCASECMP
360         if (!strcasecmp(j->first.c_str(), name))
361 #else
362         if (!stricmp(j->first.c_str(), name))
363 #endif
364             return j->second;
365     }
366     
367     return emptyVector;
368 }
369
370 string CURLSOAPTransport::getContentType() const
371 {
372     char* content_type=NULL;
373     curl_easy_getinfo(m_handle,CURLINFO_CONTENT_TYPE,&content_type);
374     return content_type ? content_type : "";
375 }
376
377 void CURLSOAPTransport::send(istream& in)
378 {
379 #ifdef _DEBUG
380     xmltooling::NDC ndc("send");
381 #endif
382     Category& log=Category::getInstance(XMLTOOLING_LOGCAT".SOAPTransport");
383     Category& log_curl=Category::getInstance(XMLTOOLING_LOGCAT".libcurl");
384
385     string msg;
386
387     // By this time, the handle has been prepared with the URL to use and the
388     // caller should have executed any set functions to manipulate it.
389
390     // Setup standard per-call curl properties.
391     curl_easy_setopt(m_handle,CURLOPT_DEBUGDATA,&log_curl);
392     curl_easy_setopt(m_handle,CURLOPT_FILE,&m_stream);
393     curl_easy_setopt(m_handle,CURLOPT_POST,1);
394     if (m_chunked) {
395         m_headers=curl_slist_append(m_headers,"Transfer-Encoding: chunked");
396         curl_easy_setopt(m_handle,CURLOPT_READFUNCTION,&curl_read_hook);
397         curl_easy_setopt(m_handle,CURLOPT_READDATA,&in);
398     }
399     else {
400         char buf[1024];
401         while (in) {
402             in.read(buf,1024);
403             msg.append(buf,in.gcount());
404         }
405         curl_easy_setopt(m_handle,CURLOPT_READFUNCTION,NULL);
406         curl_easy_setopt(m_handle,CURLOPT_POSTFIELDS,msg.c_str());
407         curl_easy_setopt(m_handle,CURLOPT_POSTFIELDSIZE,msg.length());
408     }
409
410     char curl_errorbuf[CURL_ERROR_SIZE];
411     curl_errorbuf[0]=0;
412     curl_easy_setopt(m_handle,CURLOPT_ERRORBUFFER,curl_errorbuf);
413     if (log_curl.isDebugEnabled())
414         curl_easy_setopt(m_handle,CURLOPT_VERBOSE,1);
415
416     // Set request headers.
417     curl_easy_setopt(m_handle,CURLOPT_HTTPHEADER,m_headers);
418
419 #ifndef XMLTOOLING_NO_XMLSEC
420     if (m_ssl_callback || m_cred || m_trustEngine) {
421 #else
422     if (m_ssl_callback) {
423 #endif
424         curl_easy_setopt(m_handle,CURLOPT_SSL_CTX_FUNCTION,xml_ssl_ctx_callback);
425         curl_easy_setopt(m_handle,CURLOPT_SSL_CTX_DATA,this);
426
427         // Restore security "state". Necessary because the callback only runs
428         // when handshakes occur. Even new TCP connections won't execute it.
429         char* priv=NULL;
430         curl_easy_getinfo(m_handle,CURLINFO_PRIVATE,&priv);
431         if (priv)
432             m_secure=true;
433     }
434     else {
435         curl_easy_setopt(m_handle,CURLOPT_SSL_CTX_FUNCTION,NULL);
436         curl_easy_setopt(m_handle,CURLOPT_SSL_CTX_DATA,NULL);
437     }
438     
439     // Make the call.
440     log.debug("sending SOAP message to %s", m_endpoint.c_str());
441     if (curl_easy_perform(m_handle) != CURLE_OK) {
442         throw IOException(
443             string("CURLSOAPTransport failed while contacting SOAP responder: ") +
444                 (curl_errorbuf[0] ? curl_errorbuf : "no further information available"));
445     }
446 }
447
448 // callback to buffer headers from server
449 size_t xmltooling::curl_header_hook(void* ptr, size_t size, size_t nmemb, void* stream)
450 {
451     // only handle single-byte data
452     if (size!=1)
453         return 0;
454     CURLSOAPTransport* ctx = reinterpret_cast<CURLSOAPTransport*>(stream);
455     char* buf = (char*)malloc(nmemb + 1);
456     if (buf) {
457         memset(buf,0,nmemb + 1);
458         memcpy(buf,ptr,nmemb);
459         char* sep=(char*)strchr(buf,':');
460         if (sep) {
461             *(sep++)=0;
462             while (*sep==' ')
463                 *(sep++)=0;
464             char* white=buf+nmemb-1;
465             while (isspace(*white))
466                 *(white--)=0;
467             ctx->m_response_headers[buf].push_back(sep);
468         }
469         free(buf);
470         return nmemb;
471     }
472     return 0;
473 }
474
475 // callback to send data to server
476 size_t xmltooling::curl_read_hook(void* ptr, size_t size, size_t nmemb, void* stream)
477 {
478     // *stream is actually an istream object
479     istream& buf=*(reinterpret_cast<istream*>(stream));
480     buf.read(reinterpret_cast<char*>(ptr),size*nmemb);
481     return buf.gcount();
482 }
483
484 // callback to buffer data from server
485 size_t xmltooling::curl_write_hook(void* ptr, size_t size, size_t nmemb, void* stream)
486 {
487     size_t len = size*nmemb;
488     reinterpret_cast<stringstream*>(stream)->write(reinterpret_cast<const char*>(ptr),len);
489     return len;
490 }
491
492 // callback for curl debug data
493 int xmltooling::curl_debug_hook(CURL* handle, curl_infotype type, char* data, size_t len, void* ptr)
494 {
495     // *ptr is actually a logging object
496     if (!ptr) return 0;
497     CategoryStream log=reinterpret_cast<Category*>(ptr)->debugStream();
498     for (unsigned char* ch=(unsigned char*)data; len && (isprint(*ch) || isspace(*ch)); len--)
499         log << *ch++;
500     return 0;
501 }
502
503 #ifndef XMLTOOLING_NO_XMLSEC
504 int xmltooling::verify_callback(X509_STORE_CTX* x509_ctx, void* arg)
505 {
506     Category& log = Category::getInstance("OpenSSL");
507     log.debug("invoking X509 verify callback");
508 #if (OPENSSL_VERSION_NUMBER >= 0x00907000L)
509     CURLSOAPTransport* ctx = reinterpret_cast<CURLSOAPTransport*>(arg);
510 #else
511     // Yes, this sucks. I'd use TLS, but there's no really obvious spot to put the thread key
512     // and global variables suck too. We can't access the X509_STORE_CTX depth directly because
513     // OpenSSL only copies it into the context if it's >=0, and the unsigned pointer may be
514     // negative in the SSL structure's int member.
515     CURLSOAPTransport* ctx = reinterpret_cast<CURLSOAPTransport*>(
516         SSL_get_verify_depth(
517             reinterpret_cast<SSL*>(X509_STORE_CTX_get_ex_data(x509_ctx,SSL_get_ex_data_X509_STORE_CTX_idx()))
518             )
519         );
520 #endif
521
522     bool success=false;
523     if (ctx->m_criteria) {
524         ctx->m_criteria->setUsage(CredentialCriteria::TLS_CREDENTIAL);
525         // Bypass name check (handled for us by curl).
526         ctx->m_criteria->setPeerName(NULL);
527         success = ctx->m_trustEngine->validate(x509_ctx->cert,x509_ctx->untrusted,*(ctx->m_peerResolver),ctx->m_criteria);
528     }
529     else {
530         // Bypass name check (handled for us by curl).
531         CredentialCriteria cc;
532         cc.setUsage(CredentialCriteria::TLS_CREDENTIAL);
533         success = ctx->m_trustEngine->validate(x509_ctx->cert,x509_ctx->untrusted,*(ctx->m_peerResolver),&cc);
534     }
535     
536     if (!success) {
537         log.error("supplied TrustEngine failed to validate SSL/TLS server certificate");
538         x509_ctx->error=X509_V_ERR_APPLICATION_VERIFICATION;     // generic error, check log for plugin specifics
539         ctx->setSecure(false);
540         return ctx->m_mandatory ? 0 : 1;
541     }
542     
543     // Signal success. Hopefully it doesn't matter what's actually in the structure now.
544     ctx->setSecure(true);
545     return 1;
546 }
547 #endif
548
549 // callback to invoke a caller-defined SSL callback
550 CURLcode xmltooling::xml_ssl_ctx_callback(CURL* curl, SSL_CTX* ssl_ctx, void* userptr)
551 {
552     CURLSOAPTransport* conf = reinterpret_cast<CURLSOAPTransport*>(userptr);
553
554 #ifndef XMLTOOLING_NO_XMLSEC
555     if (conf->m_cred)
556         conf->m_cred->attach(ssl_ctx);
557
558     if (conf->m_trustEngine) {
559         SSL_CTX_set_verify(ssl_ctx,SSL_VERIFY_PEER,NULL);
560 #if (OPENSSL_VERSION_NUMBER >= 0x00907000L)
561         // With 0.9.7, we can pass a callback argument directly.
562         SSL_CTX_set_cert_verify_callback(ssl_ctx,verify_callback,userptr);
563 #else
564         // With 0.9.6, there's no argument, so we're going to use a really embarrassing hack and
565         // stuff the argument in the depth property where it will get copied to the context object
566         // that's handed to the callback.
567         SSL_CTX_set_cert_verify_callback(ssl_ctx,reinterpret_cast<int (*)()>(verify_callback),NULL);
568         SSL_CTX_set_verify_depth(ssl_ctx,reinterpret_cast<int>(userptr));
569 #endif
570     }
571 #endif
572         
573     if (conf->m_ssl_callback && !conf->m_ssl_callback(conf, ssl_ctx, conf->m_ssl_userptr))
574         return CURLE_SSL_CERTPROBLEM;
575         
576     return CURLE_OK;
577 }