X-Git-Url: http://www.project-moonshot.org/gitweb/?a=blobdiff_plain;f=xmltooling%2Futil%2FCurlURLInputStream.cpp;h=4d1e4b3caf2f8ec87f8dfc7f836d29fb2d4dd2c7;hb=81b488b2790e7bdeb2f43560b1d4a7d22c3dfdf5;hp=92ac8b1b6aedaa45a888b5ed9ebae62dfb6792f3;hpb=a285d7f74216f07df3ff2586a5aa6e783fc5204a;p=shibboleth%2Fcpp-xmltooling.git diff --git a/xmltooling/util/CurlURLInputStream.cpp b/xmltooling/util/CurlURLInputStream.cpp index 92ac8b1..4d1e4b3 100644 --- a/xmltooling/util/CurlURLInputStream.cpp +++ b/xmltooling/util/CurlURLInputStream.cpp @@ -1,18 +1,21 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at +/** + * Licensed to the University Corporation for Advanced Internet + * Development, Inc. (UCAID) under one or more contributor license + * agreements. See the NOTICE file distributed with this work for + * additional information regarding copyright ownership. + * + * UCAID licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the + * License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific + * language governing permissions and limitations under the License. */ /** @@ -89,6 +92,32 @@ namespace { break; } // append until whitespace + cacheTag->erase(); + while (remaining > 0) { + if (!isspace(*hdr)) { + (*cacheTag) += *hdr++; + --remaining; + continue; + } + break; + } + + if (!cacheTag->empty()) + *cacheTag = "If-None-Match: " + *cacheTag; + } + else if (cacheTag->empty() && strncmp(hdr, "Last-Modified:", 14) == 0) { + hdr += 14; + size_t remaining = nmemb - 14; + // skip leading spaces + while (remaining > 0) { + if (*hdr == ' ') { + ++hdr; + --remaining; + continue; + } + break; + } + // append until whitespace while (remaining > 0) { if (!isspace(*hdr)) { (*cacheTag) += *hdr++; @@ -97,6 +126,9 @@ namespace { } break; } + + if (!cacheTag->empty()) + *cacheTag = "If-Modified-Since: " + *cacheTag; } return nmemb; @@ -106,8 +138,8 @@ namespace { CurlURLInputStream::CurlURLInputStream(const char* url, string* cacheTag) : fLog(logging::Category::getInstance(XMLTOOLING_LOGCAT".libcurl.InputStream")) , fCacheTag(cacheTag) - , fOpenSSLOps(SSL_OP_ALL|SSL_OP_NO_SSLv2) , fURL(url ? url : "") + , fOpenSSLOps(SSL_OP_ALL|SSL_OP_NO_SSLv2) , fMulti(0) , fEasy(0) , fHeaders(0) @@ -116,8 +148,10 @@ CurlURLInputStream::CurlURLInputStream(const char* url, string* cacheTag) , fBytesRead(0) , fBytesToRead(0) , fDataAvailable(false) - , fBufferHeadPtr(fBuffer) - , fBufferTailPtr(fBuffer) + , fBuffer(0) + , fBufferHeadPtr(0) + , fBufferTailPtr(0) + , fBufferSize(0) , fContentType(0) , fStatusCode(200) { @@ -138,8 +172,10 @@ CurlURLInputStream::CurlURLInputStream(const XMLCh* url, string* cacheTag) , fBytesRead(0) , fBytesToRead(0) , fDataAvailable(false) - , fBufferHeadPtr(fBuffer) - , fBufferTailPtr(fBuffer) + , fBuffer(0) + , fBufferHeadPtr(0) + , fBufferTailPtr(0) + , fBufferSize(0) , fContentType(0) , fStatusCode(200) { @@ -164,14 +200,16 @@ CurlURLInputStream::CurlURLInputStream(const DOMElement* e, string* cacheTag) , fBytesRead(0) , fBytesToRead(0) , fDataAvailable(false) - , fBufferHeadPtr(fBuffer) - , fBufferTailPtr(fBuffer) + , fBuffer(0) + , fBufferHeadPtr(0) + , fBufferTailPtr(0) + , fBufferSize(0) , fContentType(0) , fStatusCode(200) { - const XMLCh* attr = e->getAttributeNS(NULL, url); + const XMLCh* attr = e->getAttributeNS(nullptr, url); if (!attr || !*attr) { - attr = e->getAttributeNS(NULL, uri); + attr = e->getAttributeNS(nullptr, uri); if (!attr || !*attr) throw IOException("No URL supplied via DOM to CurlURLInputStream constructor."); } @@ -201,6 +239,7 @@ CurlURLInputStream::~CurlURLInputStream() } XMLString::release(&fContentType); + free(fBuffer); } void CurlURLInputStream::init(const DOMElement* e) @@ -225,17 +264,18 @@ void CurlURLInputStream::init(const DOMElement* e) curl_easy_setopt(fEasy, CURLOPT_MAXREDIRS, 6); // Default settings. - curl_easy_setopt(fEasy, CURLOPT_CONNECTTIMEOUT,10); - curl_easy_setopt(fEasy, CURLOPT_TIMEOUT,60); - curl_easy_setopt(fEasy, CURLOPT_HTTPAUTH,0); - curl_easy_setopt(fEasy, CURLOPT_USERPWD,NULL); + curl_easy_setopt(fEasy, CURLOPT_CONNECTTIMEOUT, 10); + curl_easy_setopt(fEasy, CURLOPT_TIMEOUT, 60); + curl_easy_setopt(fEasy, CURLOPT_HTTPAUTH, 0); + curl_easy_setopt(fEasy, CURLOPT_USERPWD, nullptr); curl_easy_setopt(fEasy, CURLOPT_SSL_VERIFYHOST, 2); curl_easy_setopt(fEasy, CURLOPT_SSL_VERIFYPEER, 0); - curl_easy_setopt(fEasy, CURLOPT_CAINFO, NULL); + curl_easy_setopt(fEasy, CURLOPT_CAINFO, nullptr); curl_easy_setopt(fEasy, CURLOPT_SSL_CIPHER_LIST, "ALL:!aNULL:!LOW:!EXPORT:!SSLv2"); curl_easy_setopt(fEasy, CURLOPT_NOPROGRESS, 1); curl_easy_setopt(fEasy, CURLOPT_NOSIGNAL, 1); curl_easy_setopt(fEasy, CURLOPT_FAILONERROR, 1); + curl_easy_setopt(fEasy, CURLOPT_ENCODING, ""); // Install SSL callback. curl_easy_setopt(fEasy, CURLOPT_SSL_CTX_FUNCTION, ssl_ctx_callback); @@ -248,18 +288,24 @@ void CurlURLInputStream::init(const DOMElement* e) if (fCacheTag) { // Outgoing tag. if (!fCacheTag->empty()) { - string hdr("If-None-Match: "); - hdr += *fCacheTag; - fHeaders = curl_slist_append(fHeaders, hdr.c_str()); - curl_easy_setopt(fEasy, CURLOPT_HTTPHEADER, fHeaders); + fHeaders = curl_slist_append(fHeaders, fCacheTag->c_str()); } // Incoming tag. curl_easy_setopt(fEasy, CURLOPT_HEADERFUNCTION, curl_header_hook); curl_easy_setopt(fEasy, CURLOPT_HEADERDATA, fCacheTag); } + // Add User-Agent as a header for now. TODO: Add private member to hold the + // value for the standard UA option. + string ua = string("User-Agent: ") + XMLToolingConfig::getConfig().user_agent + + " libcurl/" + LIBCURL_VERSION + ' ' + OPENSSL_VERSION_TEXT; + fHeaders = curl_slist_append(fHeaders, ua.c_str()); + + // Add User-Agent and cache headers. + curl_easy_setopt(fEasy, CURLOPT_HTTPHEADER, fHeaders); + if (e) { - const XMLCh* flag = e->getAttributeNS(NULL, verifyHost); + const XMLCh* flag = e->getAttributeNS(nullptr, verifyHost); if (flag && (*flag == chLatin_f || *flag == chDigit_0)) curl_easy_setopt(fEasy, CURLOPT_SSL_VERIFYHOST, 0); @@ -267,8 +313,8 @@ void CurlURLInputStream::init(const DOMElement* e) bool success; DOMElement* child = XMLHelper::getLastChildElement(e, TransportOption); while (child) { - if (child->hasChildNodes() && XMLString::equals(child->getAttributeNS(NULL,_provider), _OpenSSL)) { - auto_ptr_char option(child->getAttributeNS(NULL,_option)); + if (child->hasChildNodes() && XMLString::equals(child->getAttributeNS(nullptr,_provider), _OpenSSL)) { + auto_ptr_char option(child->getAttributeNS(nullptr,_option)); auto_ptr_char value(child->getFirstChild()->getNodeValue()); if (option.get() && value.get() && !strcmp(option.get(), "SSL_OP_ALLOW_UNSAFE_LEGACY_RENEGOTIATION") && (*value.get()=='1' || *value.get()=='t')) { @@ -292,14 +338,14 @@ void CurlURLInputStream::init(const DOMElement* e) if (!success) fLog.error("failed to set OpenSSL transport option (%s)", option.get()); } - else if (child->hasChildNodes() && XMLString::equals(child->getAttributeNS(NULL,_provider), _CURL)) { - auto_ptr_char option(child->getAttributeNS(NULL,_option)); + else if (child->hasChildNodes() && XMLString::equals(child->getAttributeNS(nullptr,_provider), _CURL)) { + auto_ptr_char option(child->getAttributeNS(nullptr,_option)); auto_ptr_char value(child->getFirstChild()->getNodeValue()); if (option.get() && *option.get() && value.get() && *value.get()) { // For libcurl, the option is an enum and the value type depends on the option. - CURLoption opt = static_cast(strtol(option.get(), NULL, 10)); + CURLoption opt = static_cast(strtol(option.get(), nullptr, 10)); if (opt < CURLOPTTYPE_OBJECTPOINT) - success = (curl_easy_setopt(fEasy, opt, strtol(value.get(), NULL, 10)) == CURLE_OK); + success = (curl_easy_setopt(fEasy, opt, strtol(value.get(), nullptr, 10)) == CURLE_OK); #ifdef CURLOPTTYPE_OFF_T else if (opt < CURLOPTTYPE_OFF_T) { fSavedOptions.push_back(value.get()); @@ -307,10 +353,10 @@ void CurlURLInputStream::init(const DOMElement* e) } # ifdef HAVE_CURL_OFF_T else if (sizeof(curl_off_t) == sizeof(long)) - success = (curl_easy_setopt(fEasy, opt, strtol(value.get(), NULL, 10)) == CURLE_OK); + success = (curl_easy_setopt(fEasy, opt, strtol(value.get(), nullptr, 10)) == CURLE_OK); # else else if (sizeof(off_t) == sizeof(long)) - success = (curl_easy_setopt(fEasy, opt, strtol(value.get(), NULL, 10)) == CURLE_OK); + success = (curl_easy_setopt(fEasy, opt, strtol(value.get(), nullptr, 10)) == CURLE_OK); # endif else success = false; @@ -342,9 +388,9 @@ void CurlURLInputStream::init(const DOMElement* e) catch (XMLException&) { curl_multi_remove_handle(fMulti, fEasy); curl_easy_cleanup(fEasy); - fEasy = NULL; + fEasy = nullptr; curl_multi_cleanup(fMulti); - fMulti = NULL; + fMulti = nullptr; throw; } if(runningHandles == 0) break; @@ -359,8 +405,17 @@ void CurlURLInputStream::init(const DOMElement* e) << fStatusCode << "'; string specialxml = specialdoc.str(); + fBufferTailPtr = fBuffer = reinterpret_cast(malloc(specialxml.length())); + if (!fBuffer) { + curl_multi_remove_handle(fMulti, fEasy); + curl_easy_cleanup(fEasy); + fEasy = nullptr; + curl_multi_cleanup(fMulti); + fMulti = nullptr; + throw bad_alloc(); + } memcpy(fBuffer, specialxml.c_str(), specialxml.length()); - fBufferHeadPtr += specialxml.length(); + fBufferHeadPtr = fBuffer + specialxml.length(); } } else { @@ -368,7 +423,7 @@ void CurlURLInputStream::init(const DOMElement* e) } // Find the content type - char* contentType8 = NULL; + char* contentType8 = nullptr; if(curl_easy_getinfo(fEasy, CURLINFO_CONTENT_TYPE, &contentType8) == CURLE_OK && contentType8) fContentType = XMLString::transcode(contentType8); } @@ -392,7 +447,7 @@ size_t CurlURLInputStream::writeCallback(char* buffer, size_t size, size_t nitem fTotalBytesRead += consume; fBytesToRead -= consume; - //fLog.debug("write callback consuming %d bytes", consume); + fLog.debug("write callback consuming %u bytes", consume); // If bytes remain, rebuffer as many as possible into our holding buffer buffer += consume; @@ -400,13 +455,22 @@ size_t CurlURLInputStream::writeCallback(char* buffer, size_t size, size_t nitem cnt -= consume; if (cnt > 0) { - size_t bufAvail = sizeof(fBuffer) - (fBufferHeadPtr - fBuffer); - consume = (cnt > bufAvail) ? bufAvail : cnt; - memcpy(fBufferHeadPtr, buffer, consume); - fBufferHeadPtr += consume; - buffer += consume; - totalConsumed += consume; - //fLog.debug("write callback rebuffering %d bytes", consume); + size_t bufAvail = fBufferSize - (fBufferHeadPtr - fBuffer); + if (bufAvail < cnt) { + // Enlarge the buffer. TODO: limit max size + XMLByte* newbuf = reinterpret_cast(realloc(fBuffer, fBufferSize + (cnt - bufAvail))); + if (newbuf) { + fBufferSize = fBufferSize + (cnt - bufAvail); + fLog.debug("enlarged buffer to %u bytes", fBufferSize); + fBufferHeadPtr = newbuf + (fBufferHeadPtr - fBuffer); + fBuffer = fBufferTailPtr = newbuf; + } + } + memcpy(fBufferHeadPtr, buffer, cnt); + fBufferHeadPtr += cnt; + buffer += cnt; + totalConsumed += cnt; + fLog.debug("write callback rebuffering %u bytes", cnt); } // Return the total amount we've consumed. If we don't consume all the bytes @@ -423,9 +487,9 @@ bool CurlURLInputStream::readMore(int* runningHandles) // Process messages from curl int msgsInQueue = 0; - for (CURLMsg* msg = NULL; (msg = curl_multi_info_read(fMulti, &msgsInQueue)) != NULL; ) + for (CURLMsg* msg = nullptr; (msg = curl_multi_info_read(fMulti, &msgsInQueue)) != nullptr; ) { - //fLog.debug("msg %d, %d from curl", msg->msg, msg->data.result); + fLog.debug("msg %d, %d from curl", msg->msg, msg->data.result); if (msg->msg != CURLMSG_DONE) return true; @@ -517,7 +581,7 @@ xsecsize_t CurlURLInputStream::readBytes(XMLByte* const toFill, const xsecsize_t if (fBufferTailPtr == fBufferHeadPtr) fBufferHeadPtr = fBufferTailPtr = fBuffer; - //fLog.debug("consuming %d buffered bytes", bufCnt); + fLog.debug("consuming %d buffered bytes", bufCnt); tryAgain = true; continue;