2 * Licensed to the Apache Software Foundation (ASF) under one or more
3 * contributor license agreements. See the NOTICE file distributed with
4 * this work for additional information regarding copyright ownership.
5 * The ASF licenses this file to You under the Apache License, Version 2.0
6 * (the "License"); you may not use this file except in compliance with
7 * the License. You may obtain a copy of the License at
9 * http://www.apache.org/licenses/LICENSE-2.0
11 * Unless required by applicable law or agreed to in writing, software
12 * distributed under the License is distributed on an "AS IS" BASIS,
13 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 * See the License for the specific language governing permissions and
15 * limitations under the License.
31 //#include <sys/types.h>
32 //#include <sys/time.h>
34 #include <xercesc/util/XercesDefs.hpp>
35 #include <xercesc/util/XMLNetAccessor.hpp>
36 #include <xercesc/util/XMLString.hpp>
37 #include <xercesc/util/XMLExceptMsgs.hpp>
38 #include <xercesc/util/Janitor.hpp>
39 #include <xercesc/util/XMLUniDefs.hpp>
40 #include <xercesc/util/TransService.hpp>
41 #include <xercesc/util/TranscodingException.hpp>
42 #include <xercesc/util/PlatformUtils.hpp>
44 #include <xmltooling/logging.h>
45 #include <xmltooling/util/CurlURLInputStream.hpp>
47 using namespace xmltooling;
50 CurlURLInputStream::CurlURLInputStream(const XMLURL& urlSource, const XMLNetHTTPInfo* httpInfo/*=0*/)
53 , fMemoryManager(urlSource.getMemoryManager())
54 , fURLSource(urlSource)
60 , fDataAvailable(false)
61 , fBufferHeadPtr(fBuffer)
62 , fBufferTailPtr(fBuffer)
64 // Allocate the curl multi handle
65 fMulti = curl_multi_init();
67 // Allocate the curl easy handle
68 fEasy = curl_easy_init();
70 // Get the text of the URL we're going to use
71 fURL.reset(XMLString::transcode(fURLSource.getURLText(), fMemoryManager), fMemoryManager);
73 //printf("Curl trying to fetch %s\n", fURL.get());
76 curl_easy_setopt(fEasy, CURLOPT_URL, fURL.get());
77 curl_easy_setopt(fEasy, CURLOPT_WRITEDATA, this); // Pass this pointer to write function
78 curl_easy_setopt(fEasy, CURLOPT_WRITEFUNCTION, staticWriteCallback); // Our static write function
79 curl_easy_setopt(fEasy, CURLOPT_CONNECTTIMEOUT, 15);
80 curl_easy_setopt(fEasy, CURLOPT_TIMEOUT, 30);
81 curl_easy_setopt(fEasy, CURLOPT_SSLVERSION, CURL_SSLVERSION_SSLv3);
82 curl_easy_setopt(fEasy, CURLOPT_SSL_VERIFYHOST, 0);
83 curl_easy_setopt(fEasy, CURLOPT_SSL_VERIFYPEER, 0);
84 curl_easy_setopt(fEasy, CURLOPT_NOPROGRESS, 1);
85 curl_easy_setopt(fEasy, CURLOPT_NOSIGNAL, 1);
86 curl_easy_setopt(fEasy, CURLOPT_FAILONERROR, 1);
88 // Add easy handle to the multi stack
89 curl_multi_add_handle(fMulti, fEasy);
93 CurlURLInputStream::~CurlURLInputStream()
95 // Remove the easy handle from the multi stack
96 curl_multi_remove_handle(fMulti, fEasy);
98 // Cleanup the easy handle
99 curl_easy_cleanup(fEasy);
101 // Cleanup the multi handle
102 curl_multi_cleanup(fMulti);
107 CurlURLInputStream::staticWriteCallback(char *buffer,
112 return ((CurlURLInputStream*)outstream)->writeCallback(buffer, size, nitems);
118 CurlURLInputStream::writeCallback(char *buffer,
122 size_t cnt = size * nitems;
123 size_t totalConsumed = 0;
125 // Consume as many bytes as possible immediately into the buffer
126 size_t consume = (cnt > fBytesToRead) ? fBytesToRead : cnt;
127 memcpy(fWritePtr, buffer, consume);
128 fWritePtr += consume;
129 fBytesRead += consume;
130 fTotalBytesRead += consume;
131 fBytesToRead -= consume;
133 //printf("write callback consuming %d bytes\n", consume);
135 // If bytes remain, rebuffer as many as possible into our holding buffer
137 totalConsumed += consume;
141 size_t bufAvail = sizeof(fBuffer) - (fBufferHeadPtr - fBuffer);
142 consume = (cnt > bufAvail) ? bufAvail : cnt;
143 memcpy(fBufferHeadPtr, buffer, consume);
144 fBufferHeadPtr += consume;
146 totalConsumed += consume;
147 //printf("write callback rebuffering %d bytes\n", consume);
150 // Return the total amount we've consumed. If we don't consume all the bytes
151 // then an error will be generated. Since our buffer size is equal to the
152 // maximum size that curl will write, this should never happen unless there
153 // is a logic error somewhere here.
154 return totalConsumed;
159 CurlURLInputStream::readBytes(XMLByte* const toFill
160 , const unsigned int maxToRead)
163 fBytesToRead = maxToRead;
166 for (bool tryAgain = true; fBytesToRead > 0 && (tryAgain || fBytesRead == 0); )
168 // First, any buffered data we have available
169 size_t bufCnt = fBufferHeadPtr - fBufferTailPtr;
170 bufCnt = (bufCnt > fBytesToRead) ? fBytesToRead : bufCnt;
173 memcpy(fWritePtr, fBufferTailPtr, bufCnt);
175 fBytesRead += bufCnt;
176 fTotalBytesRead += bufCnt;
177 fBytesToRead -= bufCnt;
179 fBufferTailPtr += bufCnt;
180 if (fBufferTailPtr == fBufferHeadPtr)
181 fBufferHeadPtr = fBufferTailPtr = fBuffer;
183 //printf("consuming %d buffered bytes\n", bufCnt);
189 // Ask the curl to do some work
190 int runningHandles = 0;
191 CURLMcode curlResult = curl_multi_perform(fMulti, &runningHandles);
192 tryAgain = (curlResult == CURLM_CALL_MULTI_PERFORM);
194 // Process messages from curl
196 for (CURLMsg* msg = NULL; (msg = curl_multi_info_read(fMulti, &msgsInQueue)) != NULL; )
198 //printf("msg %d, %d from curl\n", msg->msg, msg->data.result);
200 if (msg->msg != CURLMSG_DONE)
203 switch (msg->data.result)
206 // We completed successfully. runningHandles should have dropped to zero, so we'll bail out below...
209 case CURLE_UNSUPPORTED_PROTOCOL:
210 ThrowXMLwithMemMgr(MalformedURLException, XMLExcepts::URL_UnsupportedProto, fMemoryManager);
213 case CURLE_COULDNT_RESOLVE_HOST:
214 case CURLE_COULDNT_RESOLVE_PROXY:
215 ThrowXMLwithMemMgr1(NetAccessorException, XMLExcepts::NetAcc_TargetResolution, fURLSource.getHost(), fMemoryManager);
218 case CURLE_COULDNT_CONNECT:
219 ThrowXMLwithMemMgr1(NetAccessorException, XMLExcepts::NetAcc_ConnSocket, fURLSource.getURLText(), fMemoryManager);
221 case CURLE_RECV_ERROR:
222 ThrowXMLwithMemMgr1(NetAccessorException, XMLExcepts::NetAcc_ReadSocket, fURLSource.getURLText(), fMemoryManager);
226 logging::Category::getInstance(XMLTOOLING_LOGCAT"libcurl.NetAccessor").error(
227 "curl NetAccessor encountered error from libcurl (%d)", msg->data.result
229 ThrowXMLwithMemMgr1(NetAccessorException, XMLExcepts::NetAcc_InternalError, fURLSource.getURLText(), fMemoryManager);
234 // If nothing is running any longer, bail out
235 if (runningHandles == 0)
238 // If there is no further data to read, and we haven't
239 // read any yet on this invocation, call select to wait for data
240 if (!tryAgain && fBytesRead == 0)
247 // Ask curl for the file descriptors to wait on
251 (void) curl_multi_fdset(fMulti, &readSet, &writeSet, &exceptSet, &fdcnt);
253 // Wait on the file descriptors
257 (void) select(fdcnt+1, &readSet, &writeSet, &exceptSet, &tv);