2 * Licensed to the Apache Software Foundation (ASF) under one or more
3 * contributor license agreements. See the NOTICE file distributed with
4 * this work for additional information regarding copyright ownership.
5 * The ASF licenses this file to You under the Apache License, Version 2.0
6 * (the "License"); you may not use this file except in compliance with
7 * the License. You may obtain a copy of the License at
9 * http://www.apache.org/licenses/LICENSE-2.0
11 * Unless required by applicable law or agreed to in writing, software
12 * distributed under the License is distributed on an "AS IS" BASIS,
13 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 * See the License for the specific language governing permissions and
15 * limitations under the License.
31 //#include <sys/types.h>
32 //#include <sys/time.h>
34 #include <xercesc/util/XercesDefs.hpp>
35 #include <xercesc/util/XMLNetAccessor.hpp>
36 #include <xercesc/util/XMLString.hpp>
37 #include <xercesc/util/XMLExceptMsgs.hpp>
38 #include <xercesc/util/Janitor.hpp>
39 #include <xercesc/util/XMLUniDefs.hpp>
40 #include <xercesc/util/TransService.hpp>
41 #include <xercesc/util/TranscodingException.hpp>
42 #include <xercesc/util/PlatformUtils.hpp>
44 #include <xmltooling/util/CurlURLInputStream.hpp>
46 using namespace xmltooling;
49 CurlURLInputStream::CurlURLInputStream(const XMLURL& urlSource, const XMLNetHTTPInfo* httpInfo/*=0*/)
52 , fMemoryManager(urlSource.getMemoryManager())
53 , fURLSource(urlSource)
59 , fDataAvailable(false)
60 , fBufferHeadPtr(fBuffer)
61 , fBufferTailPtr(fBuffer)
62 , m_log(logging::Category::getInstance(XMLTOOLING_LOGCAT".libcurl.NetAccessor"))
64 // Allocate the curl multi handle
65 fMulti = curl_multi_init();
67 // Allocate the curl easy handle
68 fEasy = curl_easy_init();
70 // Get the text of the URL we're going to use
71 fURL.reset(XMLString::transcode(fURLSource.getURLText(), fMemoryManager), fMemoryManager);
73 m_log.debug("libcurl trying to fetch %s", fURL.get());
76 curl_easy_setopt(fEasy, CURLOPT_URL, fURL.get());
77 curl_easy_setopt(fEasy, CURLOPT_WRITEDATA, this); // Pass this pointer to write function
78 curl_easy_setopt(fEasy, CURLOPT_WRITEFUNCTION, staticWriteCallback); // Our static write function
79 curl_easy_setopt(fEasy, CURLOPT_CONNECTTIMEOUT, 30);
80 curl_easy_setopt(fEasy, CURLOPT_TIMEOUT, 60);
81 curl_easy_setopt(fEasy, CURLOPT_SSL_VERIFYHOST, 0);
82 curl_easy_setopt(fEasy, CURLOPT_SSL_VERIFYPEER, 0);
83 curl_easy_setopt(fEasy, CURLOPT_NOPROGRESS, 1);
84 curl_easy_setopt(fEasy, CURLOPT_NOSIGNAL, 1);
85 curl_easy_setopt(fEasy, CURLOPT_FAILONERROR, 1);
87 // Add easy handle to the multi stack
88 curl_multi_add_handle(fMulti, fEasy);
92 CurlURLInputStream::~CurlURLInputStream()
94 // Remove the easy handle from the multi stack
95 curl_multi_remove_handle(fMulti, fEasy);
97 // Cleanup the easy handle
98 curl_easy_cleanup(fEasy);
100 // Cleanup the multi handle
101 curl_multi_cleanup(fMulti);
106 CurlURLInputStream::staticWriteCallback(char *buffer,
111 return ((CurlURLInputStream*)outstream)->writeCallback(buffer, size, nitems);
117 CurlURLInputStream::writeCallback(char *buffer,
121 size_t cnt = size * nitems;
122 size_t totalConsumed = 0;
124 // Consume as many bytes as possible immediately into the buffer
125 size_t consume = (cnt > fBytesToRead) ? fBytesToRead : cnt;
126 memcpy(fWritePtr, buffer, consume);
127 fWritePtr += consume;
128 fBytesRead += consume;
129 fTotalBytesRead += consume;
130 fBytesToRead -= consume;
132 //m_log.debug("write callback consuming %d bytes", consume);
134 // If bytes remain, rebuffer as many as possible into our holding buffer
136 totalConsumed += consume;
140 size_t bufAvail = sizeof(fBuffer) - (fBufferHeadPtr - fBuffer);
141 consume = (cnt > bufAvail) ? bufAvail : cnt;
142 memcpy(fBufferHeadPtr, buffer, consume);
143 fBufferHeadPtr += consume;
145 totalConsumed += consume;
146 //m_log.debug("write callback rebuffering %d bytes", consume);
149 // Return the total amount we've consumed. If we don't consume all the bytes
150 // then an error will be generated. Since our buffer size is equal to the
151 // maximum size that curl will write, this should never happen unless there
152 // is a logic error somewhere here.
153 return totalConsumed;
158 CurlURLInputStream::readBytes(XMLByte* const toFill
159 , const unsigned int maxToRead)
162 fBytesToRead = maxToRead;
165 for (bool tryAgain = true; fBytesToRead > 0 && (tryAgain || fBytesRead == 0); )
167 // First, any buffered data we have available
168 size_t bufCnt = fBufferHeadPtr - fBufferTailPtr;
169 bufCnt = (bufCnt > fBytesToRead) ? fBytesToRead : bufCnt;
172 memcpy(fWritePtr, fBufferTailPtr, bufCnt);
174 fBytesRead += bufCnt;
175 fTotalBytesRead += bufCnt;
176 fBytesToRead -= bufCnt;
178 fBufferTailPtr += bufCnt;
179 if (fBufferTailPtr == fBufferHeadPtr)
180 fBufferHeadPtr = fBufferTailPtr = fBuffer;
182 //m_log.debug("consuming %d buffered bytes", bufCnt);
188 // Ask the curl to do some work
189 int runningHandles = 0;
190 CURLMcode curlResult = curl_multi_perform(fMulti, &runningHandles);
191 //m_log.debug("curl_multi_perform returned %d", curlResult);
192 tryAgain = (curlResult == CURLM_CALL_MULTI_PERFORM);
194 // Process messages from curl
196 for (CURLMsg* msg = NULL; (msg = curl_multi_info_read(fMulti, &msgsInQueue)) != NULL; )
198 m_log.debug("msg %d, %d from curl", msg->msg, msg->data.result);
200 if (msg->msg != CURLMSG_DONE)
203 switch (msg->data.result)
206 // We completed successfully. runningHandles should have dropped to zero, so we'll bail out below...
209 case CURLE_UNSUPPORTED_PROTOCOL:
210 ThrowXMLwithMemMgr(MalformedURLException, XMLExcepts::URL_UnsupportedProto, fMemoryManager);
213 case CURLE_COULDNT_RESOLVE_HOST:
214 case CURLE_COULDNT_RESOLVE_PROXY:
215 ThrowXMLwithMemMgr1(NetAccessorException, XMLExcepts::NetAcc_TargetResolution, fURLSource.getHost(), fMemoryManager);
218 case CURLE_COULDNT_CONNECT:
219 ThrowXMLwithMemMgr1(NetAccessorException, XMLExcepts::NetAcc_ConnSocket, fURLSource.getURLText(), fMemoryManager);
221 case CURLE_RECV_ERROR:
222 ThrowXMLwithMemMgr1(NetAccessorException, XMLExcepts::NetAcc_ReadSocket, fURLSource.getURLText(), fMemoryManager);
226 m_log.error("curl NetAccessor encountered error from libcurl (%d)", msg->data.result);
227 ThrowXMLwithMemMgr1(NetAccessorException, XMLExcepts::NetAcc_InternalError, fURLSource.getURLText(), fMemoryManager);
232 // If nothing is running any longer, bail out
233 if (runningHandles == 0) {
234 //m_log.debug("libcurl indicated no running handles");
238 // If there is no further data to read, and we haven't
239 // read any yet on this invocation, call select to wait for data
240 if (!tryAgain && fBytesRead == 0)
247 // Ask curl for the file descriptors to wait on
251 (void) curl_multi_fdset(fMulti, &readSet, &writeSet, &exceptSet, &fdcnt);
253 // Wait on the file descriptors
257 (void) select(fdcnt+1, &readSet, &writeSet, &exceptSet, &tv);
261 //m_log.debug("returning with %d bytes to parser", fBytesRead);