https://issues.shibboleth.net/jira/browse/SSPCPP-411
[shibboleth/cpp-xmltooling.git] / xmltooling / util / ReloadableXMLFile.cpp
1 /**
2  * Licensed to the University Corporation for Advanced Internet
3  * Development, Inc. (UCAID) under one or more contributor license
4  * agreements. See the NOTICE file distributed with this work for
5  * additional information regarding copyright ownership.
6  *
7  * UCAID licenses this file to you under the Apache License,
8  * Version 2.0 (the "License"); you may not use this file except
9  * in compliance with the License. You may obtain a copy of the
10  * License at
11  *
12  * http://www.apache.org/licenses/LICENSE-2.0
13  *
14  * Unless required by applicable law or agreed to in writing,
15  * software distributed under the License is distributed on an
16  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
17  * either express or implied. See the License for the specific
18  * language governing permissions and limitations under the License.
19  */
20
21 /**
22  * @file ReloadableXMLFile.cpp
23  *
24  * Base class for file-based XML configuration.
25  */
26
27 #include "internal.h"
28 #include "io/HTTPResponse.h"
29 #ifndef XMLTOOLING_LITE
30 # include "security/Credential.h"
31 # include "security/CredentialCriteria.h"
32 # include "security/CredentialResolver.h"
33 # include "security/SignatureTrustEngine.h"
34 # include "signature/Signature.h"
35 # include "signature/SignatureValidator.h"
36 #endif
37 #include "util/NDC.h"
38 #include "util/PathResolver.h"
39 #include "util/ReloadableXMLFile.h"
40 #include "util/Threads.h"
41 #include "util/XMLConstants.h"
42 #include "util/XMLHelper.h"
43
44 #if defined(XMLTOOLING_LOG4SHIB)
45 # include <log4shib/NDC.hh>
46 #elif defined(XMLTOOLING_LOG4CPP)
47 # include <log4cpp/NDC.hh>
48 #endif
49
50 #include <memory>
51 #include <fstream>
52 #include <sys/types.h>
53 #include <sys/stat.h>
54
55 #include <xercesc/framework/LocalFileInputSource.hpp>
56 #include <xercesc/framework/Wrapper4InputSource.hpp>
57 #include <xercesc/util/XMLUniDefs.hpp>
58
59 #ifndef XMLTOOLING_LITE
60 # include <xsec/dsig/DSIGReference.hpp>
61 # include <xsec/dsig/DSIGTransformList.hpp>
62 using namespace xmlsignature;
63 #endif
64
65 using namespace xmltooling::logging;
66 using namespace xmltooling;
67 using namespace xercesc;
68 using namespace std;
69
70 static const XMLCh id[] =               UNICODE_LITERAL_2(i,d);
71 static const XMLCh uri[] =              UNICODE_LITERAL_3(u,r,i);
72 static const XMLCh url[] =              UNICODE_LITERAL_3(u,r,l);
73 static const XMLCh path[] =             UNICODE_LITERAL_4(p,a,t,h);
74 static const XMLCh pathname[] =         UNICODE_LITERAL_8(p,a,t,h,n,a,m,e);
75 static const XMLCh file[] =             UNICODE_LITERAL_4(f,i,l,e);
76 static const XMLCh filename[] =         UNICODE_LITERAL_8(f,i,l,e,n,a,m,e);
77 static const XMLCh validate[] =         UNICODE_LITERAL_8(v,a,l,i,d,a,t,e);
78 static const XMLCh reloadChanges[] =    UNICODE_LITERAL_13(r,e,l,o,a,d,C,h,a,n,g,e,s);
79 static const XMLCh reloadInterval[] =   UNICODE_LITERAL_14(r,e,l,o,a,d,I,n,t,e,r,v,a,l);
80 static const XMLCh maxRefreshDelay[] =  UNICODE_LITERAL_15(m,a,x,R,e,f,r,e,s,h,D,e,l,a,y);
81 static const XMLCh backingFilePath[] =  UNICODE_LITERAL_15(b,a,c,k,i,n,g,F,i,l,e,P,a,t,h);
82 static const XMLCh type[] =             UNICODE_LITERAL_4(t,y,p,e);
83 static const XMLCh certificate[] =      UNICODE_LITERAL_11(c,e,r,t,i,f,i,c,a,t,e);
84 static const XMLCh signerName[] =       UNICODE_LITERAL_10(s,i,g,n,e,r,N,a,m,e);
85 static const XMLCh _TrustEngine[] =     UNICODE_LITERAL_11(T,r,u,s,t,E,n,g,i,n,e);
86 static const XMLCh _CredentialResolver[] = UNICODE_LITERAL_18(C,r,e,d,e,n,t,i,a,l,R,e,s,o,l,v,e,r);
87
88
89 ReloadableXMLFile::ReloadableXMLFile(const DOMElement* e, Category& log, bool startReloadThread)
90     : m_root(e), m_local(true), m_validate(false), m_filestamp(0), m_reloadInterval(0),
91       m_lock(nullptr), m_log(log), m_loaded(false),
92 #ifndef XMLTOOLING_LITE
93       m_credResolver(nullptr), m_trust(nullptr),
94 #endif
95       m_shutdown(false), m_reload_wait(nullptr), m_reload_thread(nullptr)
96 {
97 #ifdef _DEBUG
98     NDC ndc("ReloadableXMLFile");
99 #endif
100
101     // Establish source of data...
102     const XMLCh* source=e->getAttributeNS(nullptr,uri);
103     if (!source || !*source) {
104         source=e->getAttributeNS(nullptr,url);
105         if (!source || !*source) {
106             source=e->getAttributeNS(nullptr,path);
107             if (!source || !*source) {
108                 source=e->getAttributeNS(nullptr,pathname);
109                 if (!source || !*source) {
110                     source=e->getAttributeNS(nullptr,file);
111                     if (!source || !*source) {
112                         source=e->getAttributeNS(nullptr,filename);
113                     }
114                 }
115             }
116         }
117         else {
118             m_local=false;
119         }
120     }
121     else {
122         m_local=false;
123     }
124
125     if (source && *source) {
126         m_validate = XMLHelper::getAttrBool(e, false, validate);
127
128         auto_ptr_char temp(source);
129         m_source = temp.get();
130
131         if (!m_local && !strstr(m_source.c_str(),"://")) {
132             log.warn("deprecated usage of uri/url attribute for a local resource, use path instead");
133             m_local = true;
134         }
135
136 #ifndef XMLTOOLING_LITE
137         // Check for signature bits.
138         if (e->hasAttributeNS(nullptr, certificate)) {
139             // Use a file-based credential resolver rooted here.
140             m_credResolver = XMLToolingConfig::getConfig().CredentialResolverManager.newPlugin(FILESYSTEM_CREDENTIAL_RESOLVER, e);
141         }
142         else {
143             const DOMElement* sub = XMLHelper::getFirstChildElement(e, _CredentialResolver);
144             string t(XMLHelper::getAttrString(sub, nullptr, type));
145             if (!t.empty()) {
146                 m_credResolver = XMLToolingConfig::getConfig().CredentialResolverManager.newPlugin(t.c_str(), sub);
147             }
148             else {
149                 sub = XMLHelper::getFirstChildElement(e, _TrustEngine);
150                 t = XMLHelper::getAttrString(sub, nullptr, type);
151                 if (!t.empty()) {
152                     TrustEngine* trust = XMLToolingConfig::getConfig().TrustEngineManager.newPlugin(t.c_str(), sub);
153                     if (!(m_trust = dynamic_cast<SignatureTrustEngine*>(trust))) {
154                         delete trust;
155                         throw XMLToolingException("TrustEngine-based ReloadableXMLFile requires a SignatureTrustEngine plugin.");
156                     }
157
158                     m_signerName = XMLHelper::getAttrString(e, nullptr, signerName);
159                 }
160             }
161         }
162 #endif
163
164         if (m_local) {
165             XMLToolingConfig::getConfig().getPathResolver()->resolve(m_source, PathResolver::XMLTOOLING_CFG_FILE);
166
167             bool flag = XMLHelper::getAttrBool(e, true, reloadChanges);
168             if (flag) {
169 #ifdef WIN32
170                 struct _stat stat_buf;
171                 if (_stat(m_source.c_str(), &stat_buf) == 0)
172 #else
173                 struct stat stat_buf;
174                 if (stat(m_source.c_str(), &stat_buf) == 0)
175 #endif
176                     m_filestamp = stat_buf.st_mtime;
177                 else
178                     throw IOException("Unable to access local file ($1)", params(1,m_source.c_str()));
179                 m_lock = RWLock::create();
180             }
181             FILE* cfile = fopen(m_source.c_str(), "r");
182             if (cfile)
183                 fclose(cfile);
184             else
185                 throw IOException("Unable to access local file ($1)", params(1,m_source.c_str()));
186             log.debug("using local resource (%s), will %smonitor for changes", m_source.c_str(), m_lock ? "" : "not ");
187         }
188         else {
189             log.debug("using remote resource (%s)", m_source.c_str());
190             m_backing = XMLHelper::getAttrString(e, nullptr, backingFilePath);
191             if (!m_backing.empty()) {
192                 XMLToolingConfig::getConfig().getPathResolver()->resolve(m_backing, PathResolver::XMLTOOLING_CACHE_FILE);
193                 log.debug("backup remote resource to (%s)", m_backing.c_str());
194                 try {
195                     string tagname = m_backing + ".tag";
196                     ifstream backer(tagname.c_str());
197                     if (backer) {
198                         char cachebuf[256];
199                         if (backer.getline(cachebuf, 255)) {
200                             m_cacheTag = cachebuf;
201                             log.debug("loaded initial cache tag (%s)", m_cacheTag.c_str());
202                         }
203                     }
204                 }
205                 catch (exception&) {
206                 }
207             }
208             m_reloadInterval = XMLHelper::getAttrInt(e, 0, reloadInterval);
209             if (m_reloadInterval == 0)
210                 m_reloadInterval = XMLHelper::getAttrInt(e, 0, maxRefreshDelay);
211             if (m_reloadInterval > 0) {
212                 m_log.debug("will reload remote resource at most every %d seconds", m_reloadInterval);
213                 m_lock = RWLock::create();
214             }
215             m_filestamp = time(nullptr);   // assume it gets loaded initially
216         }
217
218         if (startReloadThread)
219             startup();
220     }
221     else {
222         log.debug("no resource uri/path/name supplied, will load inline configuration");
223     }
224
225     m_id = XMLHelper::getAttrString(e, nullptr, id);
226 }
227
228 ReloadableXMLFile::~ReloadableXMLFile()
229 {
230     shutdown();
231     delete m_lock;
232 }
233
234 void ReloadableXMLFile::startup()
235 {
236     if (m_lock && !m_reload_thread) {
237         m_reload_wait = CondWait::create();
238         m_reload_thread = Thread::create(&reload_fn, this);
239     }
240 }
241
242 void ReloadableXMLFile::shutdown()
243 {
244     if (m_reload_thread) {
245         // Shut down the reload thread and let it know.
246         m_shutdown = true;
247         m_reload_wait->signal();
248         m_reload_thread->join(nullptr);
249         delete m_reload_thread;
250         delete m_reload_wait;
251         m_reload_thread = nullptr;
252         m_reload_wait = nullptr;
253     }
254 }
255
256 void* ReloadableXMLFile::reload_fn(void* pv)
257 {
258     ReloadableXMLFile* r = reinterpret_cast<ReloadableXMLFile*>(pv);
259
260 #ifndef WIN32
261     // First, let's block all signals
262     Thread::mask_all_signals();
263 #endif
264
265     if (!r->m_id.empty()) {
266         string threadid("[");
267         threadid += r->m_id + ']';
268         logging::NDC::push(threadid);
269     }
270
271 #ifdef _DEBUG
272     NDC ndc("reload");
273 #endif
274
275     auto_ptr<Mutex> mutex(Mutex::create());
276     mutex->lock();
277
278     if (r->m_local)
279         r->m_log.info("reload thread started...running when signaled");
280     else
281         r->m_log.info("reload thread started...running every %d seconds", r->m_reloadInterval);
282
283     while (!r->m_shutdown) {
284         if (r->m_local)
285             r->m_reload_wait->wait(mutex.get());
286         else
287             r->m_reload_wait->timedwait(mutex.get(), r->m_reloadInterval);
288         if (r->m_shutdown)
289             break;
290
291         try {
292             r->m_log.info("reloading %s resource...", r->m_local ? "local" : "remote");
293             pair<bool,DOMElement*> ret = r->background_load();
294             if (ret.first)
295                 ret.second->getOwnerDocument()->release();
296         }
297         catch (long& ex) {
298             if (ex == HTTPResponse::XMLTOOLING_HTTP_STATUS_NOTMODIFIED) {
299                 r->m_log.info("remote resource (%s) unchanged from cached version", r->m_source.c_str());
300             }
301             else {
302                 // Shouldn't happen, we should only get codes intended to be gracefully handled.
303                 r->m_log.crit("maintaining existing configuration, remote resource fetch returned atypical status code (%d)", ex);
304             }
305         }
306         catch (exception& ex) {
307             r->m_log.crit("maintaining existing configuration, error reloading resource (%s): %s", r->m_source.c_str(), ex.what());
308         }
309     }
310
311     r->m_log.info("reload thread finished");
312
313     mutex->unlock();
314
315     if (!r->m_id.empty()) {
316         logging::NDC::pop();
317     }
318
319     return nullptr;
320 }
321
322 Lockable* ReloadableXMLFile::lock()
323 {
324     if (!m_lock)
325         return this;
326
327     m_lock->rdlock();
328
329     if (m_local) {
330     // Check if we need to refresh.
331 #ifdef WIN32
332         struct _stat stat_buf;
333         if (_stat(m_source.c_str(), &stat_buf) != 0)
334             return this;
335 #else
336         struct stat stat_buf;
337         if (stat(m_source.c_str(), &stat_buf) != 0)
338             return this;
339 #endif
340         if (m_filestamp >= stat_buf.st_mtime)
341             return this;
342
343         // Elevate lock and recheck.
344         m_log.debug("timestamp of local resource changed, elevating to a write lock");
345         m_lock->unlock();
346         m_lock->wrlock();
347         if (m_filestamp >= stat_buf.st_mtime) {
348             // Somebody else handled it, just downgrade.
349             m_log.debug("update of local resource handled by another thread, downgrading lock");
350             m_lock->unlock();
351             m_lock->rdlock();
352             return this;
353         }
354
355         // Update the timestamp regardless.
356         m_filestamp = stat_buf.st_mtime;
357         if (m_reload_wait) {
358             m_log.info("change detected, signaling reload thread...");
359             m_reload_wait->signal();
360         }
361         else {
362             m_log.warn("change detected, but reload thread not started");
363         }
364     }
365
366     return this;
367 }
368
369 void ReloadableXMLFile::unlock()
370 {
371     if (m_lock)
372         m_lock->unlock();
373 }
374
375 pair<bool,DOMElement*> ReloadableXMLFile::load(bool backup)
376 {
377 #ifdef _DEBUG
378     NDC ndc("load");
379 #endif
380
381     try {
382         if (m_source.empty()) {
383             // Data comes from the DOM we were handed.
384             m_log.debug("loading inline configuration...");
385             return make_pair(false, XMLHelper::getFirstChildElement(m_root));
386         }
387         else {
388             // Data comes from a file we have to parse.
389             if (backup)
390                 m_log.info("using local backup of remote resource");
391             else
392                 m_log.debug("loading configuration from external resource...");
393
394             DOMDocument* doc=nullptr;
395             if (m_local || backup) {
396                 auto_ptr_XMLCh widenit(backup ? m_backing.c_str() : m_source.c_str());
397                 // Use library-wide lock for now, nothing else is using it anyway.
398                 Locker locker(backup ? getBackupLock() : nullptr);
399                 LocalFileInputSource src(widenit.get());
400                 Wrapper4InputSource dsrc(&src, false);
401                 if (m_validate)
402                     doc=XMLToolingConfig::getConfig().getValidatingParser().parse(dsrc);
403                 else
404                     doc=XMLToolingConfig::getConfig().getParser().parse(dsrc);
405             }
406             else {
407                 URLInputSource src(m_root, nullptr, &m_cacheTag);
408                 Wrapper4InputSource dsrc(&src, false);
409                 if (m_validate)
410                     doc=XMLToolingConfig::getConfig().getValidatingParser().parse(dsrc);
411                 else
412                     doc=XMLToolingConfig::getConfig().getParser().parse(dsrc);
413
414                 // Check for a response code signal.
415                 if (XMLHelper::isNodeNamed(doc->getDocumentElement(), xmlconstants::XMLTOOLING_NS, URLInputSource::utf16StatusCodeElementName)) {
416                     int responseCode = XMLString::parseInt(doc->getDocumentElement()->getFirstChild()->getNodeValue());
417                     doc->release();
418                     if (responseCode == HTTPResponse::XMLTOOLING_HTTP_STATUS_NOTMODIFIED)
419                         throw (long)responseCode; // toss out as a "known" case to handle gracefully
420                     else {
421                         m_log.warn("remote resource fetch returned atypical status code (%d)", responseCode);
422                         throw IOException("remote resource fetch failed, check log for status code of response");
423                     }
424                 }
425             }
426
427             m_log.infoStream() << "loaded XML resource (" << (backup ? m_backing : m_source) << ")" << logging::eol;
428 #ifndef XMLTOOLING_LITE
429             if (m_credResolver || m_trust) {
430                 m_log.debug("checking signature on XML resource");
431                 try {
432                     DOMElement* sigel = XMLHelper::getFirstChildElement(doc->getDocumentElement(), xmlconstants::XMLSIG_NS, Signature::LOCAL_NAME);
433                     if (!sigel)
434                         throw XMLSecurityException("Signature validation required, but no signature found.");
435
436                     // Wrap and unmarshall the signature for the duration of the check.
437                     auto_ptr<Signature> sigobj(dynamic_cast<Signature*>(SignatureBuilder::buildOneFromElement(sigel)));    // don't bind to document
438                     validateSignature(*sigobj.get());
439                 }
440                 catch (exception&) {
441                     doc->release();
442                     throw;
443                 }
444
445             }
446 #endif
447             return make_pair(true, doc->getDocumentElement());
448         }
449     }
450     catch (XMLException& e) {
451         auto_ptr_char msg(e.getMessage());
452         m_log.errorStream() << "Xerces error while loading resource (" << (backup ? m_backing : m_source) << "): "
453             << msg.get() << logging::eol;
454         throw XMLParserException(msg.get());
455     }
456     catch (exception& e) {
457         m_log.errorStream() << "error while loading resource ("
458             << (m_source.empty() ? "inline" : (backup ? m_backing : m_source)) << "): " << e.what() << logging::eol;
459         throw;
460     }
461 }
462
463 pair<bool,DOMElement*> ReloadableXMLFile::load()
464 {
465     // If this method is used, we're responsible for managing failover to a
466     // backup of a remote resource (if available), and for backing up remote
467     // resources.
468     try {
469         pair<bool,DOMElement*> ret = load(false);
470         if (!m_backing.empty()) {
471             m_log.debug("backing up remote resource to (%s)", m_backing.c_str());
472             try {
473                 Locker locker(getBackupLock());
474                 ofstream backer(m_backing.c_str());
475                 backer << *(ret.second->getOwnerDocument());
476                 preserveCacheTag();
477             }
478             catch (exception& ex) {
479                 m_log.crit("exception while backing up resource: %s", ex.what());
480             }
481         }
482         return ret;
483     }
484     catch (long& responseCode) {
485         // If there's an HTTP error or the document hasn't changed,
486         // use the backup iff we have no "valid" resource in place.
487         // That prevents reload of the backup copy any time the document
488         // hasn't changed.
489         if (responseCode == HTTPResponse::XMLTOOLING_HTTP_STATUS_NOTMODIFIED)
490             m_log.info("remote resource (%s) unchanged from cached version", m_source.c_str());
491         if (!m_loaded && !m_backing.empty())
492             return load(true);
493         throw;
494     }
495     catch (exception&) {
496         // Same as above, but for general load/parse errors.
497         if (!m_loaded && !m_backing.empty())
498             return load(true);
499         throw;
500     }
501 }
502
503 pair<bool,DOMElement*> ReloadableXMLFile::background_load()
504 {
505     // If this method isn't overridden, we acquire a write lock
506     // and just call the old override.
507     if (m_lock)
508         m_lock->wrlock();
509     SharedLock locker(m_lock, false);
510     return load();
511 }
512
513 Lockable* ReloadableXMLFile::getBackupLock()
514 {
515     return &XMLToolingConfig::getConfig();
516 }
517
518 void ReloadableXMLFile::preserveCacheTag()
519 {
520     if (!m_cacheTag.empty() && !m_backing.empty()) {
521         try {
522             string tagname = m_backing + ".tag";
523             ofstream backer(tagname.c_str());
524             backer << m_cacheTag;
525         }
526         catch (exception&) {
527         }
528     }
529 }
530
531 #ifndef XMLTOOLING_LITE
532
533 void ReloadableXMLFile::validateSignature(Signature& sigObj) const
534 {
535     DSIGSignature* sig=sigObj.getXMLSignature();
536     if (!sig)
537         throw XMLSecurityException("Signature does not exist yet.");
538
539     // Make sure the whole document was signed.
540     bool valid=false;
541     DSIGReferenceList* refs=sig->getReferenceList();
542     if (refs && refs->getSize()==1) {
543         DSIGReference* ref=refs->item(0);
544         if (ref) {
545             const XMLCh* URI=ref->getURI();
546             if (URI==nullptr || *URI==0) {
547                 DSIGTransformList* tlist=ref->getTransforms();
548                 if (tlist->getSize() <= 2) { 
549                     for (unsigned int i=0; tlist && i<tlist->getSize(); i++) {
550                         if (tlist->item(i)->getTransformType()==TRANSFORM_ENVELOPED_SIGNATURE)
551                             valid=true;
552                         else if (tlist->item(i)->getTransformType()!=TRANSFORM_EXC_C14N &&
553                                  tlist->item(i)->getTransformType()!=TRANSFORM_C14N
554 #ifdef XMLTOOLING_XMLSEC_C14N11
555                                  && tlist->item(i)->getTransformType()!=TRANSFORM_C14N11
556 #endif
557                                  ) {
558                             valid=false;
559                             break;
560                         }
561                     }
562                 }
563             }
564         }
565     }
566     
567     if (!valid)
568         throw XMLSecurityException("Invalid signature profile for signed configuration resource.");
569
570     // Set up criteria.
571     CredentialCriteria cc;
572     cc.setUsage(Credential::SIGNING_CREDENTIAL);
573     cc.setSignature(sigObj, CredentialCriteria::KEYINFO_EXTRACTION_KEY);
574     if (!m_signerName.empty())
575         cc.setPeerName(m_signerName.c_str());
576
577     if (m_credResolver) {
578         Locker locker(m_credResolver);
579         vector<const Credential*> creds;
580         if (m_credResolver->resolve(creds, &cc)) {
581             SignatureValidator sigValidator;
582             for (vector<const Credential*>::const_iterator i = creds.begin(); i != creds.end(); ++i) {
583                 try {
584                     sigValidator.setCredential(*i);
585                     sigValidator.validate(&sigObj);
586                     return; // success!
587                 }
588                 catch (exception&) {
589                 }
590             }
591             throw XMLSecurityException("Unable to verify signature with supplied key(s).");
592         }
593         else {
594             throw XMLSecurityException("CredentialResolver did not supply any candidate keys.");
595         }
596     }
597     else if (m_trust) {
598         auto_ptr<CredentialResolver> dummy(
599             XMLToolingConfig::getConfig().CredentialResolverManager.newPlugin(DUMMY_CREDENTIAL_RESOLVER, nullptr)
600             );
601         if (m_trust->validate(sigObj, *(dummy.get()), &cc))
602             return;
603         throw XMLSecurityException("TrustEngine unable to verify signature.");
604     }
605
606     throw XMLSecurityException("Unable to verify signature.");
607 }
608
609 #endif