Merge branch '1.x' of ssh://authdev.it.ohio-state.edu/~scantor/git/cpp-xmltooling...
[shibboleth/cpp-xmltooling.git] / xmltooling / util / ReloadableXMLFile.cpp
1 /**
2  * Licensed to the University Corporation for Advanced Internet
3  * Development, Inc. (UCAID) under one or more contributor license
4  * agreements. See the NOTICE file distributed with this work for
5  * additional information regarding copyright ownership.
6  *
7  * UCAID licenses this file to you under the Apache License,
8  * Version 2.0 (the "License"); you may not use this file except
9  * in compliance with the License. You may obtain a copy of the
10  * License at
11  *
12  * http://www.apache.org/licenses/LICENSE-2.0
13  *
14  * Unless required by applicable law or agreed to in writing,
15  * software distributed under the License is distributed on an
16  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
17  * either express or implied. See the License for the specific
18  * language governing permissions and limitations under the License.
19  */
20
21 /**
22  * @file ReloadableXMLFile.cpp
23  *
24  * Base class for file-based XML configuration.
25  */
26
27 #include "internal.h"
28 #include "io/HTTPResponse.h"
29 #ifndef XMLTOOLING_LITE
30 # include "security/Credential.h"
31 # include "security/CredentialCriteria.h"
32 # include "security/CredentialResolver.h"
33 # include "security/SignatureTrustEngine.h"
34 # include "signature/Signature.h"
35 # include "signature/SignatureValidator.h"
36 #endif
37 #include "util/NDC.h"
38 #include "util/PathResolver.h"
39 #include "util/ReloadableXMLFile.h"
40 #include "util/Threads.h"
41 #include "util/XMLConstants.h"
42 #include "util/XMLHelper.h"
43
44 #if defined(XMLTOOLING_LOG4SHIB)
45 # include <log4shib/NDC.hh>
46 #elif defined(XMLTOOLING_LOG4CPP)
47 # include <log4cpp/NDC.hh>
48 #endif
49
50 #include <memory>
51 #include <fstream>
52 #include <sys/types.h>
53 #include <sys/stat.h>
54
55 #include <xercesc/framework/LocalFileInputSource.hpp>
56 #include <xercesc/framework/Wrapper4InputSource.hpp>
57 #include <xercesc/util/XMLUniDefs.hpp>
58
59 #ifndef XMLTOOLING_LITE
60 # include <xsec/dsig/DSIGReference.hpp>
61 # include <xsec/dsig/DSIGTransformList.hpp>
62 using namespace xmlsignature;
63 #endif
64
65 using namespace xmltooling::logging;
66 using namespace xmltooling;
67 using namespace xercesc;
68 using namespace std;
69
70 #ifndef XMLTOOLING_LITE
71 namespace {
72     class XMLTOOL_DLLLOCAL DummyCredentialResolver : public CredentialResolver
73     {
74     public:
75         DummyCredentialResolver() {}
76         ~DummyCredentialResolver() {}
77
78         Lockable* lock() {return this;}
79         void unlock() {}
80
81         const Credential* resolve(const CredentialCriteria* criteria=nullptr) const {return nullptr;}
82         vector<const Credential*>::size_type resolve(
83             vector<const Credential*>& results, const CredentialCriteria* criteria=nullptr
84             ) const {return 0;}
85     };
86 };
87 #endif
88
89 static const XMLCh id[] =               UNICODE_LITERAL_2(i,d);
90 static const XMLCh uri[] =              UNICODE_LITERAL_3(u,r,i);
91 static const XMLCh url[] =              UNICODE_LITERAL_3(u,r,l);
92 static const XMLCh path[] =             UNICODE_LITERAL_4(p,a,t,h);
93 static const XMLCh pathname[] =         UNICODE_LITERAL_8(p,a,t,h,n,a,m,e);
94 static const XMLCh file[] =             UNICODE_LITERAL_4(f,i,l,e);
95 static const XMLCh filename[] =         UNICODE_LITERAL_8(f,i,l,e,n,a,m,e);
96 static const XMLCh validate[] =         UNICODE_LITERAL_8(v,a,l,i,d,a,t,e);
97 static const XMLCh reloadChanges[] =    UNICODE_LITERAL_13(r,e,l,o,a,d,C,h,a,n,g,e,s);
98 static const XMLCh reloadInterval[] =   UNICODE_LITERAL_14(r,e,l,o,a,d,I,n,t,e,r,v,a,l);
99 static const XMLCh maxRefreshDelay[] =  UNICODE_LITERAL_15(m,a,x,R,e,f,r,e,s,h,D,e,l,a,y);
100 static const XMLCh backingFilePath[] =  UNICODE_LITERAL_15(b,a,c,k,i,n,g,F,i,l,e,P,a,t,h);
101 static const XMLCh type[] =             UNICODE_LITERAL_4(t,y,p,e);
102 static const XMLCh certificate[] =      UNICODE_LITERAL_11(c,e,r,t,i,f,i,c,a,t,e);
103 static const XMLCh signerName[] =       UNICODE_LITERAL_10(s,i,g,n,e,r,N,a,m,e);
104 static const XMLCh _TrustEngine[] =     UNICODE_LITERAL_11(T,r,u,s,t,E,n,g,i,n,e);
105 static const XMLCh _CredentialResolver[] = UNICODE_LITERAL_18(C,r,e,d,e,n,t,i,a,l,R,e,s,o,l,v,e,r);
106
107
108 ReloadableXMLFile::ReloadableXMLFile(const DOMElement* e, Category& log, bool startReloadThread)
109     : m_root(e), m_local(true), m_validate(false), m_filestamp(0), m_reloadInterval(0),
110       m_lock(nullptr), m_log(log), m_loaded(false),
111 #ifndef XMLTOOLING_LITE
112       m_credResolver(nullptr), m_trust(nullptr),
113 #endif
114       m_shutdown(false), m_reload_wait(nullptr), m_reload_thread(nullptr)
115 {
116 #ifdef _DEBUG
117     NDC ndc("ReloadableXMLFile");
118 #endif
119
120     // Establish source of data...
121     const XMLCh* source=e->getAttributeNS(nullptr,uri);
122     if (!source || !*source) {
123         source=e->getAttributeNS(nullptr,url);
124         if (!source || !*source) {
125             source=e->getAttributeNS(nullptr,path);
126             if (!source || !*source) {
127                 source=e->getAttributeNS(nullptr,pathname);
128                 if (!source || !*source) {
129                     source=e->getAttributeNS(nullptr,file);
130                     if (!source || !*source) {
131                         source=e->getAttributeNS(nullptr,filename);
132                     }
133                 }
134             }
135         }
136         else {
137             m_local=false;
138         }
139     }
140     else {
141         m_local=false;
142     }
143
144     if (source && *source) {
145         m_validate = XMLHelper::getAttrBool(e, false, validate);
146
147         auto_ptr_char temp(source);
148         m_source = temp.get();
149
150         if (!m_local && !strstr(m_source.c_str(),"://")) {
151             log.warn("deprecated usage of uri/url attribute for a local resource, use path instead");
152             m_local = true;
153         }
154
155 #ifndef XMLTOOLING_LITE
156         // Check for signature bits.
157         if (e->hasAttributeNS(nullptr, certificate)) {
158             // Use a file-based credential resolver rooted here.
159             m_credResolver = XMLToolingConfig::getConfig().CredentialResolverManager.newPlugin(FILESYSTEM_CREDENTIAL_RESOLVER, e);
160         }
161         else {
162             const DOMElement* sub = XMLHelper::getFirstChildElement(e, _CredentialResolver);
163             string t(XMLHelper::getAttrString(sub, nullptr, type));
164             if (!t.empty()) {
165                 m_credResolver = XMLToolingConfig::getConfig().CredentialResolverManager.newPlugin(t.c_str(), sub);
166             }
167             else {
168                 sub = XMLHelper::getFirstChildElement(e, _TrustEngine);
169                 t = XMLHelper::getAttrString(sub, nullptr, type);
170                 if (!t.empty()) {
171                     TrustEngine* trust = XMLToolingConfig::getConfig().TrustEngineManager.newPlugin(t.c_str(), sub);
172                     if (!(m_trust = dynamic_cast<SignatureTrustEngine*>(trust))) {
173                         delete trust;
174                         throw XMLToolingException("TrustEngine-based ReloadableXMLFile requires a SignatureTrustEngine plugin.");
175                     }
176
177                     m_signerName = XMLHelper::getAttrString(e, nullptr, signerName);
178                 }
179             }
180         }
181 #endif
182
183         if (m_local) {
184             XMLToolingConfig::getConfig().getPathResolver()->resolve(m_source, PathResolver::XMLTOOLING_CFG_FILE);
185
186             bool flag = XMLHelper::getAttrBool(e, true, reloadChanges);
187             if (flag) {
188 #ifdef WIN32
189                 struct _stat stat_buf;
190                 if (_stat(m_source.c_str(), &stat_buf) == 0)
191 #else
192                 struct stat stat_buf;
193                 if (stat(m_source.c_str(), &stat_buf) == 0)
194 #endif
195                     m_filestamp = stat_buf.st_mtime;
196                 else
197                     throw IOException("Unable to access local file ($1)", params(1,m_source.c_str()));
198                 m_lock = RWLock::create();
199             }
200             FILE* cfile = fopen(m_source.c_str(), "r");
201             if (cfile)
202                 fclose(cfile);
203             else
204                 throw IOException("Unable to access local file ($1)", params(1,m_source.c_str()));
205             log.debug("using local resource (%s), will %smonitor for changes", m_source.c_str(), m_lock ? "" : "not ");
206         }
207         else {
208             log.debug("using remote resource (%s)", m_source.c_str());
209             m_backing = XMLHelper::getAttrString(e, nullptr, backingFilePath);
210             if (!m_backing.empty()) {
211                 XMLToolingConfig::getConfig().getPathResolver()->resolve(m_backing, PathResolver::XMLTOOLING_RUN_FILE);
212                 log.debug("backup remote resource to (%s)", m_backing.c_str());
213                 try {
214                     string tagname = m_backing + ".tag";
215                     ifstream backer(tagname.c_str());
216                     if (backer) {
217                         char cachebuf[256];
218                         if (backer.getline(cachebuf, 255)) {
219                             m_cacheTag = cachebuf;
220                             log.debug("loaded initial cache tag (%s)", m_cacheTag.c_str());
221                         }
222                     }
223                 }
224                 catch (exception&) {
225                 }
226             }
227             m_reloadInterval = XMLHelper::getAttrInt(e, 0, reloadInterval);
228             if (m_reloadInterval == 0)
229                 m_reloadInterval = XMLHelper::getAttrInt(e, 0, maxRefreshDelay);
230             if (m_reloadInterval > 0) {
231                 m_log.debug("will reload remote resource at most every %d seconds", m_reloadInterval);
232                 m_lock = RWLock::create();
233             }
234             m_filestamp = time(nullptr);   // assume it gets loaded initially
235         }
236
237         if (startReloadThread)
238             startup();
239     }
240     else {
241         log.debug("no resource uri/path/name supplied, will load inline configuration");
242     }
243
244     m_id = XMLHelper::getAttrString(e, nullptr, id);
245 }
246
247 ReloadableXMLFile::~ReloadableXMLFile()
248 {
249     shutdown();
250     delete m_lock;
251 }
252
253 void ReloadableXMLFile::startup()
254 {
255     if (m_lock && !m_reload_thread) {
256         m_reload_wait = CondWait::create();
257         m_reload_thread = Thread::create(&reload_fn, this);
258     }
259 }
260
261 void ReloadableXMLFile::shutdown()
262 {
263     if (m_reload_thread) {
264         // Shut down the reload thread and let it know.
265         m_shutdown = true;
266         m_reload_wait->signal();
267         m_reload_thread->join(nullptr);
268         delete m_reload_thread;
269         delete m_reload_wait;
270         m_reload_thread = nullptr;
271         m_reload_wait = nullptr;
272     }
273 }
274
275 void* ReloadableXMLFile::reload_fn(void* pv)
276 {
277     ReloadableXMLFile* r = reinterpret_cast<ReloadableXMLFile*>(pv);
278
279 #ifndef WIN32
280     // First, let's block all signals
281     Thread::mask_all_signals();
282 #endif
283
284     if (!r->m_id.empty()) {
285         string threadid("[");
286         threadid += r->m_id + ']';
287         logging::NDC::push(threadid);
288     }
289
290 #ifdef _DEBUG
291     NDC ndc("reload");
292 #endif
293
294     auto_ptr<Mutex> mutex(Mutex::create());
295     mutex->lock();
296
297     if (r->m_local)
298         r->m_log.info("reload thread started...running when signaled");
299     else
300         r->m_log.info("reload thread started...running every %d seconds", r->m_reloadInterval);
301
302     while (!r->m_shutdown) {
303         if (r->m_local)
304             r->m_reload_wait->wait(mutex.get());
305         else
306             r->m_reload_wait->timedwait(mutex.get(), r->m_reloadInterval);
307         if (r->m_shutdown)
308             break;
309
310         try {
311             r->m_log.info("reloading %s resource...", r->m_local ? "local" : "remote");
312             pair<bool,DOMElement*> ret = r->background_load();
313             if (ret.first)
314                 ret.second->getOwnerDocument()->release();
315         }
316         catch (long& ex) {
317             if (ex == HTTPResponse::XMLTOOLING_HTTP_STATUS_NOTMODIFIED) {
318                 r->m_log.info("remote resource (%s) unchanged from cached version", r->m_source.c_str());
319             }
320             else {
321                 // Shouldn't happen, we should only get codes intended to be gracefully handled.
322                 r->m_log.crit("maintaining existing configuration, remote resource fetch returned atypical status code (%d)", ex);
323             }
324         }
325         catch (exception& ex) {
326             r->m_log.crit("maintaining existing configuration, error reloading resource (%s): %s", r->m_source.c_str(), ex.what());
327         }
328     }
329
330     r->m_log.info("reload thread finished");
331
332     mutex->unlock();
333
334     if (!r->m_id.empty()) {
335         logging::NDC::pop();
336     }
337
338     return nullptr;
339 }
340
341 Lockable* ReloadableXMLFile::lock()
342 {
343     if (!m_lock)
344         return this;
345
346     m_lock->rdlock();
347
348     if (m_local) {
349     // Check if we need to refresh.
350 #ifdef WIN32
351         struct _stat stat_buf;
352         if (_stat(m_source.c_str(), &stat_buf) != 0)
353             return this;
354 #else
355         struct stat stat_buf;
356         if (stat(m_source.c_str(), &stat_buf) != 0)
357             return this;
358 #endif
359         if (m_filestamp >= stat_buf.st_mtime)
360             return this;
361
362         // Elevate lock and recheck.
363         m_log.debug("timestamp of local resource changed, elevating to a write lock");
364         m_lock->unlock();
365         m_lock->wrlock();
366         if (m_filestamp >= stat_buf.st_mtime) {
367             // Somebody else handled it, just downgrade.
368             m_log.debug("update of local resource handled by another thread, downgrading lock");
369             m_lock->unlock();
370             m_lock->rdlock();
371             return this;
372         }
373
374         // Update the timestamp regardless.
375         m_filestamp = stat_buf.st_mtime;
376         if (m_reload_wait) {
377             m_log.info("change detected, signaling reload thread...");
378             m_reload_wait->signal();
379         }
380         else {
381             m_log.warn("change detected, but reload thread not started");
382         }
383     }
384
385     return this;
386 }
387
388 void ReloadableXMLFile::unlock()
389 {
390     if (m_lock)
391         m_lock->unlock();
392 }
393
394 pair<bool,DOMElement*> ReloadableXMLFile::load(bool backup)
395 {
396 #ifdef _DEBUG
397     NDC ndc("load");
398 #endif
399
400     try {
401         if (m_source.empty()) {
402             // Data comes from the DOM we were handed.
403             m_log.debug("loading inline configuration...");
404             return make_pair(false, XMLHelper::getFirstChildElement(m_root));
405         }
406         else {
407             // Data comes from a file we have to parse.
408             if (backup)
409                 m_log.info("using local backup of remote resource");
410             else
411                 m_log.debug("loading configuration from external resource...");
412
413             DOMDocument* doc=nullptr;
414             if (m_local || backup) {
415                 auto_ptr_XMLCh widenit(backup ? m_backing.c_str() : m_source.c_str());
416                 // Use library-wide lock for now, nothing else is using it anyway.
417                 Locker locker(backup ? getBackupLock() : nullptr);
418                 LocalFileInputSource src(widenit.get());
419                 Wrapper4InputSource dsrc(&src, false);
420                 if (m_validate)
421                     doc=XMLToolingConfig::getConfig().getValidatingParser().parse(dsrc);
422                 else
423                     doc=XMLToolingConfig::getConfig().getParser().parse(dsrc);
424             }
425             else {
426                 URLInputSource src(m_root, nullptr, &m_cacheTag);
427                 Wrapper4InputSource dsrc(&src, false);
428                 if (m_validate)
429                     doc=XMLToolingConfig::getConfig().getValidatingParser().parse(dsrc);
430                 else
431                     doc=XMLToolingConfig::getConfig().getParser().parse(dsrc);
432
433                 // Check for a response code signal.
434                 if (XMLHelper::isNodeNamed(doc->getDocumentElement(), xmlconstants::XMLTOOLING_NS, URLInputSource::utf16StatusCodeElementName)) {
435                     int responseCode = XMLString::parseInt(doc->getDocumentElement()->getFirstChild()->getNodeValue());
436                     doc->release();
437                     if (responseCode == HTTPResponse::XMLTOOLING_HTTP_STATUS_NOTMODIFIED)
438                         throw (long)responseCode; // toss out as a "known" case to handle gracefully
439                     else {
440                         m_log.warn("remote resource fetch returned atypical status code (%d)", responseCode);
441                         throw IOException("remote resource fetch failed, check log for status code of response");
442                     }
443                 }
444             }
445
446             m_log.infoStream() << "loaded XML resource (" << (backup ? m_backing : m_source) << ")" << logging::eol;
447 #ifndef XMLTOOLING_LITE
448             if (m_credResolver || m_trust) {
449                 m_log.debug("checking signature on XML resource");
450                 try {
451                     DOMElement* sigel = XMLHelper::getFirstChildElement(doc->getDocumentElement(), xmlconstants::XMLSIG_NS, Signature::LOCAL_NAME);
452                     if (!sigel)
453                         throw XMLSecurityException("Signature validation required, but no signature found.");
454
455                     // Wrap and unmarshall the signature for the duration of the check.
456                     auto_ptr<Signature> sigobj(dynamic_cast<Signature*>(SignatureBuilder::buildOneFromElement(sigel)));    // don't bind to document
457                     validateSignature(*sigobj.get());
458                 }
459                 catch (exception&) {
460                     doc->release();
461                     throw;
462                 }
463
464             }
465 #endif
466             return make_pair(true, doc->getDocumentElement());
467         }
468     }
469     catch (XMLException& e) {
470         auto_ptr_char msg(e.getMessage());
471         m_log.errorStream() << "Xerces error while loading resource (" << (backup ? m_backing : m_source) << "): "
472             << msg.get() << logging::eol;
473         throw XMLParserException(msg.get());
474     }
475     catch (exception& e) {
476         m_log.errorStream() << "error while loading resource ("
477             << (m_source.empty() ? "inline" : (backup ? m_backing : m_source)) << "): " << e.what() << logging::eol;
478         throw;
479     }
480 }
481
482 pair<bool,DOMElement*> ReloadableXMLFile::load()
483 {
484     // If this method is used, we're responsible for managing failover to a
485     // backup of a remote resource (if available), and for backing up remote
486     // resources.
487     try {
488         pair<bool,DOMElement*> ret = load(false);
489         if (!m_backing.empty()) {
490             m_log.debug("backing up remote resource to (%s)", m_backing.c_str());
491             try {
492                 Locker locker(getBackupLock());
493                 ofstream backer(m_backing.c_str());
494                 backer << *(ret.second->getOwnerDocument());
495                 preserveCacheTag();
496             }
497             catch (exception& ex) {
498                 m_log.crit("exception while backing up resource: %s", ex.what());
499             }
500         }
501         return ret;
502     }
503     catch (long& responseCode) {
504         // If there's an HTTP error or the document hasn't changed,
505         // use the backup iff we have no "valid" resource in place.
506         // That prevents reload of the backup copy any time the document
507         // hasn't changed.
508         if (responseCode == HTTPResponse::XMLTOOLING_HTTP_STATUS_NOTMODIFIED)
509             m_log.info("remote resource (%s) unchanged from cached version", m_source.c_str());
510         if (!m_loaded && !m_backing.empty())
511             return load(true);
512         throw;
513     }
514     catch (exception&) {
515         // Same as above, but for general load/parse errors.
516         if (!m_loaded && !m_backing.empty())
517             return load(true);
518         throw;
519     }
520 }
521
522 pair<bool,DOMElement*> ReloadableXMLFile::background_load()
523 {
524     // If this method isn't overridden, we acquire a write lock
525     // and just call the old override.
526     if (m_lock)
527         m_lock->wrlock();
528     SharedLock locker(m_lock, false);
529     return load();
530 }
531
532 Lockable* ReloadableXMLFile::getBackupLock()
533 {
534     return &XMLToolingConfig::getConfig();
535 }
536
537 void ReloadableXMLFile::preserveCacheTag()
538 {
539     if (!m_cacheTag.empty() && !m_backing.empty()) {
540         try {
541             string tagname = m_backing + ".tag";
542             ofstream backer(tagname.c_str());
543             backer << m_cacheTag;
544         }
545         catch (exception&) {
546         }
547     }
548 }
549
550 #ifndef XMLTOOLING_LITE
551
552 void ReloadableXMLFile::validateSignature(Signature& sigObj) const
553 {
554     DSIGSignature* sig=sigObj.getXMLSignature();
555     if (!sig)
556         throw XMLSecurityException("Signature does not exist yet.");
557
558     // Make sure the whole document was signed.
559     bool valid=false;
560     DSIGReferenceList* refs=sig->getReferenceList();
561     if (refs && refs->getSize()==1) {
562         DSIGReference* ref=refs->item(0);
563         if (ref) {
564             const XMLCh* URI=ref->getURI();
565             if (URI==nullptr || *URI==0) {
566                 DSIGTransformList* tlist=ref->getTransforms();
567                 if (tlist->getSize() <= 2) { 
568                     for (unsigned int i=0; tlist && i<tlist->getSize(); i++) {
569                         if (tlist->item(i)->getTransformType()==TRANSFORM_ENVELOPED_SIGNATURE)
570                             valid=true;
571                         else if (tlist->item(i)->getTransformType()!=TRANSFORM_EXC_C14N &&
572                                  tlist->item(i)->getTransformType()!=TRANSFORM_C14N
573 #ifdef XMLTOOLING_XMLSEC_C14N11
574                                  && tlist->item(i)->getTransformType()!=TRANSFORM_C14N11
575 #endif
576                                  ) {
577                             valid=false;
578                             break;
579                         }
580                     }
581                 }
582             }
583         }
584     }
585     
586     if (!valid)
587         throw XMLSecurityException("Invalid signature profile for signed configuration resource.");
588
589     // Set up criteria.
590     CredentialCriteria cc;
591     cc.setUsage(Credential::SIGNING_CREDENTIAL);
592     cc.setSignature(sigObj, CredentialCriteria::KEYINFO_EXTRACTION_KEY);
593     if (!m_signerName.empty())
594         cc.setPeerName(m_signerName.c_str());
595
596     if (m_credResolver) {
597         Locker locker(m_credResolver);
598         vector<const Credential*> creds;
599         if (m_credResolver->resolve(creds, &cc)) {
600             SignatureValidator sigValidator;
601             for (vector<const Credential*>::const_iterator i = creds.begin(); i != creds.end(); ++i) {
602                 try {
603                     sigValidator.setCredential(*i);
604                     sigValidator.validate(&sigObj);
605                     return; // success!
606                 }
607                 catch (exception&) {
608                 }
609             }
610             throw XMLSecurityException("Unable to verify signature with supplied key(s).");
611         }
612         else {
613             throw XMLSecurityException("CredentialResolver did not supply any candidate keys.");
614         }
615     }
616     else if (m_trust) {
617         DummyCredentialResolver dummy;
618         if (m_trust->validate(sigObj, dummy, &cc))
619             return;
620         throw XMLSecurityException("TrustEngine unable to verify signature.");
621     }
622
623     throw XMLSecurityException("Unable to verify signature.");
624 }
625
626 #endif