https://issues.shibboleth.net/jira/browse/CPPOST-47
[shibboleth/cpp-xmltooling.git] / xmltooling / util / ReloadableXMLFile.cpp
1 /*
2  *  Copyright 2001-2010 Internet2
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *     http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16
17 /**
18  * @file ReloadableXMLFile.cpp
19  *
20  * Base class for file-based XML configuration.
21  */
22
23 #include "internal.h"
24 #include "io/HTTPResponse.h"
25 #ifndef XMLTOOLING_LITE
26 # include "security/Credential.h"
27 # include "security/CredentialCriteria.h"
28 # include "security/CredentialResolver.h"
29 # include "security/SignatureTrustEngine.h"
30 # include "signature/Signature.h"
31 # include "signature/SignatureValidator.h"
32 #endif
33 #include "util/NDC.h"
34 #include "util/PathResolver.h"
35 #include "util/ReloadableXMLFile.h"
36 #include "util/Threads.h"
37 #include "util/XMLConstants.h"
38 #include "util/XMLHelper.h"
39
40 #if defined(XMLTOOLING_LOG4SHIB)
41 # include <log4shib/NDC.hh>
42 #elif defined(XMLTOOLING_LOG4CPP)
43 # include <log4cpp/NDC.hh>
44 #endif
45
46 #include <memory>
47 #include <fstream>
48 #include <sys/types.h>
49 #include <sys/stat.h>
50
51 #include <xercesc/framework/LocalFileInputSource.hpp>
52 #include <xercesc/framework/Wrapper4InputSource.hpp>
53 #include <xercesc/util/XMLUniDefs.hpp>
54
55 #ifndef XMLTOOLING_LITE
56 # include <xsec/dsig/DSIGReference.hpp>
57 # include <xsec/dsig/DSIGTransformList.hpp>
58 using namespace xmlsignature;
59 #endif
60
61 using namespace xmltooling::logging;
62 using namespace xmltooling;
63 using namespace xercesc;
64 using namespace std;
65
66 #ifndef XMLTOOLING_LITE
67 namespace {
68     class XMLTOOL_DLLLOCAL DummyCredentialResolver : public CredentialResolver
69     {
70     public:
71         DummyCredentialResolver() {}
72         ~DummyCredentialResolver() {}
73
74         Lockable* lock() {return this;}
75         void unlock() {}
76
77         const Credential* resolve(const CredentialCriteria* criteria=nullptr) const {return nullptr;}
78         vector<const Credential*>::size_type resolve(
79             vector<const Credential*>& results, const CredentialCriteria* criteria=nullptr
80             ) const {return 0;}
81     };
82 };
83 #endif
84
85 static const XMLCh id[] =               UNICODE_LITERAL_2(i,d);
86 static const XMLCh uri[] =              UNICODE_LITERAL_3(u,r,i);
87 static const XMLCh url[] =              UNICODE_LITERAL_3(u,r,l);
88 static const XMLCh path[] =             UNICODE_LITERAL_4(p,a,t,h);
89 static const XMLCh pathname[] =         UNICODE_LITERAL_8(p,a,t,h,n,a,m,e);
90 static const XMLCh file[] =             UNICODE_LITERAL_4(f,i,l,e);
91 static const XMLCh filename[] =         UNICODE_LITERAL_8(f,i,l,e,n,a,m,e);
92 static const XMLCh validate[] =         UNICODE_LITERAL_8(v,a,l,i,d,a,t,e);
93 static const XMLCh reloadChanges[] =    UNICODE_LITERAL_13(r,e,l,o,a,d,C,h,a,n,g,e,s);
94 static const XMLCh reloadInterval[] =   UNICODE_LITERAL_14(r,e,l,o,a,d,I,n,t,e,r,v,a,l);
95 static const XMLCh maxRefreshDelay[] =  UNICODE_LITERAL_15(m,a,x,R,e,f,r,e,s,h,D,e,l,a,y);
96 static const XMLCh backingFilePath[] =  UNICODE_LITERAL_15(b,a,c,k,i,n,g,F,i,l,e,P,a,t,h);
97 static const XMLCh type[] =             UNICODE_LITERAL_4(t,y,p,e);
98 static const XMLCh certificate[] =      UNICODE_LITERAL_11(c,e,r,t,i,f,i,c,a,t,e);
99 static const XMLCh signerName[] =       UNICODE_LITERAL_10(s,i,g,n,e,r,N,a,m,e);
100 static const XMLCh _TrustEngine[] =     UNICODE_LITERAL_11(T,r,u,s,t,E,n,g,i,n,e);
101 static const XMLCh _CredentialResolver[] = UNICODE_LITERAL_18(C,r,e,d,e,n,t,i,a,l,R,e,s,o,l,v,e,r);
102
103
104 ReloadableXMLFile::ReloadableXMLFile(const DOMElement* e, Category& log, bool startReloadThread)
105     : m_root(e), m_local(true), m_validate(false), m_filestamp(0), m_reloadInterval(0),
106       m_lock(nullptr), m_loaded(false), m_log(log),
107 #ifndef XMLTOOLING_LITE
108       m_credResolver(nullptr), m_trust(nullptr),
109 #endif
110       m_shutdown(false), m_reload_wait(nullptr), m_reload_thread(nullptr)
111 {
112 #ifdef _DEBUG
113     NDC ndc("ReloadableXMLFile");
114 #endif
115
116     // Establish source of data...
117     const XMLCh* source=e->getAttributeNS(nullptr,uri);
118     if (!source || !*source) {
119         source=e->getAttributeNS(nullptr,url);
120         if (!source || !*source) {
121             source=e->getAttributeNS(nullptr,path);
122             if (!source || !*source) {
123                 source=e->getAttributeNS(nullptr,pathname);
124                 if (!source || !*source) {
125                     source=e->getAttributeNS(nullptr,file);
126                     if (!source || !*source) {
127                         source=e->getAttributeNS(nullptr,filename);
128                     }
129                 }
130             }
131         }
132         else {
133             m_local=false;
134         }
135     }
136     else {
137         m_local=false;
138     }
139
140     if (source && *source) {
141         const XMLCh* flag=e->getAttributeNS(nullptr,validate);
142         m_validate=(XMLString::equals(flag,xmlconstants::XML_TRUE) || XMLString::equals(flag,xmlconstants::XML_ONE));
143
144         auto_ptr_char temp(source);
145         m_source=temp.get();
146
147         if (!m_local && !strstr(m_source.c_str(),"://")) {
148             log.warn("deprecated usage of uri/url attribute for a local resource, use path instead");
149             m_local=true;
150         }
151
152 #ifndef XMLTOOLING_LITE
153         // Check for signature bits.
154         if (e && e->hasAttributeNS(nullptr, certificate)) {
155             // Use a file-based credential resolver rooted here.
156             m_credResolver = XMLToolingConfig::getConfig().CredentialResolverManager.newPlugin(FILESYSTEM_CREDENTIAL_RESOLVER, e);
157         }
158         else {
159             const DOMElement* sub = e ? XMLHelper::getFirstChildElement(e, _CredentialResolver) : nullptr;
160             auto_ptr_char t(sub ? sub->getAttributeNS(nullptr, type) : nullptr);
161             if (t.get()) {
162                 m_credResolver = XMLToolingConfig::getConfig().CredentialResolverManager.newPlugin(t.get(), sub);
163             }
164             else {
165                 sub = e ? XMLHelper::getFirstChildElement(e, _TrustEngine) : nullptr;
166                 auto_ptr_char t2(sub ? sub->getAttributeNS(nullptr, type) : nullptr);
167                 if (t2.get()) {
168                     TrustEngine* trust = XMLToolingConfig::getConfig().TrustEngineManager.newPlugin(t2.get(), sub);
169                     if (!(m_trust = dynamic_cast<SignatureTrustEngine*>(trust))) {
170                         delete trust;
171                         throw XMLToolingException("TrustEngine-based ReloadableXMLFile requires a SignatureTrustEngine plugin.");
172                     }
173
174                     flag = e->getAttributeNS(nullptr, signerName);
175                     if (flag && *flag) {
176                         auto_ptr_char sn(flag);
177                         m_signerName = sn.get();
178                     }
179                 }
180             }
181         }
182 #endif
183
184         if (m_local) {
185             XMLToolingConfig::getConfig().getPathResolver()->resolve(m_source, PathResolver::XMLTOOLING_CFG_FILE);
186
187             flag=e->getAttributeNS(nullptr,reloadChanges);
188             if (!XMLString::equals(flag,xmlconstants::XML_FALSE) && !XMLString::equals(flag,xmlconstants::XML_ZERO)) {
189 #ifdef WIN32
190                 struct _stat stat_buf;
191                 if (_stat(m_source.c_str(), &stat_buf) == 0)
192 #else
193                 struct stat stat_buf;
194                 if (stat(m_source.c_str(), &stat_buf) == 0)
195 #endif
196                     m_filestamp=stat_buf.st_mtime;
197                 else
198                     throw IOException("Unable to access local file ($1)", params(1,m_source.c_str()));
199                 m_lock=RWLock::create();
200             }
201             log.debug("using local resource (%s), will %smonitor for changes", m_source.c_str(), m_lock ? "" : "not ");
202         }
203         else {
204             log.debug("using remote resource (%s)", m_source.c_str());
205             source = e->getAttributeNS(nullptr,backingFilePath);
206             if (source && *source) {
207                 auto_ptr_char temp2(source);
208                 m_backing=temp2.get();
209                 XMLToolingConfig::getConfig().getPathResolver()->resolve(m_backing, PathResolver::XMLTOOLING_RUN_FILE);
210                 log.debug("backup remote resource to (%s)", m_backing.c_str());
211                 try {
212                     ifstream backer(m_backing + ".tag");
213                     if (backer) {
214                         char cachebuf[256];
215                         if (backer.getline(cachebuf, 255)) {
216                             m_cacheTag = cachebuf;
217                             log.debug("loaded initial cache tag (%s)", m_cacheTag.c_str());
218                         }
219                     }
220                 }
221                 catch (exception&) {
222                 }
223             }
224             source = e->getAttributeNS(nullptr,reloadInterval);
225             if (!source || !*source)
226                 source = e->getAttributeNS(nullptr,maxRefreshDelay);
227             if (source && *source) {
228                 m_reloadInterval = XMLString::parseInt(source);
229                 if (m_reloadInterval > 0) {
230                     m_log.debug("will reload remote resource at most every %d seconds", m_reloadInterval);
231                     m_lock=RWLock::create();
232                 }
233             }
234             m_filestamp = time(nullptr);   // assume it gets loaded initially
235         }
236
237         if (startReloadThread)
238             startup();
239     }
240     else {
241         log.debug("no resource uri/path/name supplied, will load inline configuration");
242     }
243
244     source = e->getAttributeNS(nullptr, id);
245     if (source && *source) {
246         auto_ptr_char tempid(source);
247         m_id = tempid.get();
248     }
249 }
250
251 ReloadableXMLFile::~ReloadableXMLFile()
252 {
253     shutdown();
254     delete m_lock;
255 }
256
257 void ReloadableXMLFile::startup()
258 {
259     if (m_lock && !m_reload_thread) {
260         m_reload_wait = CondWait::create();
261         m_reload_thread = Thread::create(&reload_fn, this);
262     }
263 }
264
265 void ReloadableXMLFile::shutdown()
266 {
267     if (m_reload_thread) {
268         // Shut down the reload thread and let it know.
269         m_shutdown = true;
270         m_reload_wait->signal();
271         m_reload_thread->join(nullptr);
272         delete m_reload_thread;
273         delete m_reload_wait;
274         m_reload_thread = nullptr;
275         m_reload_wait = nullptr;
276     }
277 }
278
279 void* ReloadableXMLFile::reload_fn(void* pv)
280 {
281     ReloadableXMLFile* r = reinterpret_cast<ReloadableXMLFile*>(pv);
282
283 #ifndef WIN32
284     // First, let's block all signals
285     Thread::mask_all_signals();
286 #endif
287
288     if (!r->m_id.empty()) {
289         string threadid("[");
290         threadid += r->m_id + ']';
291         logging::NDC::push(threadid);
292     }
293
294 #ifdef _DEBUG
295     NDC ndc("reload");
296 #endif
297
298     auto_ptr<Mutex> mutex(Mutex::create());
299     mutex->lock();
300
301     if (r->m_local)
302         r->m_log.info("reload thread started...running when signaled");
303     else
304         r->m_log.info("reload thread started...running every %d seconds", r->m_reloadInterval);
305
306     while (!r->m_shutdown) {
307         if (r->m_local)
308             r->m_reload_wait->wait(mutex.get());
309         else
310             r->m_reload_wait->timedwait(mutex.get(), r->m_reloadInterval);
311         if (r->m_shutdown)
312             break;
313
314         try {
315             r->m_log.info("reloading %s resource...", r->m_local ? "local" : "remote");
316             pair<bool,DOMElement*> ret = r->background_load();
317             if (ret.first)
318                 ret.second->getOwnerDocument()->release();
319         }
320         catch (long& ex) {
321             if (ex == HTTPResponse::XMLTOOLING_HTTP_STATUS_NOTMODIFIED) {
322                 r->m_log.info("remote resource (%s) unchanged from cached version", r->m_source.c_str());
323             }
324             else {
325                 // Shouldn't happen, we should only get codes intended to be gracefully handled.
326                 r->m_log.crit("maintaining existing configuration, remote resource fetch returned atypical status code (%d)", ex);
327             }
328         }
329         catch (exception& ex) {
330             r->m_log.crit("maintaining existing configuration, error reloading resource (%s): %s", r->m_source.c_str(), ex.what());
331         }
332     }
333
334     r->m_log.info("reload thread finished");
335
336     mutex->unlock();
337
338     if (!r->m_id.empty()) {
339         logging::NDC::pop();
340     }
341
342     return nullptr;
343 }
344
345 Lockable* ReloadableXMLFile::lock()
346 {
347     if (!m_lock)
348         return this;
349
350     m_lock->rdlock();
351
352     if (m_local) {
353     // Check if we need to refresh.
354 #ifdef WIN32
355         struct _stat stat_buf;
356         if (_stat(m_source.c_str(), &stat_buf) != 0)
357             return this;
358 #else
359         struct stat stat_buf;
360         if (stat(m_source.c_str(), &stat_buf) != 0)
361             return this;
362 #endif
363         if (m_filestamp >= stat_buf.st_mtime)
364             return this;
365
366         // Elevate lock and recheck.
367         m_log.debug("timestamp of local resource changed, elevating to a write lock");
368         m_lock->unlock();
369         m_lock->wrlock();
370         if (m_filestamp >= stat_buf.st_mtime) {
371             // Somebody else handled it, just downgrade.
372             m_log.debug("update of local resource handled by another thread, downgrading lock");
373             m_lock->unlock();
374             m_lock->rdlock();
375             return this;
376         }
377
378         // Update the timestamp regardless.
379         m_filestamp = stat_buf.st_mtime;
380         m_log.info("change detected, signaling reload thread...");
381         m_reload_wait->signal();
382     }
383
384     return this;
385 }
386
387 void ReloadableXMLFile::unlock()
388 {
389     if (m_lock)
390         m_lock->unlock();
391 }
392
393 pair<bool,DOMElement*> ReloadableXMLFile::load(bool backup)
394 {
395 #ifdef _DEBUG
396     NDC ndc("load");
397 #endif
398
399     try {
400         if (m_source.empty()) {
401             // Data comes from the DOM we were handed.
402             m_log.debug("loading inline configuration...");
403             return make_pair(false, XMLHelper::getFirstChildElement(m_root));
404         }
405         else {
406             // Data comes from a file we have to parse.
407             if (backup)
408                 m_log.warn("using local backup of remote resource");
409             else
410                 m_log.debug("loading configuration from external resource...");
411
412             DOMDocument* doc=nullptr;
413             if (m_local || backup) {
414                 auto_ptr_XMLCh widenit(backup ? m_backing.c_str() : m_source.c_str());
415                 // Use library-wide lock for now, nothing else is using it anyway.
416                 Locker locker(backup ? getBackupLock() : nullptr);
417                 LocalFileInputSource src(widenit.get());
418                 Wrapper4InputSource dsrc(&src, false);
419                 if (m_validate)
420                     doc=XMLToolingConfig::getConfig().getValidatingParser().parse(dsrc);
421                 else
422                     doc=XMLToolingConfig::getConfig().getParser().parse(dsrc);
423             }
424             else {
425                 URLInputSource src(m_root, nullptr, &m_cacheTag);
426                 Wrapper4InputSource dsrc(&src, false);
427                 if (m_validate)
428                     doc=XMLToolingConfig::getConfig().getValidatingParser().parse(dsrc);
429                 else
430                     doc=XMLToolingConfig::getConfig().getParser().parse(dsrc);
431
432                 // Check for a response code signal.
433                 if (XMLHelper::isNodeNamed(doc->getDocumentElement(), xmlconstants::XMLTOOLING_NS, URLInputSource::utf16StatusCodeElementName)) {
434                     int responseCode = XMLString::parseInt(doc->getDocumentElement()->getFirstChild()->getNodeValue());
435                     doc->release();
436                     if (responseCode == HTTPResponse::XMLTOOLING_HTTP_STATUS_NOTMODIFIED)
437                         throw (long)responseCode; // toss out as a "known" case to handle gracefully
438                     else {
439                         m_log.warn("remote resource fetch returned atypical status code (%d)", responseCode);
440                         throw IOException("remote resource fetch failed, check log for status code of response");
441                     }
442                 }
443             }
444
445             m_log.infoStream() << "loaded XML resource (" << (backup ? m_backing : m_source) << ")" << logging::eol;
446 #ifndef XMLTOOLING_LITE
447             if (m_credResolver || m_trust) {
448                 m_log.debug("checking signature on XML resource");
449                 try {
450                     DOMElement* sigel = XMLHelper::getFirstChildElement(doc->getDocumentElement(), xmlconstants::XMLSIG_NS, Signature::LOCAL_NAME);
451                     if (!sigel)
452                         throw XMLSecurityException("Signature validation required, but no signature found.");
453
454                     // Wrap and unmarshall the signature for the duration of the check.
455                     auto_ptr<Signature> sigobj(dynamic_cast<Signature*>(SignatureBuilder::buildOneFromElement(sigel)));    // don't bind to document
456                     validateSignature(*sigobj.get());
457                 }
458                 catch (exception&) {
459                     doc->release();
460                     throw;
461                 }
462
463             }
464 #endif
465             return make_pair(true, doc->getDocumentElement());
466         }
467     }
468     catch (XMLException& e) {
469         auto_ptr_char msg(e.getMessage());
470         m_log.errorStream() << "Xerces error while loading resource (" << (backup ? m_backing : m_source) << "): "
471             << msg.get() << logging::eol;
472         throw XMLParserException(msg.get());
473     }
474     catch (exception& e) {
475         m_log.errorStream() << "error while loading resource ("
476             << (m_source.empty() ? "inline" : (backup ? m_backing : m_source)) << "): " << e.what() << logging::eol;
477         throw;
478     }
479 }
480
481 pair<bool,DOMElement*> ReloadableXMLFile::load()
482 {
483     // If this method is used, we're responsible for managing failover to a
484     // backup of a remote resource (if available), and for backing up remote
485     // resources.
486     try {
487         pair<bool,DOMElement*> ret = load(false);
488         if (!m_backing.empty()) {
489             m_log.debug("backing up remote resource to (%s)", m_backing.c_str());
490             try {
491                 Locker locker(getBackupLock());
492                 ofstream backer(m_backing);
493                 backer << *(ret.second->getOwnerDocument());
494                 preserveCacheTag();
495             }
496             catch (exception& ex) {
497                 m_log.crit("exception while backing up resource: %s", ex.what());
498             }
499         }
500         return ret;
501     }
502     catch (long& responseCode) {
503         // If there's an HTTP error or the document hasn't changed,
504         // use the backup iff we have no "valid" resource in place.
505         // That prevents reload of the backup copy any time the document
506         // hasn't changed.
507         if (responseCode == HTTPResponse::XMLTOOLING_HTTP_STATUS_NOTMODIFIED)
508             m_log.info("remote resource (%s) unchanged from cached version", m_source.c_str());
509         if (!m_loaded && !m_backing.empty())
510             return load(true);
511         throw;
512     }
513     catch (exception&) {
514         // Same as above, but for general load/parse errors.
515         if (!m_loaded && !m_backing.empty())
516             return load(true);
517         throw;
518     }
519 }
520
521 pair<bool,DOMElement*> ReloadableXMLFile::background_load()
522 {
523     // If this method isn't overridden, we acquire a write lock
524     // and just call the old override.
525     if (m_lock)
526         m_lock->wrlock();
527     SharedLock locker(m_lock, false);
528     return load();
529 }
530
531 Lockable* ReloadableXMLFile::getBackupLock()
532 {
533     return &XMLToolingConfig::getConfig();
534 }
535
536 void ReloadableXMLFile::preserveCacheTag()
537 {
538     if (!m_cacheTag.empty() && !m_backing.empty()) {
539         try {
540             ofstream backer(m_backing + ".tag");
541             backer << m_cacheTag;
542         }
543         catch (exception&) {
544         }
545     }
546 }
547
548 #ifndef XMLTOOLING_LITE
549
550 void ReloadableXMLFile::validateSignature(Signature& sigObj) const
551 {
552     DSIGSignature* sig=sigObj.getXMLSignature();
553     if (!sig)
554         throw XMLSecurityException("Signature does not exist yet.");
555
556     // Make sure the whole document was signed.
557     bool valid=false;
558     DSIGReferenceList* refs=sig->getReferenceList();
559     if (refs && refs->getSize()==1) {
560         DSIGReference* ref=refs->item(0);
561         if (ref) {
562             const XMLCh* URI=ref->getURI();
563             if (URI==nullptr || *URI==0) {
564                 DSIGTransformList* tlist=ref->getTransforms();
565                 if (tlist->getSize() <= 2) { 
566                     for (unsigned int i=0; tlist && i<tlist->getSize(); i++) {
567                         if (tlist->item(i)->getTransformType()==TRANSFORM_ENVELOPED_SIGNATURE)
568                             valid=true;
569                         else if (tlist->item(i)->getTransformType()!=TRANSFORM_EXC_C14N &&
570                                  tlist->item(i)->getTransformType()!=TRANSFORM_C14N &&
571                                  tlist->item(i)->getTransformType()!=TRANSFORM_C14N11) {
572                             valid=false;
573                             break;
574                         }
575                     }
576                 }
577             }
578         }
579     }
580     
581     if (!valid)
582         throw XMLSecurityException("Invalid signature profile for signed configuration resource.");
583
584     // Set up criteria.
585     CredentialCriteria cc;
586     cc.setUsage(Credential::SIGNING_CREDENTIAL);
587     cc.setSignature(sigObj, CredentialCriteria::KEYINFO_EXTRACTION_KEY);
588     if (!m_signerName.empty())
589         cc.setPeerName(m_signerName.c_str());
590
591     if (m_credResolver) {
592         Locker locker(m_credResolver);
593         vector<const Credential*> creds;
594         if (m_credResolver->resolve(creds, &cc)) {
595             SignatureValidator sigValidator;
596             for (vector<const Credential*>::const_iterator i = creds.begin(); i != creds.end(); ++i) {
597                 try {
598                     sigValidator.setCredential(*i);
599                     sigValidator.validate(&sigObj);
600                     return; // success!
601                 }
602                 catch (exception&) {
603                 }
604             }
605             throw XMLSecurityException("Unable to verify signature with supplied key(s).");
606         }
607         else {
608             throw XMLSecurityException("CredentialResolver did not supply any candidate keys.");
609         }
610     }
611     else if (m_trust) {
612         DummyCredentialResolver dummy;
613         if (m_trust->validate(sigObj, dummy, &cc))
614             return;
615         throw XMLSecurityException("TrustEngine unable to verify signature.");
616     }
617
618     throw XMLSecurityException("Unable to verify signature.");
619 }
620
621 #endif