xmltooling/util/ReloadableXMLFile.cpp

   1 /*
   2  *  Copyright 2001-2010 Internet2
   3  *
   4  * Licensed under the Apache License, Version 2.0 (the "License");
   5  * you may not use this file except in compliance with the License.
   6  * You may obtain a copy of the License at
   7  *
   8  *     http://www.apache.org/licenses/LICENSE-2.0
   9  *
  10  * Unless required by applicable law or agreed to in writing, software
  11  * distributed under the License is distributed on an "AS IS" BASIS,
  12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13  * See the License for the specific language governing permissions and
  14  * limitations under the License.
  15  */
  16
  17 /**
  18  * @file ReloadableXMLFile.cpp
  19  *
  20  * Base class for file-based XML configuration.
  21  */
  22
  23 #include "internal.h"
  24 #include "io/HTTPResponse.h"
  25 #ifndef XMLTOOLING_LITE
  26 # include "security/Credential.h"
  27 # include "security/CredentialCriteria.h"
  28 # include "security/CredentialResolver.h"
  29 # include "security/SignatureTrustEngine.h"
  30 # include "signature/Signature.h"
  31 # include "signature/SignatureValidator.h"
  32 #endif
  33 #include "util/NDC.h"
  34 #include "util/PathResolver.h"
  35 #include "util/ReloadableXMLFile.h"
  36 #include "util/Threads.h"
  37 #include "util/XMLConstants.h"
  38 #include "util/XMLHelper.h"
  39
  40 #if defined(XMLTOOLING_LOG4SHIB)
  41 # include <log4shib/NDC.hh>
  42 #elif defined(XMLTOOLING_LOG4CPP)
  43 # include <log4cpp/NDC.hh>
  44 #endif
  45
  46 #include <memory>
  47 #include <fstream>
  48 #include <sys/types.h>
  49 #include <sys/stat.h>
  50
  51 #include <xercesc/framework/LocalFileInputSource.hpp>
  52 #include <xercesc/framework/Wrapper4InputSource.hpp>
  53 #include <xercesc/util/XMLUniDefs.hpp>
  54
  55 #ifndef XMLTOOLING_LITE
  56 # include <xsec/dsig/DSIGReference.hpp>
  57 # include <xsec/dsig/DSIGTransformList.hpp>
  58 using namespace xmlsignature;
  59 #endif
  60
  61 using namespace xmltooling::logging;
  62 using namespace xmltooling;
  63 using namespace xercesc;
  64 using namespace std;
  65
  66 #ifndef XMLTOOLING_LITE
  67 namespace {
  68     class XMLTOOL_DLLLOCAL DummyCredentialResolver : public CredentialResolver
  69     {
  70     public:
  71         DummyCredentialResolver() {}
  72         ~DummyCredentialResolver() {}
  73
  74         Lockable* lock() {return this;}
  75         void unlock() {}
  76
  77         const Credential* resolve(const CredentialCriteria* criteria=nullptr) const {return nullptr;}
  78         vector<const Credential*>::size_type resolve(
  79             vector<const Credential*>& results, const CredentialCriteria* criteria=nullptr
  80             ) const {return 0;}
  81     };
  82 };
  83 #endif
  84
  85 static const XMLCh id[] =               UNICODE_LITERAL_2(i,d);
  86 static const XMLCh uri[] =              UNICODE_LITERAL_3(u,r,i);
  87 static const XMLCh url[] =              UNICODE_LITERAL_3(u,r,l);
  88 static const XMLCh path[] =             UNICODE_LITERAL_4(p,a,t,h);
  89 static const XMLCh pathname[] =         UNICODE_LITERAL_8(p,a,t,h,n,a,m,e);
  90 static const XMLCh file[] =             UNICODE_LITERAL_4(f,i,l,e);
  91 static const XMLCh filename[] =         UNICODE_LITERAL_8(f,i,l,e,n,a,m,e);
  92 static const XMLCh validate[] =         UNICODE_LITERAL_8(v,a,l,i,d,a,t,e);
  93 static const XMLCh reloadChanges[] =    UNICODE_LITERAL_13(r,e,l,o,a,d,C,h,a,n,g,e,s);
  94 static const XMLCh reloadInterval[] =   UNICODE_LITERAL_14(r,e,l,o,a,d,I,n,t,e,r,v,a,l);
  95 static const XMLCh maxRefreshDelay[] =  UNICODE_LITERAL_15(m,a,x,R,e,f,r,e,s,h,D,e,l,a,y);
  96 static const XMLCh backingFilePath[] =  UNICODE_LITERAL_15(b,a,c,k,i,n,g,F,i,l,e,P,a,t,h);
  97 static const XMLCh type[] =             UNICODE_LITERAL_4(t,y,p,e);
  98 static const XMLCh certificate[] =      UNICODE_LITERAL_11(c,e,r,t,i,f,i,c,a,t,e);
  99 static const XMLCh signerName[] =       UNICODE_LITERAL_10(s,i,g,n,e,r,N,a,m,e);
 100 static const XMLCh _TrustEngine[] =     UNICODE_LITERAL_11(T,r,u,s,t,E,n,g,i,n,e);
 101 static const XMLCh _CredentialResolver[] = UNICODE_LITERAL_18(C,r,e,d,e,n,t,i,a,l,R,e,s,o,l,v,e,r);
 102
 103
 104 ReloadableXMLFile::ReloadableXMLFile(const DOMElement* e, Category& log, bool startReloadThread)
 105     : m_root(e), m_local(true), m_validate(false), m_filestamp(0), m_reloadInterval(0),
 106       m_lock(nullptr), m_loaded(false), m_log(log),
 107 #ifndef XMLTOOLING_LITE
 108       m_credResolver(nullptr), m_trust(nullptr),
 109 #endif
 110       m_shutdown(false), m_reload_wait(nullptr), m_reload_thread(nullptr)
 111 {
 112 #ifdef _DEBUG
 113     NDC ndc("ReloadableXMLFile");
 114 #endif
 115
 116     // Establish source of data...
 117     const XMLCh* source=e->getAttributeNS(nullptr,uri);
 118     if (!source || !*source) {
 119         source=e->getAttributeNS(nullptr,url);
 120         if (!source || !*source) {
 121             source=e->getAttributeNS(nullptr,path);
 122             if (!source || !*source) {
 123                 source=e->getAttributeNS(nullptr,pathname);
 124                 if (!source || !*source) {
 125                     source=e->getAttributeNS(nullptr,file);
 126                     if (!source || !*source) {
 127                         source=e->getAttributeNS(nullptr,filename);
 128                     }
 129                 }
 130             }
 131         }
 132         else {
 133             m_local=false;
 134         }
 135     }
 136     else {
 137         m_local=false;
 138     }
 139
 140     if (source && *source) {
 141         const XMLCh* flag=e->getAttributeNS(nullptr,validate);
 142         m_validate=(XMLString::equals(flag,xmlconstants::XML_TRUE) || XMLString::equals(flag,xmlconstants::XML_ONE));
 143
 144         auto_ptr_char temp(source);
 145         m_source=temp.get();
 146
 147         if (!m_local && !strstr(m_source.c_str(),"://")) {
 148             log.warn("deprecated usage of uri/url attribute for a local resource, use path instead");
 149             m_local=true;
 150         }
 151
 152 #ifndef XMLTOOLING_LITE
 153         // Check for signature bits.
 154         if (e && e->hasAttributeNS(nullptr, certificate)) {
 155             // Use a file-based credential resolver rooted here.
 156             m_credResolver = XMLToolingConfig::getConfig().CredentialResolverManager.newPlugin(FILESYSTEM_CREDENTIAL_RESOLVER, e);
 157         }
 158         else {
 159             const DOMElement* sub = e ? XMLHelper::getFirstChildElement(e, _CredentialResolver) : nullptr;
 160             auto_ptr_char t(sub ? sub->getAttributeNS(nullptr, type) : nullptr);
 161             if (t.get()) {
 162                 m_credResolver = XMLToolingConfig::getConfig().CredentialResolverManager.newPlugin(t.get(), sub);
 163             }
 164             else {
 165                 sub = e ? XMLHelper::getFirstChildElement(e, _TrustEngine) : nullptr;
 166                 auto_ptr_char t2(sub ? sub->getAttributeNS(nullptr, type) : nullptr);
 167                 if (t2.get()) {
 168                     TrustEngine* trust = XMLToolingConfig::getConfig().TrustEngineManager.newPlugin(t2.get(), sub);
 169                     if (!(m_trust = dynamic_cast<SignatureTrustEngine*>(trust))) {
 170                         delete trust;
 171                         throw XMLToolingException("TrustEngine-based ReloadableXMLFile requires a SignatureTrustEngine plugin.");
 172                     }
 173
 174                     flag = e->getAttributeNS(nullptr, signerName);
 175                     if (flag && *flag) {
 176                         auto_ptr_char sn(flag);
 177                         m_signerName = sn.get();
 178                     }
 179                 }
 180             }
 181         }
 182 #endif
 183
 184         if (m_local) {
 185             XMLToolingConfig::getConfig().getPathResolver()->resolve(m_source, PathResolver::XMLTOOLING_CFG_FILE);
 186
 187             flag=e->getAttributeNS(nullptr,reloadChanges);
 188             if (!XMLString::equals(flag,xmlconstants::XML_FALSE) && !XMLString::equals(flag,xmlconstants::XML_ZERO)) {
 189 #ifdef WIN32
 190                 struct _stat stat_buf;
 191                 if (_stat(m_source.c_str(), &stat_buf) == 0)
 192 #else
 193                 struct stat stat_buf;
 194                 if (stat(m_source.c_str(), &stat_buf) == 0)
 195 #endif
 196                     m_filestamp=stat_buf.st_mtime;
 197                 else
 198                     throw IOException("Unable to access local file ($1)", params(1,m_source.c_str()));
 199                 m_lock=RWLock::create();
 200             }
 201             log.debug("using local resource (%s), will %smonitor for changes", m_source.c_str(), m_lock ? "" : "not ");
 202         }
 203         else {
 204             log.debug("using remote resource (%s)", m_source.c_str());
 205             source = e->getAttributeNS(nullptr,backingFilePath);
 206             if (source && *source) {
 207                 auto_ptr_char temp2(source);
 208                 m_backing=temp2.get();
 209                 XMLToolingConfig::getConfig().getPathResolver()->resolve(m_backing, PathResolver::XMLTOOLING_RUN_FILE);
 210                 log.debug("backup remote resource to (%s)", m_backing.c_str());
 211             }
 212             source = e->getAttributeNS(nullptr,reloadInterval);
 213             if (!source || !*source)
 214                 source = e->getAttributeNS(nullptr,maxRefreshDelay);
 215             if (source && *source) {
 216                 m_reloadInterval = XMLString::parseInt(source);
 217                 if (m_reloadInterval > 0) {
 218                     m_log.debug("will reload remote resource at most every %d seconds", m_reloadInterval);
 219                     m_lock=RWLock::create();
 220                 }
 221             }
 222             m_filestamp = time(nullptr);   // assume it gets loaded initially
 223         }
 224
 225         if (startReloadThread)
 226             startup();
 227     }
 228     else {
 229         log.debug("no resource uri/path/name supplied, will load inline configuration");
 230     }
 231
 232     source = e->getAttributeNS(nullptr, id);
 233     if (source && *source) {
 234         auto_ptr_char tempid(source);
 235         m_id = tempid.get();
 236     }
 237 }
 238
 239 ReloadableXMLFile::~ReloadableXMLFile()
 240 {
 241     shutdown();
 242     delete m_lock;
 243 }
 244
 245 void ReloadableXMLFile::startup()
 246 {
 247     if (m_lock && !m_reload_thread) {
 248         m_reload_wait = CondWait::create();
 249         m_reload_thread = Thread::create(&reload_fn, this);
 250     }
 251 }
 252
 253 void ReloadableXMLFile::shutdown()
 254 {
 255     if (m_reload_thread) {
 256         // Shut down the reload thread and let it know.
 257         m_shutdown = true;
 258         m_reload_wait->signal();
 259         m_reload_thread->join(nullptr);
 260         delete m_reload_thread;
 261         delete m_reload_wait;
 262         m_reload_thread = nullptr;
 263         m_reload_wait = nullptr;
 264     }
 265 }
 266
 267 void* ReloadableXMLFile::reload_fn(void* pv)
 268 {
 269     ReloadableXMLFile* r = reinterpret_cast<ReloadableXMLFile*>(pv);
 270
 271 #ifndef WIN32
 272     // First, let's block all signals
 273     Thread::mask_all_signals();
 274 #endif
 275
 276     if (!r->m_id.empty()) {
 277         string threadid("[");
 278         threadid += r->m_id + ']';
 279         logging::NDC::push(threadid);
 280     }
 281
 282 #ifdef _DEBUG
 283     NDC ndc("reload");
 284 #endif
 285
 286     auto_ptr<Mutex> mutex(Mutex::create());
 287     mutex->lock();
 288
 289     if (r->m_local)
 290         r->m_log.info("reload thread started...running when signaled");
 291     else
 292         r->m_log.info("reload thread started...running every %d seconds", r->m_reloadInterval);
 293
 294     while (!r->m_shutdown) {
 295         if (r->m_local)
 296             r->m_reload_wait->wait(mutex.get());
 297         else
 298             r->m_reload_wait->timedwait(mutex.get(), r->m_reloadInterval);
 299         if (r->m_shutdown)
 300             break;
 301
 302         try {
 303             r->m_log.info("reloading %s resource...", r->m_local ? "local" : "remote");
 304             pair<bool,DOMElement*> ret = r->background_load();
 305             if (ret.first)
 306                 ret.second->getOwnerDocument()->release();
 307         }
 308         catch (long& ex) {
 309             if (ex == HTTPResponse::XMLTOOLING_HTTP_STATUS_NOTMODIFIED) {
 310                 r->m_log.info("remote resource (%s) unchanged from cached version", r->m_source.c_str());
 311             }
 312             else {
 313                 // Shouldn't happen, we should only get codes intended to be gracefully handled.
 314                 r->m_log.crit("maintaining existing configuration, remote resource fetch returned atypical status code (%d)", ex);
 315             }
 316         }
 317         catch (exception& ex) {
 318             r->m_log.crit("maintaining existing configuration, error reloading resource (%s): %s", r->m_source.c_str(), ex.what());
 319         }
 320     }
 321
 322     r->m_log.info("reload thread finished");
 323
 324     mutex->unlock();
 325
 326     if (!r->m_id.empty()) {
 327         logging::NDC::pop();
 328     }
 329
 330     return nullptr;
 331 }
 332
 333 Lockable* ReloadableXMLFile::lock()
 334 {
 335     if (!m_lock)
 336         return this;
 337
 338     m_lock->rdlock();
 339
 340     if (m_local) {
 341     // Check if we need to refresh.
 342 #ifdef WIN32
 343         struct _stat stat_buf;
 344         if (_stat(m_source.c_str(), &stat_buf) != 0)
 345             return this;
 346 #else
 347         struct stat stat_buf;
 348         if (stat(m_source.c_str(), &stat_buf) != 0)
 349             return this;
 350 #endif
 351         if (m_filestamp >= stat_buf.st_mtime)
 352             return this;
 353
 354         // Elevate lock and recheck.
 355         m_log.debug("timestamp of local resource changed, elevating to a write lock");
 356         m_lock->unlock();
 357         m_lock->wrlock();
 358         if (m_filestamp >= stat_buf.st_mtime) {
 359             // Somebody else handled it, just downgrade.
 360             m_log.debug("update of local resource handled by another thread, downgrading lock");
 361             m_lock->unlock();
 362             m_lock->rdlock();
 363             return this;
 364         }
 365
 366         // Update the timestamp regardless.
 367         m_filestamp = stat_buf.st_mtime;
 368         m_log.info("change detected, signaling reload thread...");
 369         m_reload_wait->signal();
 370     }
 371
 372     return this;
 373 }
 374
 375 void ReloadableXMLFile::unlock()
 376 {
 377     if (m_lock)
 378         m_lock->unlock();
 379 }
 380
 381 pair<bool,DOMElement*> ReloadableXMLFile::load(bool backup)
 382 {
 383 #ifdef _DEBUG
 384     NDC ndc("load");
 385 #endif
 386
 387     try {
 388         if (m_source.empty()) {
 389             // Data comes from the DOM we were handed.
 390             m_log.debug("loading inline configuration...");
 391             return make_pair(false, XMLHelper::getFirstChildElement(m_root));
 392         }
 393         else {
 394             // Data comes from a file we have to parse.
 395             if (backup)
 396                 m_log.warn("using local backup of remote resource");
 397             else
 398                 m_log.debug("loading configuration from external resource...");
 399
 400             DOMDocument* doc=nullptr;
 401             if (m_local || backup) {
 402                 auto_ptr_XMLCh widenit(backup ? m_backing.c_str() : m_source.c_str());
 403                 // Use library-wide lock for now, nothing else is using it anyway.
 404                 Locker locker(backup ? getBackupLock() : nullptr);
 405                 LocalFileInputSource src(widenit.get());
 406                 Wrapper4InputSource dsrc(&src, false);
 407                 if (m_validate)
 408                     doc=XMLToolingConfig::getConfig().getValidatingParser().parse(dsrc);
 409                 else
 410                     doc=XMLToolingConfig::getConfig().getParser().parse(dsrc);
 411             }
 412             else {
 413                 URLInputSource src(m_root, nullptr, &m_cacheTag);
 414                 Wrapper4InputSource dsrc(&src, false);
 415                 if (m_validate)
 416                     doc=XMLToolingConfig::getConfig().getValidatingParser().parse(dsrc);
 417                 else
 418                     doc=XMLToolingConfig::getConfig().getParser().parse(dsrc);
 419
 420                 // Check for a response code signal.
 421                 if (XMLHelper::isNodeNamed(doc->getDocumentElement(), xmlconstants::XMLTOOLING_NS, URLInputSource::utf16StatusCodeElementName)) {
 422                     int responseCode = XMLString::parseInt(doc->getDocumentElement()->getFirstChild()->getNodeValue());
 423                     doc->release();
 424                     if (responseCode == HTTPResponse::XMLTOOLING_HTTP_STATUS_NOTMODIFIED) {
 425                         throw (long)responseCode; // toss out as a "known" case to handle gracefully
 426                     }
 427                     else {
 428                         m_log.warn("remote resource fetch returned atypical status code (%d)", responseCode);
 429                         throw IOException("remote resource fetch failed, check log for status code of response");
 430                     }
 431                 }
 432             }
 433
 434             m_log.infoStream() << "loaded XML resource (" << (backup ? m_backing : m_source) << ")" << logging::eol;
 435 #ifndef XMLTOOLING_LITE
 436             if (m_credResolver || m_trust) {
 437                 m_log.debug("checking signature on XML resource");
 438                 try {
 439                     DOMElement* sigel = XMLHelper::getFirstChildElement(doc->getDocumentElement(), xmlconstants::XMLSIG_NS, Signature::LOCAL_NAME);
 440                     if (!sigel)
 441                         throw XMLSecurityException("Signature validation required, but no signature found.");
 442
 443                     // Wrap and unmarshall the signature for the duration of the check.
 444                     auto_ptr<Signature> sigobj(dynamic_cast<Signature*>(SignatureBuilder::buildOneFromElement(sigel)));    // don't bind to document
 445                     validateSignature(*sigobj.get());
 446                 }
 447                 catch (exception&) {
 448                     doc->release();
 449                     throw;
 450                 }
 451
 452             }
 453 #endif
 454             return make_pair(true, doc->getDocumentElement());
 455         }
 456     }
 457     catch (XMLException& e) {
 458         auto_ptr_char msg(e.getMessage());
 459         m_log.errorStream() << "Xerces error while loading resource (" << (backup ? m_backing : m_source) << "): "
 460             << msg.get() << logging::eol;
 461         throw XMLParserException(msg.get());
 462     }
 463     catch (exception& e) {
 464         m_log.errorStream() << "error while loading resource ("
 465             << (m_source.empty() ? "inline" : (backup ? m_backing : m_source)) << "): " << e.what() << logging::eol;
 466         throw;
 467     }
 468 }
 469
 470 pair<bool,DOMElement*> ReloadableXMLFile::load()
 471 {
 472     // If this method is used, we're responsible for managing failover to a
 473     // backup of a remote resource (if available), and for backing up remote
 474     // resources.
 475     try {
 476         pair<bool,DOMElement*> ret = load(false);
 477         if (!m_backing.empty()) {
 478             m_log.debug("backing up remote resource to (%s)", m_backing.c_str());
 479             try {
 480                 Locker locker(getBackupLock());
 481                 ofstream backer(m_backing.c_str());
 482                 backer << *(ret.second->getOwnerDocument());
 483             }
 484             catch (exception& ex) {
 485                 m_log.crit("exception while backing up resource: %s", ex.what());
 486             }
 487         }
 488         return ret;
 489     }
 490     catch (long&) {
 491         // If there's an HTTP error or the document hasn't changed,
 492         // use the backup iff we have no "valid" resource in place.
 493         // That prevents reload of the backup copy any time the document
 494         // hasn't changed.
 495         if (!m_loaded && !m_backing.empty())
 496             return load(true);
 497         throw;
 498     }
 499     catch (exception&) {
 500         // Same as above, but for general load/parse errors.
 501         if (!m_loaded && !m_backing.empty())
 502             return load(true);
 503         throw;
 504     }
 505 }
 506
 507 pair<bool,DOMElement*> ReloadableXMLFile::background_load()
 508 {
 509     // If this method isn't overridden, we acquire a write lock
 510     // and just call the old override.
 511     if (m_lock)
 512         m_lock->wrlock();
 513     SharedLock locker(m_lock, false);
 514     return load();
 515 }
 516
 517 Lockable* ReloadableXMLFile::getBackupLock()
 518 {
 519     return &XMLToolingConfig::getConfig();
 520 }
 521
 522 #ifndef XMLTOOLING_LITE
 523
 524 void ReloadableXMLFile::validateSignature(Signature& sigObj) const
 525 {
 526     DSIGSignature* sig=sigObj.getXMLSignature();
 527     if (!sig)
 528         throw XMLSecurityException("Signature does not exist yet.");
 529
 530     // Make sure the whole document was signed.
 531     bool valid=false;
 532     DSIGReferenceList* refs=sig->getReferenceList();
 533     if (refs && refs->getSize()==1) {
 534         DSIGReference* ref=refs->item(0);
 535         if (ref) {
 536             const XMLCh* URI=ref->getURI();
 537             if (URI==nullptr || *URI==0) {
 538                 DSIGTransformList* tlist=ref->getTransforms();
 539                 if (tlist->getSize() <= 2) {
 540                     for (unsigned int i=0; tlist && i<tlist->getSize(); i++) {
 541                         if (tlist->item(i)->getTransformType()==TRANSFORM_ENVELOPED_SIGNATURE)
 542                             valid=true;
 543                         else if (tlist->item(i)->getTransformType()!=TRANSFORM_EXC_C14N &&
 544                                  tlist->item(i)->getTransformType()!=TRANSFORM_C14N &&
 545                                  tlist->item(i)->getTransformType()!=TRANSFORM_C14N11) {
 546                             valid=false;
 547                             break;
 548                         }
 549                     }
 550                 }
 551             }
 552         }
 553     }
 554
 555     if (!valid)
 556         throw XMLSecurityException("Invalid signature profile for signed configuration resource.");
 557
 558     // Set up criteria.
 559     CredentialCriteria cc;
 560     cc.setUsage(Credential::SIGNING_CREDENTIAL);
 561     cc.setSignature(sigObj, CredentialCriteria::KEYINFO_EXTRACTION_KEY);
 562     if (!m_signerName.empty())
 563         cc.setPeerName(m_signerName.c_str());
 564
 565     if (m_credResolver) {
 566         Locker locker(m_credResolver);
 567         vector<const Credential*> creds;
 568         if (m_credResolver->resolve(creds, &cc)) {
 569             SignatureValidator sigValidator;
 570             for (vector<const Credential*>::const_iterator i = creds.begin(); i != creds.end(); ++i) {
 571                 try {
 572                     sigValidator.setCredential(*i);
 573                     sigValidator.validate(&sigObj);
 574                     return; // success!
 575                 }
 576                 catch (exception&) {
 577                 }
 578             }
 579             throw XMLSecurityException("Unable to verify signature with supplied key(s).");
 580         }
 581         else {
 582             throw XMLSecurityException("CredentialResolver did not supply any candidate keys.");
 583         }
 584     }
 585     else if (m_trust) {
 586         DummyCredentialResolver dummy;
 587         if (m_trust->validate(sigObj, dummy, &cc))
 588             return;
 589         throw XMLSecurityException("TrustEngine unable to verify signature.");
 590     }
 591
 592     throw XMLSecurityException("Unable to verify signature.");
 593 }
 594
 595 #endif