From d6379bae77dd178313af9ec8e117a3dbd835ed0a Mon Sep 17 00:00:00 2001 From: Scott Cantor Date: Mon, 24 May 2010 14:33:03 +0000 Subject: [PATCH] https://issues.shibboleth.net/jira/browse/CPPOST-47 --- saml/saml2/metadata/AbstractMetadataProvider.h | 28 +++- .../metadata/impl/AbstractMetadataProvider.cpp | 65 ++++++--- saml/saml2/metadata/impl/XMLMetadataProvider.cpp | 152 +++++++++++++++++---- 3 files changed, 200 insertions(+), 45 deletions(-) diff --git a/saml/saml2/metadata/AbstractMetadataProvider.h b/saml/saml2/metadata/AbstractMetadataProvider.h index 3bccb93..9e60e27 100644 --- a/saml/saml2/metadata/AbstractMetadataProvider.h +++ b/saml/saml2/metadata/AbstractMetadataProvider.h @@ -90,8 +90,29 @@ namespace opensaml { xmltooling::KeyInfoResolver* m_resolver; /** - * Loads an entity into the cache for faster lookup. This includes - * processing known reverse lookup strategies for artifacts. + * Loads an entity into the cache for faster lookup. + *

This includes processing known reverse lookup strategies for artifacts. + * The validUntil parameter will contain the smallest value found on output. + * + * @param site entity definition + * @param validUntil maximum expiration time of the entity definition + * @param replace true iff existing entries for the same entity should be cleared/replaced + */ + virtual void indexEntity(EntityDescriptor* site, time_t& validUntil, bool replace=false) const; + + /** + * Loads a group of entities into the cache for faster lookup. + *

The validUntil parameter will contain the smallest value found on output. + * + * @param group group definition + * @param validUntil maximum expiration time of the group definition + */ + virtual void indexGroup(EntitiesDescriptor* group, time_t& validUntil) const; + + /** + * @deprecated + * Loads an entity into the cache for faster lookup. + *

This includes processing known reverse lookup strategies for artifacts. * * @param site entity definition * @param validUntil maximum expiration time of the entity definition @@ -100,13 +121,14 @@ namespace opensaml { virtual void index(EntityDescriptor* site, time_t validUntil, bool replace=false) const; /** + * @deprecated * Loads a group of entities into the cache for faster lookup. * * @param group group definition * @param validUntil maximum expiration time of the group definition */ virtual void index(EntitiesDescriptor* group, time_t validUntil) const; - + /** * Clear the cache of known entities and groups. * diff --git a/saml/saml2/metadata/impl/AbstractMetadataProvider.cpp b/saml/saml2/metadata/impl/AbstractMetadataProvider.cpp index 4c89489..3c404e1 100644 --- a/saml/saml2/metadata/impl/AbstractMetadataProvider.cpp +++ b/saml/saml2/metadata/impl/AbstractMetadataProvider.cpp @@ -75,10 +75,13 @@ void AbstractMetadataProvider::emitChangeEvent() const ObservableMetadataProvider::emitChangeEvent(); } -void AbstractMetadataProvider::index(EntityDescriptor* site, time_t validUntil, bool replace) const +void AbstractMetadataProvider::indexEntity(EntityDescriptor* site, time_t& validUntil, bool replace) const { + // If child expires later than input, reset child, otherwise lower input to match. if (validUntil < site->getValidUntilEpoch()) site->setValidUntil(validUntil); + else + validUntil = site->getValidUntilEpoch(); auto_ptr_char id(site->getEntityID()); if (id.get()) { @@ -99,16 +102,16 @@ void AbstractMetadataProvider::index(EntityDescriptor* site, time_t validUntil, } // Process each IdP role. - const vector& roles=const_cast(site)->getIDPSSODescriptors(); - for (vector::const_iterator i=roles.begin(); i!=roles.end(); i++) { + const vector& roles = const_cast(site)->getIDPSSODescriptors(); + for (vector::const_iterator i = roles.begin(); i != roles.end(); i++) { // SAML 1.x? if ((*i)->hasSupport(samlconstants::SAML10_PROTOCOL_ENUM) || (*i)->hasSupport(samlconstants::SAML11_PROTOCOL_ENUM)) { // Check for SourceID extension element. - const Extensions* exts=(*i)->getExtensions(); + const Extensions* exts = (*i)->getExtensions(); if (exts && exts->hasChildren()) { - const vector& children=exts->getUnknownXMLObjects(); - for (vector::const_iterator ext=children.begin(); ext!=children.end(); ++ext) { - SourceID* sid=dynamic_cast(*ext); + const vector& children = exts->getUnknownXMLObjects(); + for (vector::const_iterator ext = children.begin(); ext != children.end(); ++ext) { + SourceID* sid = dynamic_cast(*ext); if (sid) { auto_ptr_char sourceid(sid->getID()); if (sourceid.get()) { @@ -123,8 +126,8 @@ void AbstractMetadataProvider::index(EntityDescriptor* site, time_t validUntil, m_sources.insert(sitemap_t::value_type(SecurityHelper::doHash("SHA1", id.get(), strlen(id.get())),site)); // Load endpoints for type 0x0002 artifacts. - const vector& locs=const_cast(*i)->getArtifactResolutionServices(); - for (vector::const_iterator loc=locs.begin(); loc!=locs.end(); loc++) { + const vector& locs = const_cast(*i)->getArtifactResolutionServices(); + for (vector::const_iterator loc = locs.begin(); loc != locs.end(); loc++) { auto_ptr_char location((*loc)->getLocation()); if (location.get()) m_sources.insert(sitemap_t::value_type(location.get(),site)); @@ -139,23 +142,53 @@ void AbstractMetadataProvider::index(EntityDescriptor* site, time_t validUntil, } } -void AbstractMetadataProvider::index(EntitiesDescriptor* group, time_t validUntil) const +void AbstractMetadataProvider::indexGroup(EntitiesDescriptor* group, time_t& validUntil) const { + // If child expires later than input, reset child, otherwise lower input to match. if (validUntil < group->getValidUntilEpoch()) group->setValidUntil(validUntil); + else + validUntil = group->getValidUntilEpoch(); auto_ptr_char name(group->getName()); if (name.get()) { m_groups.insert(groupmap_t::value_type(name.get(),group)); } - const vector& groups=const_cast(group)->getEntitiesDescriptors(); - for (vector::const_iterator i=groups.begin(); i!=groups.end(); i++) - index(*i,group->getValidUntilEpoch()); + // Track the smallest validUntil amongst the children. + time_t minValidUntil = validUntil; + + const vector& groups = const_cast(group)->getEntitiesDescriptors(); + for (vector::const_iterator i = groups.begin(); i != groups.end(); i++) { + // Use the current validUntil fence for each child, but track the smallest we find. + time_t subValidUntil = validUntil; + indexGroup(*i, subValidUntil); + if (subValidUntil < minValidUntil) + minValidUntil = subValidUntil; + } + + const vector& sites = const_cast(group)->getEntityDescriptors(); + for (vector::const_iterator j = sites.begin(); j != sites.end(); j++) { + // Use the current validUntil fence for each child, but track the smallest we find. + time_t subValidUntil = validUntil; + indexEntity(*j, subValidUntil); + if (subValidUntil < minValidUntil) + minValidUntil = subValidUntil; + } - const vector& sites=const_cast(group)->getEntityDescriptors(); - for (vector::const_iterator j=sites.begin(); j!=sites.end(); j++) - index(*j,group->getValidUntilEpoch()); + // Pass back up the smallest child we found. + if (minValidUntil < validUntil) + validUntil = minValidUntil; +} + +void AbstractMetadataProvider::index(EntityDescriptor* site, time_t validUntil, bool replace) const +{ + indexEntity(site, validUntil, replace); +} + +void AbstractMetadataProvider::index(EntitiesDescriptor* group, time_t validUntil) const +{ + indexGroup(group, validUntil); } void AbstractMetadataProvider::clearDescriptorIndex(bool freeSites) diff --git a/saml/saml2/metadata/impl/XMLMetadataProvider.cpp b/saml/saml2/metadata/impl/XMLMetadataProvider.cpp index aa9c0cd..bf359c9 100644 --- a/saml/saml2/metadata/impl/XMLMetadataProvider.cpp +++ b/saml/saml2/metadata/impl/XMLMetadataProvider.cpp @@ -27,6 +27,7 @@ #include "saml2/metadata/AbstractMetadataProvider.h" #include +#include #include #include #include @@ -45,20 +46,22 @@ using namespace std; namespace opensaml { namespace saml2md { + static const XMLCh minRefreshDelay[] = UNICODE_LITERAL_15(m,i,n,R,e,f,r,e,s,h,D,e,l,a,y); + static const XMLCh refreshDelayFactor[] = UNICODE_LITERAL_18(r,e,f,r,e,s,h,D,e,l,a,y,F,a,c,t,o,r); + class SAML_DLLLOCAL XMLMetadataProvider : public AbstractMetadataProvider, public ReloadableXMLFile { public: - XMLMetadataProvider(const DOMElement* e) - : AbstractMetadataProvider(e), ReloadableXMLFile(e, Category::getInstance(SAML_LOGCAT".MetadataProvider.XML")), - m_object(nullptr), m_maxCacheDuration(m_reloadInterval) { - } + XMLMetadataProvider(const DOMElement* e); + virtual ~XMLMetadataProvider() { shutdown(); delete m_object; } void init() { - background_load(); // guarantees an exception or the metadata is loaded + background_load(); + startup(); } const XMLObject* getMetadata() const { @@ -66,14 +69,18 @@ namespace opensaml { } protected: + pair load(bool backup); pair background_load(); private: using AbstractMetadataProvider::index; - void index(); + void index(time_t& validUntil); + time_t computeNextRefresh(); XMLObject* m_object; - time_t m_maxCacheDuration; + double m_refreshDelayFactor; + unsigned int m_backoffFactor; + time_t m_minRefreshDelay,m_maxRefreshDelay,m_lastValidUntil; }; MetadataProvider* SAML_DLLLOCAL XMLMetadataProviderFactory(const DOMElement* const & e) @@ -88,13 +95,43 @@ namespace opensaml { #pragma warning( pop ) #endif -pair XMLMetadataProvider::background_load() +XMLMetadataProvider::XMLMetadataProvider(const DOMElement* e) + : AbstractMetadataProvider(e), ReloadableXMLFile(e, Category::getInstance(SAML_LOGCAT".MetadataProvider.XML"), false), + m_object(nullptr), m_refreshDelayFactor(0.75), m_backoffFactor(1), m_minRefreshDelay(600), + m_maxRefreshDelay(m_reloadInterval), m_lastValidUntil(SAMLTIME_MAX) { - // Turn off auto-backup so we can filter first. - m_backupIndicator = false; + if (!m_local && m_maxRefreshDelay) { + const XMLCh* setting = e ? e->getAttributeNS(nullptr, refreshDelayFactor) : NULL; + if (setting && *setting) { + auto_ptr_char delay(setting); + m_refreshDelayFactor = atof(delay.get()); + if (m_refreshDelayFactor <= 0.0 || m_refreshDelayFactor >= 1.0) { + m_log.error("invalid refreshDelayFactor setting, using default"); + m_refreshDelayFactor = 0.75; + } + } + setting = e ? e->getAttributeNS(nullptr, minRefreshDelay) : NULL; + if (setting && *setting) { + m_minRefreshDelay = XMLString::parseInt(setting); + if (m_minRefreshDelay == 0) { + m_log.error("invalid minRefreshDelay setting, using default"); + m_minRefreshDelay = 600; + } + else if (m_minRefreshDelay > m_maxRefreshDelay) { + m_log.error("minRefreshDelay setting exceeds maxRefreshDelay/refreshInterval setting, lowering to match it"); + m_minRefreshDelay = m_maxRefreshDelay; + } + } + } +} + +pair XMLMetadataProvider::load(bool backup) +{ + // Lower the refresh rate in case of an error. + m_reloadInterval = m_minRefreshDelay; - // Load from source using base class. - pair raw = ReloadableXMLFile::load(); + // Call the base class to load/parse the appropriate XML resource. + pair raw = ReloadableXMLFile::load(backup); // If we own it, wrap it for now. XercesJanitor docjanitor(raw.first ? raw.second->getOwnerDocument() : nullptr); @@ -117,17 +154,16 @@ pair XMLMetadataProvider::background_load() throw MetadataException("Metadata instance failed manual validation checking."); } - // If the backup indicator is flipped, then this was a remote load and we need a backup. // This is the best place to take a backup, since it's superficially "correct" metadata. string backupKey; - if (m_backupIndicator) { + if (!backup && !m_backing.empty()) { // We compute a random filename extension to the "real" location. SAMLConfig::getConfig().generateRandomBytes(backupKey, 2); backupKey = m_backing + '.' + SAMLArtifact::toHex(backupKey); m_log.debug("backing up remote metadata resource to (%s)", backupKey.c_str()); try { ofstream backer(backupKey.c_str()); - backer << *raw.second->getOwnerDocument(); + backer << *(raw.second->getOwnerDocument()); } catch (exception& ex) { m_log.crit("exception while backing up metadata: %s", ex.what()); @@ -162,29 +198,93 @@ pair XMLMetadataProvider::background_load() bool changed = m_object!=nullptr; delete m_object; m_object = xmlObject.release(); - index(); + m_lastValidUntil = SAMLTIME_MAX; + index(m_lastValidUntil); if (changed) emitChangeEvent(); - // If a remote resource, adjust the reload interval if cacheDuration is set. - if (!m_local) { - const CacheableSAMLObject* cacheable = dynamic_cast(m_object); - if (cacheable && cacheable->getCacheDuration() && cacheable->getCacheDurationEpoch() < m_maxCacheDuration) - m_reloadInterval = cacheable->getCacheDurationEpoch(); - else - m_reloadInterval = m_maxCacheDuration; + // Tracking cacheUntil through the tree is TBD, but + // validUntil is the tightest interval amongst the children. + + // If a remote resource, adjust the reload interval. + if (!backup) { + m_backoffFactor = 1; + m_reloadInterval = computeNextRefresh(); + m_log.info("adjusted reload interval to %d seconds", m_reloadInterval); } + m_loaded = true; return make_pair(false,(DOMElement*)nullptr); } -void XMLMetadataProvider::index() +pair XMLMetadataProvider::background_load() +{ + try { + return load(false); + } + catch (long& ex) { + if (ex == HTTPResponse::XMLTOOLING_HTTP_STATUS_NOTMODIFIED) { + // Unchanged document, so re-establish previous refresh interval. + m_reloadInterval = computeNextRefresh(); + m_log.info("adjusted reload interval to %d seconds", m_reloadInterval); + } + else { + // Any other status code, just treat as an error. + m_reloadInterval = m_minRefreshDelay * m_backoffFactor++; + if (m_reloadInterval > m_maxRefreshDelay) + m_reloadInterval = m_maxRefreshDelay; + m_log.warn("adjusted reload interval to %d seconds", m_reloadInterval); + } + if (!m_loaded && !m_backing.empty()) + return load(true); + throw; + } + catch (exception&) { + m_reloadInterval = m_minRefreshDelay * m_backoffFactor++; + if (m_reloadInterval > m_maxRefreshDelay) + m_reloadInterval = m_maxRefreshDelay; + m_log.warn("adjusted reload interval to %d seconds", m_reloadInterval); + if (!m_loaded && !m_backing.empty()) + return load(true); + throw; + } +} + +void XMLMetadataProvider::index(time_t& validUntil) { clearDescriptorIndex(); EntitiesDescriptor* group=dynamic_cast(m_object); if (group) { - AbstractMetadataProvider::index(group, SAMLTIME_MAX); + indexGroup(group, validUntil); return; } - AbstractMetadataProvider::index(dynamic_cast(m_object), SAMLTIME_MAX); + indexEntity(dynamic_cast(m_object), validUntil); +} + +time_t XMLMetadataProvider::computeNextRefresh() +{ + time_t now = time(nullptr); + + // If some or all of the metadata is already expired, reload after the minimum. + if (m_lastValidUntil < now) { + return m_minRefreshDelay; + } + else { + // Compute the smaller of the validUntil / cacheDuration constraints. + time_t ret = m_lastValidUntil - now; + const CacheableSAMLObject* cacheable = dynamic_cast(m_object); + if (cacheable && cacheable->getCacheDuration()) + ret = min(ret, cacheable->getCacheDurationEpoch()); + + // Adjust for the delay factor. + ret *= m_refreshDelayFactor; + + // Bound by max and min. + if (ret > m_maxRefreshDelay) + return m_maxRefreshDelay; + else if (ret < m_minRefreshDelay) + return m_minRefreshDelay; + + return ret; + } } -- 2.1.4