Old boost version doesn't support push_back with auto_ptr
[shibboleth/cpp-sp.git] / shibsp / attribute / resolver / impl / XMLAttributeExtractor.cpp
1 /**
2  * Licensed to the University Corporation for Advanced Internet
3  * Development, Inc. (UCAID) under one or more contributor license
4  * agreements. See the NOTICE file distributed with this work for
5  * additional information regarding copyright ownership.
6  *
7  * UCAID licenses this file to you under the Apache License,
8  * Version 2.0 (the "License"); you may not use this file except
9  * in compliance with the License. You may obtain a copy of the
10  * License at
11  *
12  * http://www.apache.org/licenses/LICENSE-2.0
13  *
14  * Unless required by applicable law or agreed to in writing,
15  * software distributed under the License is distributed on an
16  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
17  * either express or implied. See the License for the specific
18  * language governing permissions and limitations under the License.
19  */
20
21 /**
22  * XMLAttributeExtractor.cpp
23  *
24  * AttributeExtractor based on an XML mapping file.
25  */
26
27 #include "internal.h"
28 #include "exceptions.h"
29 #include "Application.h"
30 #include "ServiceProvider.h"
31 #include "attribute/Attribute.h"
32 #include "attribute/AttributeDecoder.h"
33 #include "attribute/filtering/AttributeFilter.h"
34 #include "attribute/filtering/BasicFilteringContext.h"
35 #include "attribute/resolver/AttributeExtractor.h"
36 #include "remoting/ddf.h"
37 #include "security/SecurityPolicy.h"
38 #include "util/SPConstants.h"
39
40 #include <boost/bind.hpp>
41 #include <boost/scoped_ptr.hpp>
42 #include <boost/shared_ptr.hpp>
43 #include <boost/algorithm/string.hpp>
44 #include <boost/iterator/indirect_iterator.hpp>
45 #include <boost/ptr_container/ptr_vector.hpp>
46 #include <boost/tuple/tuple.hpp>
47 #include <saml/SAMLConfig.h>
48 #include <saml/saml1/core/Assertions.h>
49 #include <saml/saml2/core/Assertions.h>
50 #include <saml/saml2/metadata/Metadata.h>
51 #include <saml/saml2/metadata/MetadataCredentialCriteria.h>
52 #include <saml/saml2/metadata/ObservableMetadataProvider.h>
53 #include <xmltooling/XMLToolingConfig.h>
54 #include <xmltooling/security/TrustEngine.h>
55 #include <xmltooling/util/NDC.h>
56 #include <xmltooling/util/ReloadableXMLFile.h>
57 #include <xmltooling/util/Threads.h>
58 #include <xmltooling/util/XMLHelper.h>
59 #include <xercesc/util/XMLUniDefs.hpp>
60
61 using namespace shibsp;
62 using namespace opensaml::saml2md;
63 using namespace opensaml;
64 using namespace xmltooling;
65 using namespace boost;
66 using namespace std;
67 using saml1::NameIdentifier;
68 using saml2::NameID;
69 using saml2::EncryptedAttribute;
70
71 namespace shibsp {
72
73 #if defined (_MSC_VER)
74     #pragma warning( push )
75     #pragma warning( disable : 4250 )
76 #endif
77
78     class XMLExtractorImpl : public ObservableMetadataProvider::Observer
79     {
80     public:
81         XMLExtractorImpl(const DOMElement* e, Category& log);
82         ~XMLExtractorImpl() {
83             for (map<const ObservableMetadataProvider*,decoded_t>::iterator i=m_decodedMap.begin(); i!=m_decodedMap.end(); ++i) {
84                 i->first->removeObserver(this);
85                 for (decoded_t::iterator attrs = i->second.begin(); attrs!=i->second.end(); ++attrs)
86                     for_each(attrs->second.begin(), attrs->second.end(), mem_fun_ref<DDF&,DDF>(&DDF::destroy));
87             }
88             if (m_document)
89                 m_document->release();
90         }
91
92         void setDocument(DOMDocument* doc) {
93             m_document = doc;
94         }
95
96         void onEvent(const ObservableMetadataProvider& metadata) const {
97             // Destroy attributes we cached from this provider.
98             m_attrLock->wrlock();
99             SharedLock wrapper(m_attrLock, false);
100             decoded_t& d = m_decodedMap[&metadata];
101             for (decoded_t::iterator a = d.begin(); a!=d.end(); ++a)
102                 for_each(a->second.begin(), a->second.end(), mem_fun_ref<DDF&,DDF>(&DDF::destroy));
103             d.clear();
104         }
105
106         void extractAttributes(
107             const Application& application,
108             const char* assertingParty,
109             const char* relyingParty,
110             const NameIdentifier& nameid,
111             ptr_vector<Attribute>& attributes
112             ) const;
113         void extractAttributes(
114             const Application& application,
115             const char* assertingParty,
116             const char* relyingParty,
117             const NameID& nameid,
118             ptr_vector<Attribute>& attributes
119             ) const;
120         void extractAttributes(
121             const Application& application,
122             const char* assertingParty,
123             const char* relyingParty,
124             const saml1::Attribute& attr,
125             ptr_vector<Attribute>& attributes
126             ) const;
127         void extractAttributes(
128             const Application& application,
129             const char* assertingParty,
130             const char* relyingParty,
131             const saml2::Attribute& attr,
132             ptr_vector<Attribute>& attributes
133             ) const;
134         void extractAttributes(
135             const Application& application,
136             const char* assertingParty,
137             const char* relyingParty,
138             const saml1::AttributeStatement& statement,
139             ptr_vector<Attribute>& attributes
140             ) const;
141         void extractAttributes(
142             const Application& application,
143             const char* assertingParty,
144             const char* relyingParty,
145             const saml2::AttributeStatement& statement,
146             ptr_vector<Attribute>& attributes
147             ) const;
148         void extractAttributes(
149             const Application& application,
150             const ObservableMetadataProvider* observable,
151             const XMLCh* entityID,
152             const char* relyingParty,
153             const Extensions& ext,
154             ptr_vector<Attribute>& attributes
155             ) const;
156
157         void getAttributeIds(vector<string>& attributes) const {
158             attributes.insert(attributes.end(), m_attributeIds.begin(), m_attributeIds.end());
159         }
160
161         void generateMetadata(SPSSODescriptor& role) const;
162
163     private:
164         Category& m_log;
165         DOMDocument* m_document;
166         typedef map< pair<xstring,xstring>,pair< boost::shared_ptr<AttributeDecoder>,vector<string> > > attrmap_t;
167         attrmap_t m_attrMap;
168         vector<string> m_attributeIds;
169         vector< tuple<xstring,xstring,bool> > m_requestedAttrs;
170
171         // settings for embedded assertions in metadata
172         string m_policyId;
173         scoped_ptr<AttributeFilter> m_filter;
174         scoped_ptr<MetadataProvider> m_metadata;
175         scoped_ptr<TrustEngine> m_trust;
176         bool m_entityAssertions;
177
178         // manages caching of decoded Attributes
179         scoped_ptr<RWLock> m_attrLock;
180         typedef map< const EntityAttributes*,vector<DDF> > decoded_t;
181         mutable map<const ObservableMetadataProvider*,decoded_t> m_decodedMap;
182     };
183
184     class XMLExtractor : public AttributeExtractor, public ReloadableXMLFile
185     {
186     public:
187         XMLExtractor(const DOMElement* e) : ReloadableXMLFile(e, Category::getInstance(SHIBSP_LOGCAT".AttributeExtractor.XML")) {
188             background_load();
189         }
190         ~XMLExtractor() {
191             shutdown();
192         }
193
194         void extractAttributes(
195             const Application& application, const RoleDescriptor* issuer, const XMLObject& xmlObject, vector<Attribute*>& attributes
196             ) const;
197
198         void getAttributeIds(std::vector<std::string>& attributes) const {
199             if (m_impl)
200                 m_impl->getAttributeIds(attributes);
201         }
202
203         void generateMetadata(SPSSODescriptor& role) const {
204             if (m_impl)
205                 m_impl->generateMetadata(role);
206         }
207
208     protected:
209         pair<bool,DOMElement*> background_load();
210
211     private:
212         scoped_ptr<XMLExtractorImpl> m_impl;
213
214         void extractAttributes(
215             const Application& application, const RoleDescriptor* issuer, const XMLObject& xmlObject, ptr_vector<Attribute>& attributes
216             ) const;
217     };
218
219 #if defined (_MSC_VER)
220     #pragma warning( pop )
221 #endif
222
223     AttributeExtractor* SHIBSP_DLLLOCAL XMLAttributeExtractorFactory(const DOMElement* const & e)
224     {
225         return new XMLExtractor(e);
226     }
227
228     static const XMLCh _aliases[] =             UNICODE_LITERAL_7(a,l,i,a,s,e,s);
229     static const XMLCh _AttributeDecoder[] =    UNICODE_LITERAL_16(A,t,t,r,i,b,u,t,e,D,e,c,o,d,e,r);
230     static const XMLCh _AttributeFilter[] =     UNICODE_LITERAL_15(A,t,t,r,i,b,u,t,e,F,i,l,t,e,r);
231     static const XMLCh Attributes[] =           UNICODE_LITERAL_10(A,t,t,r,i,b,u,t,e,s);
232     static const XMLCh _id[] =                  UNICODE_LITERAL_2(i,d);
233     static const XMLCh isRequested[] =          UNICODE_LITERAL_11(i,s,R,e,q,u,e,s,t,e,d);
234     static const XMLCh _MetadataProvider[] =    UNICODE_LITERAL_16(M,e,t,a,d,a,t,a,P,r,o,v,i,d,e,r);
235     static const XMLCh _name[] =                UNICODE_LITERAL_4(n,a,m,e);
236     static const XMLCh nameFormat[] =           UNICODE_LITERAL_10(n,a,m,e,F,o,r,m,a,t);
237     static const XMLCh metadataPolicyId[] =     UNICODE_LITERAL_16(m,e,t,a,d,a,t,a,P,o,l,i,c,y,I,d);
238     static const XMLCh _TrustEngine[] =         UNICODE_LITERAL_11(T,r,u,s,t,E,n,g,i,n,e);
239     static const XMLCh _type[] =                UNICODE_LITERAL_4(t,y,p,e);
240 };
241
242 XMLExtractorImpl::XMLExtractorImpl(const DOMElement* e, Category& log)
243     : m_log(log),
244         m_document(nullptr),
245         m_policyId(XMLHelper::getAttrString(e, nullptr, metadataPolicyId)),
246         m_entityAssertions(true)
247 {
248 #ifdef _DEBUG
249     xmltooling::NDC ndc("XMLExtractorImpl");
250 #endif
251
252     if (!XMLHelper::isNodeNamed(e, shibspconstants::SHIB2ATTRIBUTEMAP_NS, Attributes))
253         throw ConfigurationException("XML AttributeExtractor requires am:Attributes at root of configuration.");
254
255     DOMElement* child = XMLHelper::getFirstChildElement(e, shibspconstants::SHIB2ATTRIBUTEMAP_NS, _MetadataProvider);
256     if (child) {
257         try {
258             string t(XMLHelper::getAttrString(child, nullptr, _type));
259             if (t.empty())
260                 throw ConfigurationException("MetadataProvider element missing type attribute.");
261             m_log.info("building MetadataProvider of type %s...", t.c_str());
262             m_metadata.reset(SAMLConfig::getConfig().MetadataProviderManager.newPlugin(t.c_str(), child));
263             m_metadata->init();
264         }
265         catch (std::exception& ex) {
266             m_metadata.reset();
267             m_entityAssertions = false;
268             m_log.crit("error building/initializing dedicated MetadataProvider: %s", ex.what());
269             m_log.crit("disabling support for Assertions in EntityAttributes extension");
270         }
271     }
272
273     if (m_entityAssertions) {
274         child = XMLHelper::getFirstChildElement(e, shibspconstants::SHIB2ATTRIBUTEMAP_NS, _TrustEngine);
275         if (child) {
276             try {
277                 string t(XMLHelper::getAttrString(child, nullptr, _type));
278                 if (t.empty())
279                     throw ConfigurationException("TrustEngine element missing type attribute.");
280                 m_log.info("building TrustEngine of type %s...", t.c_str());
281                 m_trust.reset(XMLToolingConfig::getConfig().TrustEngineManager.newPlugin(t.c_str(), child));
282             }
283             catch (std::exception& ex) {
284                 m_entityAssertions = false;
285                 m_log.crit("error building/initializing dedicated TrustEngine: %s", ex.what());
286                 m_log.crit("disabling support for Assertions in EntityAttributes extension");
287             }
288         }
289     }
290
291     if (m_entityAssertions) {
292         child = XMLHelper::getFirstChildElement(e, shibspconstants::SHIB2ATTRIBUTEMAP_NS, _AttributeFilter);
293         if (child) {
294             try {
295                 string t(XMLHelper::getAttrString(child, nullptr, _type));
296                 if (t.empty())
297                     throw ConfigurationException("AttributeFilter element missing type attribute.");
298                 m_log.info("building AttributeFilter of type %s...", t.c_str());
299                 m_filter.reset(SPConfig::getConfig().AttributeFilterManager.newPlugin(t.c_str(), child));
300             }
301             catch (std::exception& ex) {
302                 m_entityAssertions = false;
303                 m_log.crit("error building/initializing dedicated AttributeFilter: %s", ex.what());
304                 m_log.crit("disabling support for Assertions in EntityAttributes extension");
305             }
306         }
307     }
308
309     child = XMLHelper::getFirstChildElement(e, shibspconstants::SHIB2ATTRIBUTEMAP_NS, saml1::Attribute::LOCAL_NAME);
310     while (child) {
311         // Check for missing name or id.
312         const XMLCh* name = child->getAttributeNS(nullptr, _name);
313         if (!name || !*name) {
314             m_log.warn("skipping Attribute with no name");
315             child = XMLHelper::getNextSiblingElement(child, shibspconstants::SHIB2ATTRIBUTEMAP_NS, saml1::Attribute::LOCAL_NAME);
316             continue;
317         }
318
319         auto_ptr_char id(child->getAttributeNS(nullptr, _id));
320         if (!id.get() || !*id.get()) {
321             m_log.warn("skipping Attribute with no id");
322             child = XMLHelper::getNextSiblingElement(child, shibspconstants::SHIB2ATTRIBUTEMAP_NS, saml1::Attribute::LOCAL_NAME);
323             continue;
324         }
325         else if (!strcmp(id.get(), "REMOTE_USER")) {
326             m_log.warn("skipping Attribute, id of REMOTE_USER is a reserved name");
327             child = XMLHelper::getNextSiblingElement(child, shibspconstants::SHIB2ATTRIBUTEMAP_NS, saml1::Attribute::LOCAL_NAME);
328             continue;
329         }
330
331         boost::shared_ptr<AttributeDecoder> decoder;
332         try {
333             DOMElement* dchild = XMLHelper::getFirstChildElement(child, shibspconstants::SHIB2ATTRIBUTEMAP_NS, _AttributeDecoder);
334             if (dchild) {
335                 auto_ptr<xmltooling::QName> q(XMLHelper::getXSIType(dchild));
336                 if (q.get())
337                     decoder.reset(SPConfig::getConfig().AttributeDecoderManager.newPlugin(*q.get(), dchild));
338             }
339             if (!decoder)
340                 decoder.reset(SPConfig::getConfig().AttributeDecoderManager.newPlugin(StringAttributeDecoderType, nullptr));
341         }
342         catch (std::exception& ex) {
343             m_log.error("skipping Attribute (%s), error building AttributeDecoder: %s", id.get(), ex.what());
344         }
345
346         if (!decoder) {
347             child = XMLHelper::getNextSiblingElement(child, shibspconstants::SHIB2ATTRIBUTEMAP_NS, saml1::Attribute::LOCAL_NAME);
348             continue;
349         }
350
351         // Empty NameFormat implies the usual Shib URI naming defaults.
352         const XMLCh* format = child->getAttributeNS(nullptr, nameFormat);
353         if (!format || XMLString::equals(format, shibspconstants::SHIB1_ATTRIBUTE_NAMESPACE_URI) ||
354                 XMLString::equals(format, saml2::Attribute::URI_REFERENCE))
355             format = &chNull;  // ignore default Format/Namespace values
356
357         // Fetch/create the map entry and see if it's a duplicate rule.
358         pair< boost::shared_ptr<AttributeDecoder>,vector<string> >& decl = m_attrMap[pair<xstring,xstring>(name,format)];
359         if (decl.first) {
360             m_log.warn("skipping duplicate Attribute mapping (same name and nameFormat)");
361             child = XMLHelper::getNextSiblingElement(child, shibspconstants::SHIB2ATTRIBUTEMAP_NS, saml1::Attribute::LOCAL_NAME);
362             continue;
363         }
364
365         if (m_log.isInfoEnabled()) {
366             auto_ptr_char n(name);
367             auto_ptr_char f(format);
368             m_log.info("creating mapping for Attribute %s%s%s", n.get(), *f.get() ? ", Format/Namespace:" : "", f.get());
369         }
370
371         decl.first = decoder;
372         decl.second.push_back(id.get());
373         m_attributeIds.push_back(id.get());
374
375         // Check for isRequired/isRequested.
376         bool requested = XMLHelper::getAttrBool(child, false, isRequested);
377         bool required = XMLHelper::getAttrBool(child, false, RequestedAttribute::ISREQUIRED_ATTRIB_NAME);
378         if (required || requested)
379             m_requestedAttrs.push_back(tuple<xstring,xstring,bool>(name,format,required));
380
381         name = child->getAttributeNS(nullptr, _aliases);
382         if (name && *name) {
383             auto_ptr_char aliases(name);
384             string dup(aliases.get());
385             set<string> new_aliases;
386             split(new_aliases, dup, is_space(), algorithm::token_compress_on);
387             set<string>::iterator ru = new_aliases.find("REMOTE_USER");
388             if (ru != new_aliases.end()) {
389                 m_log.warn("skipping alias, REMOTE_USER is a reserved name");
390                 new_aliases.erase(ru);
391             }
392             m_attributeIds.insert(m_attributeIds.end(), new_aliases.begin(), new_aliases.end());
393         }
394
395         child = XMLHelper::getNextSiblingElement(child, shibspconstants::SHIB2ATTRIBUTEMAP_NS, saml1::Attribute::LOCAL_NAME);
396     }
397
398     m_attrLock.reset(RWLock::create());
399 }
400
401 void XMLExtractorImpl::generateMetadata(SPSSODescriptor& role) const
402 {
403     if (m_requestedAttrs.empty())
404         return;
405     int index = 1;
406     const vector<AttributeConsumingService*>& svcs = const_cast<const SPSSODescriptor*>(&role)->getAttributeConsumingServices();
407     for (vector<AttributeConsumingService*>::const_iterator s =svcs.begin(); s != svcs.end(); ++s) {
408         pair<bool,int> i = (*s)->getIndex();
409         if (i.first && index == i.second)
410             index = i.second + 1;
411     }
412     AttributeConsumingService* svc = AttributeConsumingServiceBuilder::buildAttributeConsumingService();
413     role.getAttributeConsumingServices().push_back(svc);
414     svc->setIndex(index);
415     ServiceName* sn = ServiceNameBuilder::buildServiceName();
416     svc->getServiceNames().push_back(sn);
417     sn->setName(dynamic_cast<EntityDescriptor*>(role.getParent())->getEntityID());
418     static const XMLCh english[] = UNICODE_LITERAL_2(e,n);
419     sn->setLang(english);
420
421     for (vector< tuple<xstring,xstring,bool> >::const_iterator i = m_requestedAttrs.begin(); i != m_requestedAttrs.end(); ++i) {
422         RequestedAttribute* req = RequestedAttributeBuilder::buildRequestedAttribute();
423         svc->getRequestedAttributes().push_back(req);
424         req->setName(i->get<0>().c_str());
425         if (i->get<1>().empty())
426             req->setNameFormat(saml2::Attribute::URI_REFERENCE);
427         else
428             req->setNameFormat(i->get<1>().c_str());
429         if (i->get<2>())
430             req->isRequired(true);
431     }
432 }
433
434 void XMLExtractorImpl::extractAttributes(
435     const Application& application,
436     const char* assertingParty,
437     const char* relyingParty,
438     const NameIdentifier& nameid,
439     ptr_vector<Attribute>& attributes
440     ) const
441 {
442     const XMLCh* format = nameid.getFormat();
443     if (!format || !*format)
444         format = NameIdentifier::UNSPECIFIED;
445     attrmap_t::const_iterator rule;
446     if ((rule = m_attrMap.find(pair<xstring,xstring>(format,xstring()))) != m_attrMap.end()) {
447         auto_ptr<Attribute> a(rule->second.first->decode(rule->second.second, &nameid, assertingParty, relyingParty));
448         if (a.get()) {
449             attributes.push_back(a.get());
450             a.release();
451         }
452     }
453     else if (m_log.isDebugEnabled()) {
454         auto_ptr_char temp(format);
455         m_log.debug("skipping unmapped NameIdentifier with format (%s)", temp.get());
456     }
457 }
458
459 void XMLExtractorImpl::extractAttributes(
460     const Application& application,
461     const char* assertingParty,
462     const char* relyingParty,
463     const NameID& nameid,
464     ptr_vector<Attribute>& attributes
465     ) const
466 {
467     const XMLCh* format = nameid.getFormat();
468     if (!format || !*format)
469         format = NameID::UNSPECIFIED;
470     attrmap_t::const_iterator rule;
471     if ((rule = m_attrMap.find(pair<xstring,xstring>(format,xstring()))) != m_attrMap.end()) {
472         auto_ptr<Attribute> a(rule->second.first->decode(rule->second.second, &nameid, assertingParty, relyingParty));
473         if (a.get()) {
474             attributes.push_back(a.get());
475             a.release();
476         }
477     }
478     else if (m_log.isDebugEnabled()) {
479         auto_ptr_char temp(format);
480         m_log.debug("skipping unmapped NameID with format (%s)", temp.get());
481     }
482 }
483
484 void XMLExtractorImpl::extractAttributes(
485     const Application& application,
486     const char* assertingParty,
487     const char* relyingParty,
488     const saml1::Attribute& attr,
489     ptr_vector<Attribute>& attributes
490     ) const
491 {
492     const XMLCh* name = attr.getAttributeName();
493     const XMLCh* format = attr.getAttributeNamespace();
494     if (!name || !*name)
495         return;
496     if (!format || XMLString::equals(format, shibspconstants::SHIB1_ATTRIBUTE_NAMESPACE_URI))
497         format = &chNull;
498     attrmap_t::const_iterator rule;
499     if ((rule = m_attrMap.find(pair<xstring,xstring>(name,format))) != m_attrMap.end()) {
500         auto_ptr<Attribute> a(rule->second.first->decode(rule->second.second, &attr, assertingParty, relyingParty));
501         if (a.get()) {
502             attributes.push_back(a.get());
503             a.release();
504         }
505     }
506     else if (m_log.isInfoEnabled()) {
507         auto_ptr_char temp1(name);
508         auto_ptr_char temp2(format);
509         m_log.info("skipping unmapped SAML 1.x Attribute with Name: %s%s%s", temp1.get(), *temp2.get() ? ", Namespace:" : "", temp2.get());
510     }
511 }
512
513 void XMLExtractorImpl::extractAttributes(
514     const Application& application,
515     const char* assertingParty,
516     const char* relyingParty,
517     const saml2::Attribute& attr,
518     ptr_vector<Attribute>& attributes
519     ) const
520 {
521     const XMLCh* name = attr.getName();
522     const XMLCh* format = attr.getNameFormat();
523     if (!name || !*name)
524         return;
525     if (!format || !*format)
526         format = saml2::Attribute::UNSPECIFIED;
527     else if (XMLString::equals(format, saml2::Attribute::URI_REFERENCE))
528         format = &chNull;
529     attrmap_t::const_iterator rule;
530     if ((rule = m_attrMap.find(pair<xstring,xstring>(name,format))) != m_attrMap.end()) {
531         auto_ptr<Attribute> a(rule->second.first->decode(rule->second.second, &attr, assertingParty, relyingParty));
532         if (a.get()) {
533             attributes.push_back(a.get());
534             a.release();
535             return;
536         }
537     }
538     else if (XMLString::equals(format, saml2::Attribute::UNSPECIFIED)) {
539         // As a fallback, if the format is "unspecified", null out the value and re-map.
540         if ((rule = m_attrMap.find(pair<xstring,xstring>(name,xstring()))) != m_attrMap.end()) {
541             auto_ptr<Attribute> a(rule->second.first->decode(rule->second.second, &attr, assertingParty, relyingParty));
542             if (a.get()) {
543                 attributes.push_back(a.get());
544                 a.release();
545                 return;
546             }
547         }
548     }
549
550     if (m_log.isInfoEnabled()) {
551         auto_ptr_char temp1(name);
552         auto_ptr_char temp2(format);
553         m_log.info("skipping unmapped SAML 2.0 Attribute with Name: %s%s%s", temp1.get(), *temp2.get() ? ", Format:" : "", temp2.get());
554     }
555 }
556
557 void XMLExtractorImpl::extractAttributes(
558     const Application& application,
559     const char* assertingParty,
560     const char* relyingParty,
561     const saml1::AttributeStatement& statement,
562     ptr_vector<Attribute>& attributes
563     ) const
564 {
565     static void (XMLExtractorImpl::* extract)(
566         const Application&, const char*, const char*, const saml1::Attribute&, ptr_vector<Attribute>&
567         ) const = &XMLExtractorImpl::extractAttributes;
568     for_each(
569         make_indirect_iterator(statement.getAttributes().begin()), make_indirect_iterator(statement.getAttributes().end()),
570         boost::bind(extract, this, boost::cref(application), assertingParty, relyingParty, _1, boost::ref(attributes))
571         );
572 }
573
574 void XMLExtractorImpl::extractAttributes(
575     const Application& application,
576     const char* assertingParty,
577     const char* relyingParty,
578     const saml2::AttributeStatement& statement,
579     ptr_vector<Attribute>& attributes
580     ) const
581 {
582     static void (XMLExtractorImpl::* extract)(
583         const Application&, const char*, const char*, const saml2::Attribute&, ptr_vector<Attribute>&
584         ) const = &XMLExtractorImpl::extractAttributes;
585     for_each(
586         make_indirect_iterator(statement.getAttributes().begin()), make_indirect_iterator(statement.getAttributes().end()),
587         boost::bind(extract, this, boost::cref(application), assertingParty, relyingParty, _1, boost::ref(attributes))
588         );
589 }
590
591 void XMLExtractorImpl::extractAttributes(
592     const Application& application,
593     const ObservableMetadataProvider* observable,
594     const XMLCh* entityID,
595     const char* relyingParty,
596     const Extensions& ext,
597     ptr_vector<Attribute>& attributes
598     ) const
599 {
600     const vector<XMLObject*>& exts = ext.getUnknownXMLObjects();
601     for (vector<XMLObject*>::const_iterator i = exts.begin(); i != exts.end(); ++i) {
602         const EntityAttributes* container = dynamic_cast<const EntityAttributes*>(*i);
603         if (!container)
604             continue;
605
606         bool useCache = false;
607         map<const ObservableMetadataProvider*,decoded_t>::iterator cacheEntry;
608
609         // Check for cached result.
610         if (observable) {
611             m_attrLock->rdlock();
612             cacheEntry = m_decodedMap.find(observable);
613             if (cacheEntry == m_decodedMap.end()) {
614                 // We need to elevate the lock and retry.
615                 m_attrLock->unlock();
616                 m_attrLock->wrlock();
617                 cacheEntry = m_decodedMap.find(observable);
618                 if (cacheEntry == m_decodedMap.end()) {
619                     SharedLock locker(m_attrLock, false);   // guard in case these throw
620
621                     // It's still brand new, so hook it for cache activation.
622                     observable->addObserver(this);
623
624                     // Prime the map reference with an empty decoded map.
625                     cacheEntry = m_decodedMap.insert(make_pair(observable,decoded_t())).first;
626
627                     // Downgrade the lock.
628                     // We don't have to recheck because we never erase the master map entry entirely, even on changes.
629                     locker.release();   // unguard for lock downgrade
630                     m_attrLock->unlock();
631                     m_attrLock->rdlock();
632                 }
633             }
634             useCache = true;
635         }
636
637         if (useCache) {
638             // We're holding the lock, so check the cache.
639             decoded_t::iterator d = cacheEntry->second.find(container);
640             if (d != cacheEntry->second.end()) {
641                 SharedLock locker(m_attrLock, false);   // pop the lock when we're done
642                 for (vector<DDF>::iterator obj = d->second.begin(); obj != d->second.end(); ++obj) {
643                     auto_ptr<Attribute> wrapper(Attribute::unmarshall(*obj));
644                     m_log.debug("recovered cached metadata attribute (%s)", wrapper->getId());
645                     attributes.push_back(wrapper.get());
646                     wrapper.release();
647                 }
648                 break;
649             }
650         }
651
652         // Add a guard for the lock if we're caching.
653         SharedLock locker(useCache ? m_attrLock.get() : nullptr, false);
654
655         // Use a holding area to support caching.
656         ptr_vector<Attribute> holding;
657
658         // Extract attributes into holding area with no asserting party set.
659         static void (XMLExtractorImpl::* extractV2Attr)(
660             const Application&, const char*, const char*, const saml2::Attribute&, ptr_vector<Attribute>&
661             ) const = &XMLExtractorImpl::extractAttributes;
662         for_each(
663             make_indirect_iterator(container->getAttributes().begin()), make_indirect_iterator(container->getAttributes().end()),
664             boost::bind(extractV2Attr, this, boost::ref(application), (const char*)nullptr, relyingParty, _1, boost::ref(holding))
665             );
666
667         if (entityID && m_entityAssertions) {
668             const vector<saml2::Assertion*>& asserts = container->getAssertions();
669             for (indirect_iterator<vector<saml2::Assertion*>::const_iterator> assert = make_indirect_iterator(asserts.begin());
670                     assert != make_indirect_iterator(asserts.end()); ++assert) {
671                 if (!(assert->getSignature())) {
672                     if (m_log.isDebugEnabled()) {
673                         auto_ptr_char eid(entityID);
674                         m_log.debug("skipping unsigned assertion in metadata extension for entity (%s)", eid.get());
675                     }
676                     continue;
677                 }
678                 else if (assert->getAttributeStatements().empty()) {
679                     if (m_log.isDebugEnabled()) {
680                         auto_ptr_char eid(entityID);
681                         m_log.debug("skipping assertion with no AttributeStatement in metadata extension for entity (%s)", eid.get());
682                     }
683                     continue;
684                 }
685                 else {
686                     // Check subject.
687                     const NameID* subject = assert->getSubject() ? assert->getSubject()->getNameID() : nullptr;
688                     if (!subject ||
689                             !XMLString::equals(subject->getFormat(), NameID::ENTITY) ||
690                             !XMLString::equals(subject->getName(), entityID)) {
691                         if (m_log.isDebugEnabled()) {
692                             auto_ptr_char eid(entityID);
693                             m_log.debug("skipping assertion with improper Subject in metadata extension for entity (%s)", eid.get());
694                         }
695                         continue;
696                     }
697                 }
698
699                 try {
700                     // Set up and evaluate a policy for an AA asserting attributes to us.
701                     shibsp::SecurityPolicy policy(application, &AttributeAuthorityDescriptor::ELEMENT_QNAME, false, m_policyId.c_str());
702                     Locker locker(m_metadata.get());
703                     if (m_metadata)
704                         policy.setMetadataProvider(m_metadata.get());
705                     if (m_trust)
706                         policy.setTrustEngine(m_trust.get());
707                     // Populate recipient as audience.
708                     const XMLCh* issuer = assert->getIssuer() ? assert->getIssuer()->getName() : nullptr;
709                     policy.getAudiences().push_back(application.getRelyingParty(issuer)->getXMLString("entityID").second);
710
711                     // Extract assertion information for policy.
712                     policy.setMessageID(assert->getID());
713                     policy.setIssueInstant(assert->getIssueInstantEpoch());
714                     policy.setIssuer(assert->getIssuer());
715
716                     // Look up metadata for issuer.
717                     if (policy.getIssuer() && policy.getMetadataProvider()) {
718                         if (policy.getIssuer()->getFormat() && !XMLString::equals(policy.getIssuer()->getFormat(), saml2::NameIDType::ENTITY)) {
719                             m_log.debug("non-system entity issuer, skipping metadata lookup");
720                         }
721                         else {
722                             m_log.debug("searching metadata for entity assertion issuer...");
723                             pair<const EntityDescriptor*,const RoleDescriptor*> lookup;
724                             MetadataProvider::Criteria& mc = policy.getMetadataProviderCriteria();
725                             mc.entityID_unicode = policy.getIssuer()->getName();
726                             mc.role = &AttributeAuthorityDescriptor::ELEMENT_QNAME;
727                             mc.protocol = samlconstants::SAML20P_NS;
728                             lookup = policy.getMetadataProvider()->getEntityDescriptor(mc);
729                             if (!lookup.first) {
730                                 auto_ptr_char iname(policy.getIssuer()->getName());
731                                 m_log.debug("no metadata found, can't establish identity of issuer (%s)", iname.get());
732                             }
733                             else if (!lookup.second) {
734                                 m_log.debug("unable to find compatible AA role in metadata");
735                             }
736                             else {
737                                 policy.setIssuerMetadata(lookup.second);
738                             }
739                         }
740                     }
741
742                     // Authenticate the assertion. We have to clone and marshall it to establish the signature for verification.
743                     scoped_ptr<saml2::Assertion> tokencopy(assert->cloneAssertion());
744                     tokencopy->marshall();
745                     policy.evaluate(*tokencopy);
746                     if (!policy.isAuthenticated()) {
747                         if (m_log.isDebugEnabled()) {
748                             auto_ptr_char tempid(tokencopy->getID());
749                             auto_ptr_char eid(entityID);
750                             m_log.debug(
751                                 "failed to authenticate assertion (%s) in metadata extension for entity (%s)", tempid.get(), eid.get()
752                                 );
753                         }
754                         continue;
755                     }
756
757                     // Override the asserting/relying party names based on this new issuer.
758                     const EntityDescriptor* inlineEntity =
759                         policy.getIssuerMetadata() ? dynamic_cast<const EntityDescriptor*>(policy.getIssuerMetadata()->getParent()) : nullptr;
760                     auto_ptr_char inlineAssertingParty(inlineEntity ? inlineEntity->getEntityID() : nullptr);
761                     relyingParty = application.getRelyingParty(inlineEntity)->getString("entityID").second;
762
763                     // Use a private holding area for filtering purposes.
764                     ptr_vector<Attribute> holding2;
765                     const vector<saml2::Attribute*>& attrs2 =
766                         const_cast<const saml2::AttributeStatement*>(tokencopy->getAttributeStatements().front())->getAttributes();
767                     for_each(
768                         make_indirect_iterator(attrs2.begin()), make_indirect_iterator(attrs2.end()),
769                         boost::bind(extractV2Attr, this, boost::ref(application), inlineAssertingParty.get(), relyingParty, _1, boost::ref(holding2))
770                         );
771
772                     // Now we locally filter the attributes so that the actual issuer can be properly set.
773                     // If we relied on outside filtering, the attributes couldn't be distinguished from the
774                     // ones that come from the user's IdP.
775                     if (m_filter && !holding2.empty()) {
776
777                         // The filter API uses an unsafe container, so we have to transfer everything into one and back.
778                         vector<Attribute*> unsafe_holding2;
779
780                         // Use a local exception context since the container is unsafe.
781                         try {
782                             while (!holding2.empty()) {
783                                 ptr_vector<Attribute>::auto_type ptr = holding2.pop_back();
784                                 unsafe_holding2.push_back(ptr.get());
785                                 ptr.release();
786                             }
787                             BasicFilteringContext fc(application, unsafe_holding2, policy.getIssuerMetadata());
788                             Locker filtlocker(m_filter.get());
789                             m_filter->filterAttributes(fc, unsafe_holding2);
790
791                             // Transfer back to safe container
792                             while (!unsafe_holding2.empty()) {
793                                 auto_ptr<Attribute> ptr(unsafe_holding2.back());
794                                 unsafe_holding2.pop_back();
795                                 holding2.push_back(ptr.get());
796                                 ptr.release();
797                             }
798                         }
799                         catch (std::exception& ex) {
800                             m_log.error("caught exception filtering attributes: %s", ex.what());
801                             m_log.error("dumping extracted attributes due to filtering exception");
802                             for_each(unsafe_holding2.begin(), unsafe_holding2.end(), xmltooling::cleanup<Attribute>());
803                             holding2.clear();   // in case the exception was during transfer between containers
804                         }
805                     }
806
807                     if (!holding2.empty()) {
808                         // Copy them over to the main holding tank, which transfers ownership.
809                         holding.transfer(holding.end(), holding2);
810                     }
811                 }
812                 catch (std::exception& ex) {
813                     // Known exceptions are handled gracefully by skipping the assertion.
814                     if (m_log.isDebugEnabled()) {
815                         auto_ptr_char tempid(assert->getID());
816                         auto_ptr_char eid(entityID);
817                         m_log.debug(
818                             "exception authenticating assertion (%s) in metadata extension for entity (%s): %s",
819                             tempid.get(),
820                             eid.get(),
821                             ex.what()
822                             );
823                     }
824                     continue;
825                 }
826             }
827         }
828
829         if (!holding.empty()) {
830             if (useCache) {
831                 locker.release();   // unguard to upgrade lock
832                 m_attrLock->unlock();
833                 m_attrLock->wrlock();
834                 SharedLock locker2(m_attrLock, false);   // pop the lock when we're done
835                 if (cacheEntry->second.count(container) == 0) {
836                     static void (vector<DDF>::* push_back)(DDF const &) = &vector<DDF>::push_back;
837                     vector<DDF>& marshalled = cacheEntry->second[container];
838                     for_each(
839                         holding.begin(), holding.end(),
840                         boost::bind(push_back, boost::ref(marshalled), boost::bind(&Attribute::marshall, _1))
841                         );
842                 }
843             }
844
845             // Copy them to the output parameter, which transfers ownership.
846             attributes.transfer(attributes.end(), holding);
847         }
848
849         // If the lock is held, it's guarded.
850
851         break;  // only process a single extension element
852     }
853 }
854
855 void XMLExtractor::extractAttributes(
856     const Application& application, const RoleDescriptor* issuer, const XMLObject& xmlObject, vector<Attribute*>& attributes
857     ) const
858 {
859     if (!m_impl)
860         return;
861
862     ptr_vector<Attribute> holding;
863     extractAttributes(application, issuer, xmlObject, holding);
864
865     // Transfer ownership from the ptr_vector to the unsafe vector for API compatibility.
866     // Any throws should leave each container in a consistent state. The holding container
867     // is freed by us, and the result container by the caller.
868     while (!holding.empty()) {
869         ptr_vector<Attribute>::auto_type ptr = holding.pop_back();
870         attributes.push_back(ptr.get());
871         ptr.release();
872     }
873 }
874
875 void XMLExtractor::extractAttributes(
876     const Application& application, const RoleDescriptor* issuer, const XMLObject& xmlObject, ptr_vector<Attribute>& attributes
877     ) const
878 {
879     static void (XMLExtractor::* extractEncrypted)(
880         const Application&, const RoleDescriptor*, const XMLObject&, ptr_vector<Attribute>&
881         ) const = &XMLExtractor::extractAttributes;
882     static void (XMLExtractorImpl::* extractV1Statement)(
883         const Application&, const char*, const char*, const saml1::AttributeStatement&, ptr_vector<Attribute>&
884         ) const = &XMLExtractorImpl::extractAttributes;
885
886     const EntityDescriptor* entity = issuer ? dynamic_cast<const EntityDescriptor*>(issuer->getParent()) : nullptr;
887     const char* relyingParty = application.getRelyingParty(entity)->getString("entityID").second;
888
889     // Check for statements.
890     if (XMLString::equals(xmlObject.getElementQName().getLocalPart(), saml1::AttributeStatement::LOCAL_NAME)) {
891         const saml2::AttributeStatement* statement2 = dynamic_cast<const saml2::AttributeStatement*>(&xmlObject);
892         if (statement2) {
893             auto_ptr_char assertingParty(entity ? entity->getEntityID() : nullptr);
894             m_impl->extractAttributes(application, assertingParty.get(), relyingParty, *statement2, attributes);
895             // Handle EncryptedAttributes inline so we have access to the role descriptor.
896             const vector<saml2::EncryptedAttribute*>& encattrs = statement2->getEncryptedAttributes();
897             for_each(
898                 make_indirect_iterator(encattrs.begin()), make_indirect_iterator(encattrs.end()),
899                 boost::bind(extractEncrypted, this, boost::ref(application), issuer, _1, boost::ref(attributes))
900                 );
901             return;
902         }
903
904         const saml1::AttributeStatement* statement1 = dynamic_cast<const saml1::AttributeStatement*>(&xmlObject);
905         if (statement1) {
906             auto_ptr_char assertingParty(entity ? entity->getEntityID() : nullptr);
907             m_impl->extractAttributes(application, assertingParty.get(), relyingParty, *statement1, attributes);
908             return;
909         }
910
911         throw AttributeExtractionException("Unable to extract attributes, unknown object type.");
912     }
913
914     // Check for assertions.
915     if (XMLString::equals(xmlObject.getElementQName().getLocalPart(), saml1::Assertion::LOCAL_NAME)) {
916         const saml2::Assertion* token2 = dynamic_cast<const saml2::Assertion*>(&xmlObject);
917         if (token2) {
918             auto_ptr_char assertingParty(entity ? entity->getEntityID() : nullptr);
919             const vector<saml2::AttributeStatement*>& statements = token2->getAttributeStatements();
920             for (indirect_iterator<vector<saml2::AttributeStatement*>::const_iterator> s = make_indirect_iterator(statements.begin());
921                     s != make_indirect_iterator(statements.end()); ++s) {
922                 m_impl->extractAttributes(application, assertingParty.get(), relyingParty, *s, attributes);
923                 // Handle EncryptedAttributes inline so we have access to the role descriptor.
924                 const vector<saml2::EncryptedAttribute*>& encattrs = const_cast<const saml2::AttributeStatement&>(*s).getEncryptedAttributes();
925                 for_each(
926                     make_indirect_iterator(encattrs.begin()), make_indirect_iterator(encattrs.end()),
927                     boost::bind(extractEncrypted, this, boost::ref(application), issuer, _1, boost::ref(attributes))
928                     );
929             }
930             return;
931         }
932
933         const saml1::Assertion* token1 = dynamic_cast<const saml1::Assertion*>(&xmlObject);
934         if (token1) {
935             auto_ptr_char assertingParty(entity ? entity->getEntityID() : nullptr);
936             const vector<saml1::AttributeStatement*>& statements = token1->getAttributeStatements();
937             for_each(make_indirect_iterator(statements.begin()), make_indirect_iterator(statements.end()),
938                 boost::bind(extractV1Statement, m_impl.get(), boost::ref(application), assertingParty.get(), relyingParty, _1, boost::ref(attributes))
939                 );
940             return;
941         }
942
943         throw AttributeExtractionException("Unable to extract attributes, unknown object type.");
944     }
945
946     // Check for metadata.
947     if (XMLString::equals(xmlObject.getElementQName().getNamespaceURI(), samlconstants::SAML20MD_NS)) {
948         const RoleDescriptor* roleToExtract = dynamic_cast<const RoleDescriptor*>(&xmlObject);
949         const EntityDescriptor* entityToExtract = roleToExtract ? dynamic_cast<const EntityDescriptor*>(roleToExtract->getParent()) : nullptr;
950         if (!entityToExtract)
951             throw AttributeExtractionException("Unable to extract attributes, unknown metadata object type.");
952         const Extensions* ext = entityToExtract->getExtensions();
953         if (ext) {
954             m_impl->extractAttributes(
955                 application,
956                 dynamic_cast<const ObservableMetadataProvider*>(application.getMetadataProvider(false)),
957                 entityToExtract->getEntityID(),
958                 relyingParty,
959                 *ext,
960                 attributes
961                 );
962         }
963         const EntitiesDescriptor* group = dynamic_cast<const EntitiesDescriptor*>(entityToExtract->getParent());
964         while (group) {
965             ext = group->getExtensions();
966             if (ext) {
967                 m_impl->extractAttributes(
968                     application,
969                     dynamic_cast<const ObservableMetadataProvider*>(application.getMetadataProvider(false)),
970                     nullptr,   // not an entity, so inline assertions won't be processed
971                     relyingParty,
972                     *ext,
973                     attributes
974                     );
975             }
976             group = dynamic_cast<const EntitiesDescriptor*>(group->getParent());
977         }
978         return;
979     }
980
981     // Check for attributes.
982     if (XMLString::equals(xmlObject.getElementQName().getLocalPart(), saml1::Attribute::LOCAL_NAME)) {
983         auto_ptr_char assertingParty(entity ? entity->getEntityID() : nullptr);
984         const saml2::Attribute* attr2 = dynamic_cast<const saml2::Attribute*>(&xmlObject);
985         if (attr2)
986             return m_impl->extractAttributes(application, assertingParty.get(), relyingParty, *attr2, attributes);
987
988         const saml1::Attribute* attr1 = dynamic_cast<const saml1::Attribute*>(&xmlObject);
989         if (attr1)
990             return m_impl->extractAttributes(application, assertingParty.get(), relyingParty, *attr1, attributes);
991
992         throw AttributeExtractionException("Unable to extract attributes, unknown object type.");
993     }
994
995     if (XMLString::equals(xmlObject.getElementQName().getLocalPart(), EncryptedAttribute::LOCAL_NAME)) {
996         const EncryptedAttribute* encattr = dynamic_cast<const EncryptedAttribute*>(&xmlObject);
997         if (encattr) {
998             const XMLCh* recipient = application.getXMLString("entityID").second;
999             CredentialResolver* cr = application.getCredentialResolver();
1000             if (!cr) {
1001                 m_log.warn("found encrypted attribute, but no CredentialResolver was available");
1002                 return;
1003             }
1004
1005             try {
1006                 Locker credlocker(cr);
1007                 if (issuer) {
1008                     MetadataCredentialCriteria mcc(*issuer);
1009                     scoped_ptr<XMLObject> decrypted(encattr->decrypt(*cr, recipient, &mcc));
1010                     if (m_log.isDebugEnabled())
1011                         m_log.debugStream() << "decrypted Attribute: " << *decrypted << logging::eol;
1012                     return extractAttributes(application, issuer, *decrypted, attributes);
1013                 }
1014                 else {
1015                     scoped_ptr<XMLObject> decrypted(encattr->decrypt(*cr, recipient));
1016                     if (m_log.isDebugEnabled())
1017                         m_log.debugStream() << "decrypted Attribute: " << *decrypted << logging::eol;
1018                     return extractAttributes(application, issuer, *decrypted, attributes);
1019                 }
1020             }
1021             catch (std::exception& ex) {
1022                 m_log.error("caught exception decrypting Attribute: %s", ex.what());
1023                 return;
1024             }
1025         }
1026     }
1027
1028     // Check for NameIDs.
1029     const NameID* name2 = dynamic_cast<const NameID*>(&xmlObject);
1030     if (name2) {
1031         auto_ptr_char assertingParty(entity ? entity->getEntityID() : nullptr);
1032         return m_impl->extractAttributes(application, assertingParty.get(), relyingParty, *name2, attributes);
1033     }
1034
1035     const NameIdentifier* name1 = dynamic_cast<const NameIdentifier*>(&xmlObject);
1036     if (name1) {
1037         auto_ptr_char assertingParty(entity ? entity->getEntityID() : nullptr);
1038         return m_impl->extractAttributes(application, assertingParty.get(), relyingParty, *name1, attributes);
1039     }
1040
1041     m_log.debug("unable to extract attributes, unknown XML object type: %s", xmlObject.getElementQName().toString().c_str());
1042 }
1043
1044 pair<bool,DOMElement*> XMLExtractor::background_load()
1045 {
1046     // Load from source using base class.
1047     pair<bool,DOMElement*> raw = ReloadableXMLFile::load();
1048
1049     // If we own it, wrap it.
1050     XercesJanitor<DOMDocument> docjanitor(raw.first ? raw.second->getOwnerDocument() : nullptr);
1051
1052     scoped_ptr<XMLExtractorImpl> impl(new XMLExtractorImpl(raw.second, m_log));
1053
1054     // If we held the document, transfer it to the impl. If we didn't, it's a no-op.
1055     impl->setDocument(docjanitor.release());
1056
1057     // Perform the swap inside a lock.
1058     if (m_lock)
1059         m_lock->wrlock();
1060     SharedLock locker(m_lock, false);
1061     m_impl.swap(impl);
1062
1063     return make_pair(false,(DOMElement*)nullptr);
1064 }