Remove extra header
[shibboleth/cpp-sp.git] / shibsp / attribute / resolver / impl / XMLAttributeExtractor.cpp
1 /**
2  * Licensed to the University Corporation for Advanced Internet
3  * Development, Inc. (UCAID) under one or more contributor license
4  * agreements. See the NOTICE file distributed with this work for
5  * additional information regarding copyright ownership.
6  *
7  * UCAID licenses this file to you under the Apache License,
8  * Version 2.0 (the "License"); you may not use this file except
9  * in compliance with the License. You may obtain a copy of the
10  * License at
11  *
12  * http://www.apache.org/licenses/LICENSE-2.0
13  *
14  * Unless required by applicable law or agreed to in writing,
15  * software distributed under the License is distributed on an
16  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
17  * either express or implied. See the License for the specific
18  * language governing permissions and limitations under the License.
19  */
20
21 /**
22  * XMLAttributeExtractor.cpp
23  *
24  * AttributeExtractor based on an XML mapping file.
25  */
26
27 #include "internal.h"
28 #include "exceptions.h"
29 #include "Application.h"
30 #include "ServiceProvider.h"
31 #include "attribute/Attribute.h"
32 #include "attribute/AttributeDecoder.h"
33 #include "attribute/filtering/AttributeFilter.h"
34 #include "attribute/filtering/BasicFilteringContext.h"
35 #include "attribute/resolver/AttributeExtractor.h"
36 #include "remoting/ddf.h"
37 #include "security/SecurityPolicy.h"
38 #include "util/SPConstants.h"
39
40 #include <boost/bind.hpp>
41 #include <boost/shared_ptr.hpp>
42 #include <boost/algorithm/string.hpp>
43 #include <boost/iterator/indirect_iterator.hpp>
44 #include <boost/ptr_container/ptr_vector.hpp>
45 #include <boost/tuple/tuple.hpp>
46 #include <saml/SAMLConfig.h>
47 #include <saml/saml1/core/Assertions.h>
48 #include <saml/saml2/core/Assertions.h>
49 #include <saml/saml2/metadata/Metadata.h>
50 #include <saml/saml2/metadata/MetadataCredentialCriteria.h>
51 #include <saml/saml2/metadata/ObservableMetadataProvider.h>
52 #include <xmltooling/XMLToolingConfig.h>
53 #include <xmltooling/security/TrustEngine.h>
54 #include <xmltooling/util/NDC.h>
55 #include <xmltooling/util/ReloadableXMLFile.h>
56 #include <xmltooling/util/Threads.h>
57 #include <xmltooling/util/XMLHelper.h>
58 #include <xercesc/util/XMLUniDefs.hpp>
59
60 using namespace shibsp;
61 using namespace opensaml::saml2md;
62 using namespace opensaml;
63 using namespace xmltooling;
64 using namespace boost;
65 using namespace std;
66 using saml1::NameIdentifier;
67 using saml2::NameID;
68 using saml2::EncryptedAttribute;
69
70 namespace shibsp {
71
72 #if defined (_MSC_VER)
73     #pragma warning( push )
74     #pragma warning( disable : 4250 )
75 #endif
76
77     class XMLExtractorImpl : public ObservableMetadataProvider::Observer
78     {
79     public:
80         XMLExtractorImpl(const DOMElement* e, Category& log);
81         ~XMLExtractorImpl() {
82             for (map<const ObservableMetadataProvider*,decoded_t>::iterator i=m_decodedMap.begin(); i!=m_decodedMap.end(); ++i) {
83                 i->first->removeObserver(this);
84                 for (decoded_t::iterator attrs = i->second.begin(); attrs!=i->second.end(); ++attrs)
85                     for_each(attrs->second.begin(), attrs->second.end(), mem_fun_ref<DDF&,DDF>(&DDF::destroy));
86             }
87             if (m_document)
88                 m_document->release();
89         }
90
91         void setDocument(DOMDocument* doc) {
92             m_document = doc;
93         }
94
95         void onEvent(const ObservableMetadataProvider& metadata) const {
96             // Destroy attributes we cached from this provider.
97             m_attrLock->wrlock();
98             SharedLock wrapper(m_attrLock, false);
99             decoded_t& d = m_decodedMap[&metadata];
100             for (decoded_t::iterator a = d.begin(); a!=d.end(); ++a)
101                 for_each(a->second.begin(), a->second.end(), mem_fun_ref<DDF&,DDF>(&DDF::destroy));
102             d.clear();
103         }
104
105         void extractAttributes(
106             const Application& application,
107             const char* assertingParty,
108             const char* relyingParty,
109             const NameIdentifier& nameid,
110             ptr_vector<Attribute>& attributes
111             ) const;
112         void extractAttributes(
113             const Application& application,
114             const char* assertingParty,
115             const char* relyingParty,
116             const NameID& nameid,
117             ptr_vector<Attribute>& attributes
118             ) const;
119         void extractAttributes(
120             const Application& application,
121             const char* assertingParty,
122             const char* relyingParty,
123             const saml1::Attribute& attr,
124             ptr_vector<Attribute>& attributes
125             ) const;
126         void extractAttributes(
127             const Application& application,
128             const char* assertingParty,
129             const char* relyingParty,
130             const saml2::Attribute& attr,
131             ptr_vector<Attribute>& attributes
132             ) const;
133         void extractAttributes(
134             const Application& application,
135             const char* assertingParty,
136             const char* relyingParty,
137             const saml1::AttributeStatement& statement,
138             ptr_vector<Attribute>& attributes
139             ) const;
140         void extractAttributes(
141             const Application& application,
142             const char* assertingParty,
143             const char* relyingParty,
144             const saml2::AttributeStatement& statement,
145             ptr_vector<Attribute>& attributes
146             ) const;
147         void extractAttributes(
148             const Application& application,
149             const ObservableMetadataProvider* observable,
150             const XMLCh* entityID,
151             const char* relyingParty,
152             const Extensions& ext,
153             ptr_vector<Attribute>& attributes
154             ) const;
155
156         void getAttributeIds(vector<string>& attributes) const {
157             attributes.insert(attributes.end(), m_attributeIds.begin(), m_attributeIds.end());
158         }
159
160         void generateMetadata(SPSSODescriptor& role) const;
161
162     private:
163         Category& m_log;
164         DOMDocument* m_document;
165         typedef map< pair<xstring,xstring>,pair< boost::shared_ptr<AttributeDecoder>,vector<string> > > attrmap_t;
166         attrmap_t m_attrMap;
167         vector<string> m_attributeIds;
168         vector< tuple<xstring,xstring,bool> > m_requestedAttrs;
169
170         // settings for embedded assertions in metadata
171         string m_policyId;
172         scoped_ptr<AttributeFilter> m_filter;
173         scoped_ptr<MetadataProvider> m_metadata;
174         scoped_ptr<TrustEngine> m_trust;
175         bool m_entityAssertions;
176
177         // manages caching of decoded Attributes
178         scoped_ptr<RWLock> m_attrLock;
179         typedef map< const EntityAttributes*,vector<DDF> > decoded_t;
180         mutable map<const ObservableMetadataProvider*,decoded_t> m_decodedMap;
181     };
182
183     class XMLExtractor : public AttributeExtractor, public ReloadableXMLFile
184     {
185     public:
186         XMLExtractor(const DOMElement* e) : ReloadableXMLFile(e, Category::getInstance(SHIBSP_LOGCAT".AttributeExtractor.XML")) {
187             background_load();
188         }
189         ~XMLExtractor() {
190             shutdown();
191         }
192
193         void extractAttributes(
194             const Application& application, const RoleDescriptor* issuer, const XMLObject& xmlObject, vector<Attribute*>& attributes
195             ) const;
196
197         void getAttributeIds(std::vector<std::string>& attributes) const {
198             if (m_impl)
199                 m_impl->getAttributeIds(attributes);
200         }
201
202         void generateMetadata(SPSSODescriptor& role) const {
203             if (m_impl)
204                 m_impl->generateMetadata(role);
205         }
206
207     protected:
208         pair<bool,DOMElement*> background_load();
209
210     private:
211         scoped_ptr<XMLExtractorImpl> m_impl;
212
213         void extractAttributes(
214             const Application& application, const RoleDescriptor* issuer, const XMLObject& xmlObject, ptr_vector<Attribute>& attributes
215             ) const;
216     };
217
218 #if defined (_MSC_VER)
219     #pragma warning( pop )
220 #endif
221
222     AttributeExtractor* SHIBSP_DLLLOCAL XMLAttributeExtractorFactory(const DOMElement* const & e)
223     {
224         return new XMLExtractor(e);
225     }
226
227     static const XMLCh _aliases[] =             UNICODE_LITERAL_7(a,l,i,a,s,e,s);
228     static const XMLCh _AttributeDecoder[] =    UNICODE_LITERAL_16(A,t,t,r,i,b,u,t,e,D,e,c,o,d,e,r);
229     static const XMLCh _AttributeFilter[] =     UNICODE_LITERAL_15(A,t,t,r,i,b,u,t,e,F,i,l,t,e,r);
230     static const XMLCh Attributes[] =           UNICODE_LITERAL_10(A,t,t,r,i,b,u,t,e,s);
231     static const XMLCh _id[] =                  UNICODE_LITERAL_2(i,d);
232     static const XMLCh isRequested[] =          UNICODE_LITERAL_11(i,s,R,e,q,u,e,s,t,e,d);
233     static const XMLCh _MetadataProvider[] =    UNICODE_LITERAL_16(M,e,t,a,d,a,t,a,P,r,o,v,i,d,e,r);
234     static const XMLCh _name[] =                UNICODE_LITERAL_4(n,a,m,e);
235     static const XMLCh nameFormat[] =           UNICODE_LITERAL_10(n,a,m,e,F,o,r,m,a,t);
236     static const XMLCh metadataPolicyId[] =     UNICODE_LITERAL_16(m,e,t,a,d,a,t,a,P,o,l,i,c,y,I,d);
237     static const XMLCh _TrustEngine[] =         UNICODE_LITERAL_11(T,r,u,s,t,E,n,g,i,n,e);
238     static const XMLCh _type[] =                UNICODE_LITERAL_4(t,y,p,e);
239 };
240
241 XMLExtractorImpl::XMLExtractorImpl(const DOMElement* e, Category& log)
242     : m_log(log),
243         m_document(nullptr),
244         m_policyId(XMLHelper::getAttrString(e, nullptr, metadataPolicyId)),
245         m_entityAssertions(true)
246 {
247 #ifdef _DEBUG
248     xmltooling::NDC ndc("XMLExtractorImpl");
249 #endif
250
251     if (!XMLHelper::isNodeNamed(e, shibspconstants::SHIB2ATTRIBUTEMAP_NS, Attributes))
252         throw ConfigurationException("XML AttributeExtractor requires am:Attributes at root of configuration.");
253
254     DOMElement* child = XMLHelper::getFirstChildElement(e, shibspconstants::SHIB2ATTRIBUTEMAP_NS, _MetadataProvider);
255     if (child) {
256         try {
257             string t(XMLHelper::getAttrString(child, nullptr, _type));
258             if (t.empty())
259                 throw ConfigurationException("MetadataProvider element missing type attribute.");
260             m_log.info("building MetadataProvider of type %s...", t.c_str());
261             m_metadata.reset(SAMLConfig::getConfig().MetadataProviderManager.newPlugin(t.c_str(), child));
262             m_metadata->init();
263         }
264         catch (std::exception& ex) {
265             m_metadata.reset();
266             m_entityAssertions = false;
267             m_log.crit("error building/initializing dedicated MetadataProvider: %s", ex.what());
268             m_log.crit("disabling support for Assertions in EntityAttributes extension");
269         }
270     }
271
272     if (m_entityAssertions) {
273         child = XMLHelper::getFirstChildElement(e, shibspconstants::SHIB2ATTRIBUTEMAP_NS, _TrustEngine);
274         if (child) {
275             try {
276                 string t(XMLHelper::getAttrString(child, nullptr, _type));
277                 if (t.empty())
278                     throw ConfigurationException("TrustEngine element missing type attribute.");
279                 m_log.info("building TrustEngine of type %s...", t.c_str());
280                 m_trust.reset(XMLToolingConfig::getConfig().TrustEngineManager.newPlugin(t.c_str(), child));
281             }
282             catch (std::exception& ex) {
283                 m_entityAssertions = false;
284                 m_log.crit("error building/initializing dedicated TrustEngine: %s", ex.what());
285                 m_log.crit("disabling support for Assertions in EntityAttributes extension");
286             }
287         }
288     }
289
290     if (m_entityAssertions) {
291         child = XMLHelper::getFirstChildElement(e, shibspconstants::SHIB2ATTRIBUTEMAP_NS, _AttributeFilter);
292         if (child) {
293             try {
294                 string t(XMLHelper::getAttrString(child, nullptr, _type));
295                 if (t.empty())
296                     throw ConfigurationException("AttributeFilter element missing type attribute.");
297                 m_log.info("building AttributeFilter of type %s...", t.c_str());
298                 m_filter.reset(SPConfig::getConfig().AttributeFilterManager.newPlugin(t.c_str(), child));
299             }
300             catch (std::exception& ex) {
301                 m_entityAssertions = false;
302                 m_log.crit("error building/initializing dedicated AttributeFilter: %s", ex.what());
303                 m_log.crit("disabling support for Assertions in EntityAttributes extension");
304             }
305         }
306     }
307
308     child = XMLHelper::getFirstChildElement(e, shibspconstants::SHIB2ATTRIBUTEMAP_NS, saml1::Attribute::LOCAL_NAME);
309     while (child) {
310         // Check for missing name or id.
311         const XMLCh* name = child->getAttributeNS(nullptr, _name);
312         if (!name || !*name) {
313             m_log.warn("skipping Attribute with no name");
314             child = XMLHelper::getNextSiblingElement(child, shibspconstants::SHIB2ATTRIBUTEMAP_NS, saml1::Attribute::LOCAL_NAME);
315             continue;
316         }
317
318         auto_ptr_char id(child->getAttributeNS(nullptr, _id));
319         if (!id.get() || !*id.get()) {
320             m_log.warn("skipping Attribute with no id");
321             child = XMLHelper::getNextSiblingElement(child, shibspconstants::SHIB2ATTRIBUTEMAP_NS, saml1::Attribute::LOCAL_NAME);
322             continue;
323         }
324         else if (!strcmp(id.get(), "REMOTE_USER")) {
325             m_log.warn("skipping Attribute, id of REMOTE_USER is a reserved name");
326             child = XMLHelper::getNextSiblingElement(child, shibspconstants::SHIB2ATTRIBUTEMAP_NS, saml1::Attribute::LOCAL_NAME);
327             continue;
328         }
329
330         boost::shared_ptr<AttributeDecoder> decoder;
331         try {
332             DOMElement* dchild = XMLHelper::getFirstChildElement(child, shibspconstants::SHIB2ATTRIBUTEMAP_NS, _AttributeDecoder);
333             if (dchild) {
334                 auto_ptr<xmltooling::QName> q(XMLHelper::getXSIType(dchild));
335                 if (q.get())
336                     decoder.reset(SPConfig::getConfig().AttributeDecoderManager.newPlugin(*q.get(), dchild));
337             }
338             if (!decoder)
339                 decoder.reset(SPConfig::getConfig().AttributeDecoderManager.newPlugin(StringAttributeDecoderType, nullptr));
340         }
341         catch (std::exception& ex) {
342             m_log.error("skipping Attribute (%s), error building AttributeDecoder: %s", id.get(), ex.what());
343         }
344
345         if (!decoder) {
346             child = XMLHelper::getNextSiblingElement(child, shibspconstants::SHIB2ATTRIBUTEMAP_NS, saml1::Attribute::LOCAL_NAME);
347             continue;
348         }
349
350         // Empty NameFormat implies the usual Shib URI naming defaults.
351         const XMLCh* format = child->getAttributeNS(nullptr, nameFormat);
352         if (!format || XMLString::equals(format, shibspconstants::SHIB1_ATTRIBUTE_NAMESPACE_URI) ||
353                 XMLString::equals(format, saml2::Attribute::URI_REFERENCE))
354             format = &chNull;  // ignore default Format/Namespace values
355
356         // Fetch/create the map entry and see if it's a duplicate rule.
357         pair< boost::shared_ptr<AttributeDecoder>,vector<string> >& decl = m_attrMap[pair<xstring,xstring>(name,format)];
358         if (decl.first) {
359             m_log.warn("skipping duplicate Attribute mapping (same name and nameFormat)");
360             child = XMLHelper::getNextSiblingElement(child, shibspconstants::SHIB2ATTRIBUTEMAP_NS, saml1::Attribute::LOCAL_NAME);
361             continue;
362         }
363
364         if (m_log.isInfoEnabled()) {
365             auto_ptr_char n(name);
366             auto_ptr_char f(format);
367             m_log.info("creating mapping for Attribute %s%s%s", n.get(), *f.get() ? ", Format/Namespace:" : "", f.get());
368         }
369
370         decl.first = decoder;
371         decl.second.push_back(id.get());
372         m_attributeIds.push_back(id.get());
373
374         // Check for isRequired/isRequested.
375         bool requested = XMLHelper::getAttrBool(child, false, isRequested);
376         bool required = XMLHelper::getAttrBool(child, false, RequestedAttribute::ISREQUIRED_ATTRIB_NAME);
377         if (required || requested)
378             m_requestedAttrs.push_back(tuple<xstring,xstring,bool>(name,format,required));
379
380         name = child->getAttributeNS(nullptr, _aliases);
381         if (name && *name) {
382             auto_ptr_char aliases(name);
383             string dup(aliases.get());
384             set<string> new_aliases;
385             split(new_aliases, dup, is_space(), algorithm::token_compress_on);
386             set<string>::iterator ru = new_aliases.find("REMOTE_USER");
387             if (ru != new_aliases.end()) {
388                 m_log.warn("skipping alias, REMOTE_USER is a reserved name");
389                 new_aliases.erase(ru);
390             }
391             m_attributeIds.insert(m_attributeIds.end(), new_aliases.begin(), new_aliases.end());
392         }
393
394         child = XMLHelper::getNextSiblingElement(child, shibspconstants::SHIB2ATTRIBUTEMAP_NS, saml1::Attribute::LOCAL_NAME);
395     }
396
397     m_attrLock.reset(RWLock::create());
398 }
399
400 void XMLExtractorImpl::generateMetadata(SPSSODescriptor& role) const
401 {
402     if (m_requestedAttrs.empty())
403         return;
404     int index = 1;
405     const vector<AttributeConsumingService*>& svcs = const_cast<const SPSSODescriptor*>(&role)->getAttributeConsumingServices();
406     for (vector<AttributeConsumingService*>::const_iterator s =svcs.begin(); s != svcs.end(); ++s) {
407         pair<bool,int> i = (*s)->getIndex();
408         if (i.first && index == i.second)
409             index = i.second + 1;
410     }
411     AttributeConsumingService* svc = AttributeConsumingServiceBuilder::buildAttributeConsumingService();
412     role.getAttributeConsumingServices().push_back(svc);
413     svc->setIndex(index);
414     ServiceName* sn = ServiceNameBuilder::buildServiceName();
415     svc->getServiceNames().push_back(sn);
416     sn->setName(dynamic_cast<EntityDescriptor*>(role.getParent())->getEntityID());
417     static const XMLCh english[] = UNICODE_LITERAL_2(e,n);
418     sn->setLang(english);
419
420     for (vector< tuple<xstring,xstring,bool> >::const_iterator i = m_requestedAttrs.begin(); i != m_requestedAttrs.end(); ++i) {
421         RequestedAttribute* req = RequestedAttributeBuilder::buildRequestedAttribute();
422         svc->getRequestedAttributes().push_back(req);
423         req->setName(i->get<0>().c_str());
424         if (i->get<1>().empty())
425             req->setNameFormat(saml2::Attribute::URI_REFERENCE);
426         else
427             req->setNameFormat(i->get<1>().c_str());
428         if (i->get<2>())
429             req->isRequired(true);
430     }
431 }
432
433 void XMLExtractorImpl::extractAttributes(
434     const Application& application,
435     const char* assertingParty,
436     const char* relyingParty,
437     const NameIdentifier& nameid,
438     ptr_vector<Attribute>& attributes
439     ) const
440 {
441     const XMLCh* format = nameid.getFormat();
442     if (!format || !*format)
443         format = NameIdentifier::UNSPECIFIED;
444     attrmap_t::const_iterator rule;
445     if ((rule = m_attrMap.find(pair<xstring,xstring>(format,xstring()))) != m_attrMap.end()) {
446         auto_ptr<Attribute> a(rule->second.first->decode(rule->second.second, &nameid, assertingParty, relyingParty));
447         if (a.get()) {
448             attributes.push_back(a.get());
449             a.release();
450         }
451     }
452     else if (m_log.isDebugEnabled()) {
453         auto_ptr_char temp(format);
454         m_log.debug("skipping unmapped NameIdentifier with format (%s)", temp.get());
455     }
456 }
457
458 void XMLExtractorImpl::extractAttributes(
459     const Application& application,
460     const char* assertingParty,
461     const char* relyingParty,
462     const NameID& nameid,
463     ptr_vector<Attribute>& attributes
464     ) const
465 {
466     const XMLCh* format = nameid.getFormat();
467     if (!format || !*format)
468         format = NameID::UNSPECIFIED;
469     attrmap_t::const_iterator rule;
470     if ((rule = m_attrMap.find(pair<xstring,xstring>(format,xstring()))) != m_attrMap.end()) {
471         auto_ptr<Attribute> a(rule->second.first->decode(rule->second.second, &nameid, assertingParty, relyingParty));
472         if (a.get()) {
473             attributes.push_back(a.get());
474             a.release();
475         }
476     }
477     else if (m_log.isDebugEnabled()) {
478         auto_ptr_char temp(format);
479         m_log.debug("skipping unmapped NameID with format (%s)", temp.get());
480     }
481 }
482
483 void XMLExtractorImpl::extractAttributes(
484     const Application& application,
485     const char* assertingParty,
486     const char* relyingParty,
487     const saml1::Attribute& attr,
488     ptr_vector<Attribute>& attributes
489     ) const
490 {
491     const XMLCh* name = attr.getAttributeName();
492     const XMLCh* format = attr.getAttributeNamespace();
493     if (!name || !*name)
494         return;
495     if (!format || XMLString::equals(format, shibspconstants::SHIB1_ATTRIBUTE_NAMESPACE_URI))
496         format = &chNull;
497     attrmap_t::const_iterator rule;
498     if ((rule = m_attrMap.find(pair<xstring,xstring>(name,format))) != m_attrMap.end()) {
499         auto_ptr<Attribute> a(rule->second.first->decode(rule->second.second, &attr, assertingParty, relyingParty));
500         if (a.get()) {
501             attributes.push_back(a.get());
502             a.release();
503         }
504     }
505     else if (m_log.isInfoEnabled()) {
506         auto_ptr_char temp1(name);
507         auto_ptr_char temp2(format);
508         m_log.info("skipping unmapped SAML 1.x Attribute with Name: %s%s%s", temp1.get(), *temp2.get() ? ", Namespace:" : "", temp2.get());
509     }
510 }
511
512 void XMLExtractorImpl::extractAttributes(
513     const Application& application,
514     const char* assertingParty,
515     const char* relyingParty,
516     const saml2::Attribute& attr,
517     ptr_vector<Attribute>& attributes
518     ) const
519 {
520     const XMLCh* name = attr.getName();
521     const XMLCh* format = attr.getNameFormat();
522     if (!name || !*name)
523         return;
524     if (!format || !*format)
525         format = saml2::Attribute::UNSPECIFIED;
526     else if (XMLString::equals(format, saml2::Attribute::URI_REFERENCE))
527         format = &chNull;
528     attrmap_t::const_iterator rule;
529     if ((rule = m_attrMap.find(pair<xstring,xstring>(name,format))) != m_attrMap.end()) {
530         auto_ptr<Attribute> a(rule->second.first->decode(rule->second.second, &attr, assertingParty, relyingParty));
531         if (a.get()) {
532             attributes.push_back(a.get());
533             a.release();
534             return;
535         }
536     }
537     else if (XMLString::equals(format, saml2::Attribute::UNSPECIFIED)) {
538         // As a fallback, if the format is "unspecified", null out the value and re-map.
539         if ((rule = m_attrMap.find(pair<xstring,xstring>(name,xstring()))) != m_attrMap.end()) {
540             auto_ptr<Attribute> a(rule->second.first->decode(rule->second.second, &attr, assertingParty, relyingParty));
541             if (a.get()) {
542                 attributes.push_back(a.get());
543                 a.release();
544                 return;
545             }
546         }
547     }
548
549     if (m_log.isInfoEnabled()) {
550         auto_ptr_char temp1(name);
551         auto_ptr_char temp2(format);
552         m_log.info("skipping unmapped SAML 2.0 Attribute with Name: %s%s%s", temp1.get(), *temp2.get() ? ", Format:" : "", temp2.get());
553     }
554 }
555
556 void XMLExtractorImpl::extractAttributes(
557     const Application& application,
558     const char* assertingParty,
559     const char* relyingParty,
560     const saml1::AttributeStatement& statement,
561     ptr_vector<Attribute>& attributes
562     ) const
563 {
564     static void (XMLExtractorImpl::* extract)(
565         const Application&, const char*, const char*, const saml1::Attribute&, ptr_vector<Attribute>&
566         ) const = &XMLExtractorImpl::extractAttributes;
567     for_each(
568         make_indirect_iterator(statement.getAttributes().begin()), make_indirect_iterator(statement.getAttributes().end()),
569         boost::bind(extract, this, boost::cref(application), assertingParty, relyingParty, _1, boost::ref(attributes))
570         );
571 }
572
573 void XMLExtractorImpl::extractAttributes(
574     const Application& application,
575     const char* assertingParty,
576     const char* relyingParty,
577     const saml2::AttributeStatement& statement,
578     ptr_vector<Attribute>& attributes
579     ) const
580 {
581     static void (XMLExtractorImpl::* extract)(
582         const Application&, const char*, const char*, const saml2::Attribute&, ptr_vector<Attribute>&
583         ) const = &XMLExtractorImpl::extractAttributes;
584     for_each(
585         make_indirect_iterator(statement.getAttributes().begin()), make_indirect_iterator(statement.getAttributes().end()),
586         boost::bind(extract, this, boost::cref(application), assertingParty, relyingParty, _1, boost::ref(attributes))
587         );
588 }
589
590 void XMLExtractorImpl::extractAttributes(
591     const Application& application,
592     const ObservableMetadataProvider* observable,
593     const XMLCh* entityID,
594     const char* relyingParty,
595     const Extensions& ext,
596     ptr_vector<Attribute>& attributes
597     ) const
598 {
599     const vector<XMLObject*>& exts = ext.getUnknownXMLObjects();
600     for (vector<XMLObject*>::const_iterator i = exts.begin(); i != exts.end(); ++i) {
601         const EntityAttributes* container = dynamic_cast<const EntityAttributes*>(*i);
602         if (!container)
603             continue;
604
605         bool useCache = false;
606         map<const ObservableMetadataProvider*,decoded_t>::iterator cacheEntry;
607
608         // Check for cached result.
609         if (observable) {
610             m_attrLock->rdlock();
611             cacheEntry = m_decodedMap.find(observable);
612             if (cacheEntry == m_decodedMap.end()) {
613                 // We need to elevate the lock and retry.
614                 m_attrLock->unlock();
615                 m_attrLock->wrlock();
616                 cacheEntry = m_decodedMap.find(observable);
617                 if (cacheEntry == m_decodedMap.end()) {
618                     SharedLock locker(m_attrLock, false);   // guard in case these throw
619
620                     // It's still brand new, so hook it for cache activation.
621                     observable->addObserver(this);
622
623                     // Prime the map reference with an empty decoded map.
624                     cacheEntry = m_decodedMap.insert(make_pair(observable,decoded_t())).first;
625
626                     // Downgrade the lock.
627                     // We don't have to recheck because we never erase the master map entry entirely, even on changes.
628                     locker.release();   // unguard for lock downgrade
629                     m_attrLock->unlock();
630                     m_attrLock->rdlock();
631                 }
632             }
633             useCache = true;
634         }
635
636         if (useCache) {
637             // We're holding the lock, so check the cache.
638             decoded_t::iterator d = cacheEntry->second.find(container);
639             if (d != cacheEntry->second.end()) {
640                 SharedLock locker(m_attrLock, false);   // pop the lock when we're done
641                 for (vector<DDF>::iterator obj = d->second.begin(); obj != d->second.end(); ++obj) {
642                     auto_ptr<Attribute> wrapper(Attribute::unmarshall(*obj));
643                     m_log.debug("recovered cached metadata attribute (%s)", wrapper->getId());
644                     attributes.push_back(wrapper.get());
645                     wrapper.release();
646                 }
647                 break;
648             }
649         }
650
651         // Add a guard for the lock if we're caching.
652         SharedLock locker(useCache ? m_attrLock.get() : nullptr, false);
653
654         // Use a holding area to support caching.
655         ptr_vector<Attribute> holding;
656
657         // Extract attributes into holding area with no asserting party set.
658         static void (XMLExtractorImpl::* extractV2Attr)(
659             const Application&, const char*, const char*, const saml2::Attribute&, ptr_vector<Attribute>&
660             ) const = &XMLExtractorImpl::extractAttributes;
661         for_each(
662             make_indirect_iterator(container->getAttributes().begin()), make_indirect_iterator(container->getAttributes().end()),
663             boost::bind(extractV2Attr, this, boost::ref(application), (const char*)nullptr, relyingParty, _1, boost::ref(holding))
664             );
665
666         if (entityID && m_entityAssertions) {
667             const vector<saml2::Assertion*>& asserts = container->getAssertions();
668             for (indirect_iterator<vector<saml2::Assertion*>::const_iterator> assert = make_indirect_iterator(asserts.begin());
669                     assert != make_indirect_iterator(asserts.end()); ++assert) {
670                 if (!(assert->getSignature())) {
671                     if (m_log.isDebugEnabled()) {
672                         auto_ptr_char eid(entityID);
673                         m_log.debug("skipping unsigned assertion in metadata extension for entity (%s)", eid.get());
674                     }
675                     continue;
676                 }
677                 else if (assert->getAttributeStatements().empty()) {
678                     if (m_log.isDebugEnabled()) {
679                         auto_ptr_char eid(entityID);
680                         m_log.debug("skipping assertion with no AttributeStatement in metadata extension for entity (%s)", eid.get());
681                     }
682                     continue;
683                 }
684                 else {
685                     // Check subject.
686                     const NameID* subject = assert->getSubject() ? assert->getSubject()->getNameID() : nullptr;
687                     if (!subject ||
688                             !XMLString::equals(subject->getFormat(), NameID::ENTITY) ||
689                             !XMLString::equals(subject->getName(), entityID)) {
690                         if (m_log.isDebugEnabled()) {
691                             auto_ptr_char eid(entityID);
692                             m_log.debug("skipping assertion with improper Subject in metadata extension for entity (%s)", eid.get());
693                         }
694                         continue;
695                     }
696                 }
697
698                 try {
699                     // Set up and evaluate a policy for an AA asserting attributes to us.
700                     shibsp::SecurityPolicy policy(application, &AttributeAuthorityDescriptor::ELEMENT_QNAME, false, m_policyId.c_str());
701                     Locker locker(m_metadata.get());
702                     if (m_metadata)
703                         policy.setMetadataProvider(m_metadata.get());
704                     if (m_trust)
705                         policy.setTrustEngine(m_trust.get());
706                     // Populate recipient as audience.
707                     const XMLCh* issuer = assert->getIssuer() ? assert->getIssuer()->getName() : nullptr;
708                     policy.getAudiences().push_back(application.getRelyingParty(issuer)->getXMLString("entityID").second);
709
710                     // Extract assertion information for policy.
711                     policy.setMessageID(assert->getID());
712                     policy.setIssueInstant(assert->getIssueInstantEpoch());
713                     policy.setIssuer(assert->getIssuer());
714
715                     // Look up metadata for issuer.
716                     if (policy.getIssuer() && policy.getMetadataProvider()) {
717                         if (policy.getIssuer()->getFormat() && !XMLString::equals(policy.getIssuer()->getFormat(), saml2::NameIDType::ENTITY)) {
718                             m_log.debug("non-system entity issuer, skipping metadata lookup");
719                         }
720                         else {
721                             m_log.debug("searching metadata for entity assertion issuer...");
722                             pair<const EntityDescriptor*,const RoleDescriptor*> lookup;
723                             MetadataProvider::Criteria& mc = policy.getMetadataProviderCriteria();
724                             mc.entityID_unicode = policy.getIssuer()->getName();
725                             mc.role = &AttributeAuthorityDescriptor::ELEMENT_QNAME;
726                             mc.protocol = samlconstants::SAML20P_NS;
727                             lookup = policy.getMetadataProvider()->getEntityDescriptor(mc);
728                             if (!lookup.first) {
729                                 auto_ptr_char iname(policy.getIssuer()->getName());
730                                 m_log.debug("no metadata found, can't establish identity of issuer (%s)", iname.get());
731                             }
732                             else if (!lookup.second) {
733                                 m_log.debug("unable to find compatible AA role in metadata");
734                             }
735                             else {
736                                 policy.setIssuerMetadata(lookup.second);
737                             }
738                         }
739                     }
740
741                     // Authenticate the assertion. We have to clone and marshall it to establish the signature for verification.
742                     scoped_ptr<saml2::Assertion> tokencopy(assert->cloneAssertion());
743                     tokencopy->marshall();
744                     policy.evaluate(*tokencopy);
745                     if (!policy.isAuthenticated()) {
746                         if (m_log.isDebugEnabled()) {
747                             auto_ptr_char tempid(tokencopy->getID());
748                             auto_ptr_char eid(entityID);
749                             m_log.debug(
750                                 "failed to authenticate assertion (%s) in metadata extension for entity (%s)", tempid.get(), eid.get()
751                                 );
752                         }
753                         continue;
754                     }
755
756                     // Override the asserting/relying party names based on this new issuer.
757                     const EntityDescriptor* inlineEntity =
758                         policy.getIssuerMetadata() ? dynamic_cast<const EntityDescriptor*>(policy.getIssuerMetadata()->getParent()) : nullptr;
759                     auto_ptr_char inlineAssertingParty(inlineEntity ? inlineEntity->getEntityID() : nullptr);
760                     relyingParty = application.getRelyingParty(inlineEntity)->getString("entityID").second;
761
762                     // Use a private holding area for filtering purposes.
763                     ptr_vector<Attribute> holding2;
764                     const vector<saml2::Attribute*>& attrs2 =
765                         const_cast<const saml2::AttributeStatement*>(tokencopy->getAttributeStatements().front())->getAttributes();
766                     for_each(
767                         make_indirect_iterator(attrs2.begin()), make_indirect_iterator(attrs2.end()),
768                         boost::bind(extractV2Attr, this, boost::ref(application), inlineAssertingParty.get(), relyingParty, _1, boost::ref(holding2))
769                         );
770
771                     // Now we locally filter the attributes so that the actual issuer can be properly set.
772                     // If we relied on outside filtering, the attributes couldn't be distinguished from the
773                     // ones that come from the user's IdP.
774                     if (m_filter && !holding2.empty()) {
775
776                         // The filter API uses an unsafe container, so we have to transfer everything into one and back.
777                         vector<Attribute*> unsafe_holding2;
778
779                         // Use a local exception context since the container is unsafe.
780                         try {
781                             while (!holding2.empty()) {
782                                 ptr_vector<Attribute>::auto_type ptr = holding2.pop_back();
783                                 unsafe_holding2.push_back(ptr.get());
784                                 ptr.release();
785                             }
786                             BasicFilteringContext fc(application, unsafe_holding2, policy.getIssuerMetadata());
787                             Locker filtlocker(m_filter.get());
788                             m_filter->filterAttributes(fc, unsafe_holding2);
789
790                             // Transfer back to safe container
791                             while (!unsafe_holding2.empty()) {
792                                 auto_ptr<Attribute> ptr(unsafe_holding2.back());
793                                 unsafe_holding2.pop_back();
794                                 holding2.push_back(ptr.get());
795                                 ptr.release();
796                             }
797                         }
798                         catch (std::exception& ex) {
799                             m_log.error("caught exception filtering attributes: %s", ex.what());
800                             m_log.error("dumping extracted attributes due to filtering exception");
801                             for_each(unsafe_holding2.begin(), unsafe_holding2.end(), xmltooling::cleanup<Attribute>());
802                             holding2.clear();   // in case the exception was during transfer between containers
803                         }
804                     }
805
806                     if (!holding2.empty()) {
807                         // Copy them over to the main holding tank, which transfers ownership.
808                         holding.transfer(holding.end(), holding2);
809                     }
810                 }
811                 catch (std::exception& ex) {
812                     // Known exceptions are handled gracefully by skipping the assertion.
813                     if (m_log.isDebugEnabled()) {
814                         auto_ptr_char tempid(assert->getID());
815                         auto_ptr_char eid(entityID);
816                         m_log.debug(
817                             "exception authenticating assertion (%s) in metadata extension for entity (%s): %s",
818                             tempid.get(),
819                             eid.get(),
820                             ex.what()
821                             );
822                     }
823                     continue;
824                 }
825             }
826         }
827
828         if (!holding.empty()) {
829             if (useCache) {
830                 locker.release();   // unguard to upgrade lock
831                 m_attrLock->unlock();
832                 m_attrLock->wrlock();
833                 SharedLock locker2(m_attrLock, false);   // pop the lock when we're done
834                 if (cacheEntry->second.count(container) == 0) {
835                     static void (vector<DDF>::* push_back)(DDF const &) = &vector<DDF>::push_back;
836                     vector<DDF>& marshalled = cacheEntry->second[container];
837                     for_each(
838                         holding.begin(), holding.end(),
839                         boost::bind(push_back, boost::ref(marshalled), boost::bind(&Attribute::marshall, _1))
840                         );
841                 }
842             }
843
844             // Copy them to the output parameter, which transfers ownership.
845             attributes.transfer(attributes.end(), holding);
846         }
847
848         // If the lock is held, it's guarded.
849
850         break;  // only process a single extension element
851     }
852 }
853
854 void XMLExtractor::extractAttributes(
855     const Application& application, const RoleDescriptor* issuer, const XMLObject& xmlObject, vector<Attribute*>& attributes
856     ) const
857 {
858     if (!m_impl)
859         return;
860
861     ptr_vector<Attribute> holding;
862     extractAttributes(application, issuer, xmlObject, holding);
863
864     // Transfer ownership from the ptr_vector to the unsafe vector for API compatibility.
865     // Any throws should leave each container in a consistent state. The holding container
866     // is freed by us, and the result container by the caller.
867     while (!holding.empty()) {
868         ptr_vector<Attribute>::auto_type ptr = holding.pop_back();
869         attributes.push_back(ptr.get());
870         ptr.release();
871     }
872 }
873
874 void XMLExtractor::extractAttributes(
875     const Application& application, const RoleDescriptor* issuer, const XMLObject& xmlObject, ptr_vector<Attribute>& attributes
876     ) const
877 {
878     static void (XMLExtractor::* extractEncrypted)(
879         const Application&, const RoleDescriptor*, const XMLObject&, ptr_vector<Attribute>&
880         ) const = &XMLExtractor::extractAttributes;
881     static void (XMLExtractorImpl::* extractV1Statement)(
882         const Application&, const char*, const char*, const saml1::AttributeStatement&, ptr_vector<Attribute>&
883         ) const = &XMLExtractorImpl::extractAttributes;
884
885     const EntityDescriptor* entity = issuer ? dynamic_cast<const EntityDescriptor*>(issuer->getParent()) : nullptr;
886     const char* relyingParty = application.getRelyingParty(entity)->getString("entityID").second;
887
888     // Check for statements.
889     if (XMLString::equals(xmlObject.getElementQName().getLocalPart(), saml1::AttributeStatement::LOCAL_NAME)) {
890         const saml2::AttributeStatement* statement2 = dynamic_cast<const saml2::AttributeStatement*>(&xmlObject);
891         if (statement2) {
892             auto_ptr_char assertingParty(entity ? entity->getEntityID() : nullptr);
893             m_impl->extractAttributes(application, assertingParty.get(), relyingParty, *statement2, attributes);
894             // Handle EncryptedAttributes inline so we have access to the role descriptor.
895             const vector<saml2::EncryptedAttribute*>& encattrs = statement2->getEncryptedAttributes();
896             for_each(
897                 make_indirect_iterator(encattrs.begin()), make_indirect_iterator(encattrs.end()),
898                 boost::bind(extractEncrypted, this, boost::ref(application), issuer, _1, boost::ref(attributes))
899                 );
900             return;
901         }
902
903         const saml1::AttributeStatement* statement1 = dynamic_cast<const saml1::AttributeStatement*>(&xmlObject);
904         if (statement1) {
905             auto_ptr_char assertingParty(entity ? entity->getEntityID() : nullptr);
906             m_impl->extractAttributes(application, assertingParty.get(), relyingParty, *statement1, attributes);
907             return;
908         }
909
910         throw AttributeExtractionException("Unable to extract attributes, unknown object type.");
911     }
912
913     // Check for assertions.
914     if (XMLString::equals(xmlObject.getElementQName().getLocalPart(), saml1::Assertion::LOCAL_NAME)) {
915         const saml2::Assertion* token2 = dynamic_cast<const saml2::Assertion*>(&xmlObject);
916         if (token2) {
917             auto_ptr_char assertingParty(entity ? entity->getEntityID() : nullptr);
918             const vector<saml2::AttributeStatement*>& statements = token2->getAttributeStatements();
919             for (indirect_iterator<vector<saml2::AttributeStatement*>::const_iterator> s = make_indirect_iterator(statements.begin());
920                     s != make_indirect_iterator(statements.end()); ++s) {
921                 m_impl->extractAttributes(application, assertingParty.get(), relyingParty, *s, attributes);
922                 // Handle EncryptedAttributes inline so we have access to the role descriptor.
923                 const vector<saml2::EncryptedAttribute*>& encattrs = const_cast<const saml2::AttributeStatement&>(*s).getEncryptedAttributes();
924                 for_each(
925                     make_indirect_iterator(encattrs.begin()), make_indirect_iterator(encattrs.end()),
926                     boost::bind(extractEncrypted, this, boost::ref(application), issuer, _1, boost::ref(attributes))
927                     );
928             }
929             return;
930         }
931
932         const saml1::Assertion* token1 = dynamic_cast<const saml1::Assertion*>(&xmlObject);
933         if (token1) {
934             auto_ptr_char assertingParty(entity ? entity->getEntityID() : nullptr);
935             const vector<saml1::AttributeStatement*>& statements = token1->getAttributeStatements();
936             for_each(make_indirect_iterator(statements.begin()), make_indirect_iterator(statements.end()),
937                 boost::bind(extractV1Statement, m_impl.get(), boost::ref(application), assertingParty.get(), relyingParty, _1, boost::ref(attributes))
938                 );
939             return;
940         }
941
942         throw AttributeExtractionException("Unable to extract attributes, unknown object type.");
943     }
944
945     // Check for metadata.
946     if (XMLString::equals(xmlObject.getElementQName().getNamespaceURI(), samlconstants::SAML20MD_NS)) {
947         const RoleDescriptor* roleToExtract = dynamic_cast<const RoleDescriptor*>(&xmlObject);
948         const EntityDescriptor* entityToExtract = roleToExtract ? dynamic_cast<const EntityDescriptor*>(roleToExtract->getParent()) : nullptr;
949         if (!entityToExtract)
950             throw AttributeExtractionException("Unable to extract attributes, unknown metadata object type.");
951         const Extensions* ext = entityToExtract->getExtensions();
952         if (ext) {
953             m_impl->extractAttributes(
954                 application,
955                 dynamic_cast<const ObservableMetadataProvider*>(application.getMetadataProvider(false)),
956                 entityToExtract->getEntityID(),
957                 relyingParty,
958                 *ext,
959                 attributes
960                 );
961         }
962         const EntitiesDescriptor* group = dynamic_cast<const EntitiesDescriptor*>(entityToExtract->getParent());
963         while (group) {
964             ext = group->getExtensions();
965             if (ext) {
966                 m_impl->extractAttributes(
967                     application,
968                     dynamic_cast<const ObservableMetadataProvider*>(application.getMetadataProvider(false)),
969                     nullptr,   // not an entity, so inline assertions won't be processed
970                     relyingParty,
971                     *ext,
972                     attributes
973                     );
974             }
975             group = dynamic_cast<const EntitiesDescriptor*>(group->getParent());
976         }
977         return;
978     }
979
980     // Check for attributes.
981     if (XMLString::equals(xmlObject.getElementQName().getLocalPart(), saml1::Attribute::LOCAL_NAME)) {
982         auto_ptr_char assertingParty(entity ? entity->getEntityID() : nullptr);
983         const saml2::Attribute* attr2 = dynamic_cast<const saml2::Attribute*>(&xmlObject);
984         if (attr2)
985             return m_impl->extractAttributes(application, assertingParty.get(), relyingParty, *attr2, attributes);
986
987         const saml1::Attribute* attr1 = dynamic_cast<const saml1::Attribute*>(&xmlObject);
988         if (attr1)
989             return m_impl->extractAttributes(application, assertingParty.get(), relyingParty, *attr1, attributes);
990
991         throw AttributeExtractionException("Unable to extract attributes, unknown object type.");
992     }
993
994     if (XMLString::equals(xmlObject.getElementQName().getLocalPart(), EncryptedAttribute::LOCAL_NAME)) {
995         const EncryptedAttribute* encattr = dynamic_cast<const EncryptedAttribute*>(&xmlObject);
996         if (encattr) {
997             const XMLCh* recipient = application.getXMLString("entityID").second;
998             CredentialResolver* cr = application.getCredentialResolver();
999             if (!cr) {
1000                 m_log.warn("found encrypted attribute, but no CredentialResolver was available");
1001                 return;
1002             }
1003
1004             try {
1005                 Locker credlocker(cr);
1006                 if (issuer) {
1007                     MetadataCredentialCriteria mcc(*issuer);
1008                     scoped_ptr<XMLObject> decrypted(encattr->decrypt(*cr, recipient, &mcc));
1009                     if (m_log.isDebugEnabled())
1010                         m_log.debugStream() << "decrypted Attribute: " << *decrypted << logging::eol;
1011                     return extractAttributes(application, issuer, *decrypted, attributes);
1012                 }
1013                 else {
1014                     scoped_ptr<XMLObject> decrypted(encattr->decrypt(*cr, recipient));
1015                     if (m_log.isDebugEnabled())
1016                         m_log.debugStream() << "decrypted Attribute: " << *decrypted << logging::eol;
1017                     return extractAttributes(application, issuer, *decrypted, attributes);
1018                 }
1019             }
1020             catch (std::exception& ex) {
1021                 m_log.error("caught exception decrypting Attribute: %s", ex.what());
1022                 return;
1023             }
1024         }
1025     }
1026
1027     // Check for NameIDs.
1028     const NameID* name2 = dynamic_cast<const NameID*>(&xmlObject);
1029     if (name2) {
1030         auto_ptr_char assertingParty(entity ? entity->getEntityID() : nullptr);
1031         return m_impl->extractAttributes(application, assertingParty.get(), relyingParty, *name2, attributes);
1032     }
1033
1034     const NameIdentifier* name1 = dynamic_cast<const NameIdentifier*>(&xmlObject);
1035     if (name1) {
1036         auto_ptr_char assertingParty(entity ? entity->getEntityID() : nullptr);
1037         return m_impl->extractAttributes(application, assertingParty.get(), relyingParty, *name1, attributes);
1038     }
1039
1040     m_log.debug("unable to extract attributes, unknown XML object type: %s", xmlObject.getElementQName().toString().c_str());
1041 }
1042
1043 pair<bool,DOMElement*> XMLExtractor::background_load()
1044 {
1045     // Load from source using base class.
1046     pair<bool,DOMElement*> raw = ReloadableXMLFile::load();
1047
1048     // If we own it, wrap it.
1049     XercesJanitor<DOMDocument> docjanitor(raw.first ? raw.second->getOwnerDocument() : nullptr);
1050
1051     scoped_ptr<XMLExtractorImpl> impl(new XMLExtractorImpl(raw.second, m_log));
1052
1053     // If we held the document, transfer it to the impl. If we didn't, it's a no-op.
1054     impl->setDocument(docjanitor.release());
1055
1056     // Perform the swap inside a lock.
1057     if (m_lock)
1058         m_lock->wrlock();
1059     SharedLock locker(m_lock, false);
1060     m_impl.swap(impl);
1061
1062     return make_pair(false,(DOMElement*)nullptr);
1063 }