From 520b8608ad4f5e6998decee0e04b948fe7880cf9 Mon Sep 17 00:00:00 2001
From: Ian Young <ian@iay.org.uk>
Date: Tue, 11 Jun 2013 12:54:28 +0000
Subject: [PATCH] Make, sign and push a new "CDSALL" aggregate for specific use
 by the Central Discovery Service.

---
 build.xml              |  17 +++-
 mdx/common-beans.xml   |  44 +++++++++--
 mdx/strip-comments.xsl |  28 +++++++
 mdx/uk/generate.xml    | 174 ++++++++++++++++++++++++++++++++++++++++-
 mdx/uk/ns_norm_cds.xsl |  79 +++++++++++++++++++
 5 files changed, 330 insertions(+), 12 deletions(-)
 create mode 100644 mdx/strip-comments.xsl
 create mode 100644 mdx/uk/ns_norm_cds.xsl

diff --git a/build.xml b/build.xml
index b3ad069e..2c311e04 100644
--- a/build.xml
+++ b/build.xml
@@ -99,8 +99,8 @@
 	</taskdef>
 	
 	<!--
-		There are five separate processing "streams":  production, test, export,
-		fallback and "wayf".
+		There are six separate processing "streams":  production, test, export,
+		fallback, "wayf" and "cdsall".
 		
 		Each stream has its own unsigned file, as follows:
 	-->
@@ -109,6 +109,7 @@
 	<property name="md.export.unsigned" value="ukfederation-export-unsigned.xml"/>
 	<property name="md.back.unsigned"   value="ukfederation-back-unsigned.xml"/>
 	<property name="md.wayf.unsigned"   value="ukfederation-wayf-unsigned.xml"/>
+	<property name="md.cdsall.unsigned" value="ukfederation-cdsall-unsigned.xml"/>
 	
 	<!--
 		Each stream has its own signed file.
@@ -118,6 +119,7 @@
 	<property name="md.export.signed"   value="ukfederation-export.xml"/>
 	<property name="md.back.signed"     value="ukfederation-back.xml"/>
 	<property name="md.wayf.signed"     value="ukfederation-wayf.xml"/>
+	<property name="md.cdsall.signed"   value="ukfederation-cdsall.xml"/>
 	
 	<!--
 		Null device location.
@@ -272,6 +274,7 @@
 			<fileset dir="${xml.dir}">
 				<include name="${md.prod.signed}"/>
 				<include name="${md.wayf.signed}"/>
+				<include name="${md.cdsall.signed}"/>
 				<include name="${md.test.signed}"/>
 				<include name="${md.back.signed}"/>
 				<include name="${md.export.signed}"/>
@@ -313,6 +316,7 @@
 		<echo>Verifying metadata held at ${remote.url}</echo>
 		<VFY.remote i="${md.prod.signed}"/>
 		<VFY.remote i="${md.wayf.signed}"/>
+		<VFY.remote i="${md.cdsall.signed}"/>
 		<VFY.remote i="${md.test.signed}"/>
 		<VFY.remote i="${md.back.signed}"/>
 		<VFY.remote i="${md.export.signed}"/>
@@ -368,7 +372,7 @@
 		    to generate the following:
 		    
 		       production aggregate
-		       WAYF/DS aggregate
+		       WAYF/CDS aggregates
                test aggregate
 		       export aggregate
 		       fallback aggregate
@@ -381,6 +385,7 @@
 		-->
         <MDNORM i="${xml.dir}/${md.prod.unsigned}"/>
         <MDNORM i="${xml.dir}/${md.wayf.unsigned}"/>
+		<MDNORM.noblank i="${xml.dir}/${md.cdsall.unsigned}"/>
         <MDNORM i="${xml.dir}/${md.test.unsigned}"/>
         <MDNORM i="${xml.dir}/${md.export.unsigned}"/>
         <MDNORM i="${xml.dir}/${md.back.unsigned}"/>
@@ -745,6 +750,9 @@
 		<echo>Signing UK WAYF metadata.</echo>
 		<SIGN.uk i="${md.wayf.unsigned}" o="${md.wayf.signed}" digest="SHA-1"/>
 
+        <echo>Signing UK CDS full metadata.</echo>
+        <SIGN.uk i="${md.cdsall.unsigned}" o="${md.cdsall.signed}" digest="SHA-256"/>
+
 		<echo>Signing UK test metadata.</echo>
 		<SIGN.uk i="${md.test.unsigned}" o="${md.test.signed}" digest="SHA-256"/>
 
@@ -786,6 +794,9 @@
 		<echo>Verifying signed UK WAYF metadata.</echo>
 		<VFY.uk i="${md.wayf.signed}"/>
 
+        <echo>Verifying signed UK CDS full metadata.</echo>
+        <VFY.uk i="${md.cdsall.signed}"/>
+
 		<echo>Verifying signed UK test metadata.</echo>
 		<VFY.uk i="${md.test.signed}"/>
 
diff --git a/mdx/common-beans.xml b/mdx/common-beans.xml
index 907d4bf3..c3dd15e0 100644
--- a/mdx/common-beans.xml
+++ b/mdx/common-beans.xml
@@ -152,6 +152,17 @@
         <property name="collectionMergeStrategy" ref="deduplicateMergeStrategy"/>
     </bean>
     
+    <!--
+        PipelineDemultiplexerStage
+        
+        Parent for demultiplexer stages.  Defaults to waiting for pipelines
+        to complete.
+    -->
+    <bean id="PipelineDemultiplexerStage" abstract="true" parent="stage_parent"
+        class="net.shibboleth.metadata.pipeline.PipelineDemultiplexerStage"
+        p:waitingForPipelines="true"
+    />
+    
     <!--
         ***********************************************
         ***                                         ***
@@ -405,11 +416,11 @@
     </bean>
     
     <!--
-        *********************
-        ***               ***
-        ***   ( E N D )   ***
-        ***               ***
-        *********************
+        *************************************
+        ***                               ***
+        ***   D O M   U T I L I T I E S   ***
+        ***                               ***
+        *************************************
     -->
     
     <!--
@@ -531,6 +542,29 @@
         <property name="schemaResources" ref="schemaResources"/>
     </bean>
 
+    <!--
+        stripComments
+        
+        A pipeline stage that removes all XML comments from items.
+    -->
+    
+    <bean id="stripComments" parent="XSLTransformationStage"
+        p:id="stripComments">
+        <property name="xslResource">
+            <bean parent="ClasspathResource">
+                <constructor-arg value="strip-comments.xsl"/>
+            </bean>
+        </property>
+    </bean>
+    
+    <!--
+        *********************
+        ***               ***
+        ***   ( E N D )   ***
+        ***               ***
+        *********************
+    -->
+
     <!--
         cleanImport
         
diff --git a/mdx/strip-comments.xsl b/mdx/strip-comments.xsl
new file mode 100644
index 00000000..d42afa7e
--- /dev/null
+++ b/mdx/strip-comments.xsl
@@ -0,0 +1,28 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+	
+	strip-comments.xsl
+	
+	Remove all comment nodes from a document.
+	
+-->
+<xsl:stylesheet version="1.0"
+	xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
+	xmlns="urn:oasis:names:tc:SAML:2.0:metadata">
+
+	<!-- Force UTF-8 encoding for the output. -->
+	<xsl:output omit-xml-declaration="no" method="xml" encoding="UTF-8" indent="yes"/>
+
+	<!-- Copy text blocks and attributes unchanged. -->
+	<xsl:template match="text()|@*">
+		<xsl:copy/>
+	</xsl:template>
+	
+	<!-- Copy all elements from the input to the output, along with their attributes and contents. -->
+	<xsl:template match="*">
+		<xsl:copy>
+			<xsl:apply-templates select="node()|@*"/>
+		</xsl:copy>
+	</xsl:template>
+	
+</xsl:stylesheet>
diff --git a/mdx/uk/generate.xml b/mdx/uk/generate.xml
index 69755a83..7d811ec6 100644
--- a/mdx/uk/generate.xml
+++ b/mdx/uk/generate.xml
@@ -377,6 +377,159 @@
         </property>
     </bean>
     
+    <!--
+        ***************************************
+        ***                                 ***
+        ***   C D S   A G G R E G A T E S   ***
+        ***                                 ***
+        ***************************************
+    -->
+    
+    <!--
+        CDSStripUnwanted
+        
+        The CDS needs only a very restricted subset of
+        normal metadata in order to do its job.  This stage
+        removes everything it does not need.
+    -->
+    <bean id="CDSStripUnwanted" parent="CompositeStage"
+        p:id="CDSStripUnwanted">
+        <property name="composedStages">
+            <list>
+                <ref bean="stripComments"/>
+                
+                <ref bean="stripElabNamespace"/>
+                <ref bean="stripMdattrNamespace"/>
+                <ref bean="stripMdrpiNamespace"/>
+                <ref bean="stripUkfedlabelNamespace"/>
+                <ref bean="stripWayfNamespace"/>
+                
+                <bean id="stripAlgNamespace" parent="NamespaceStrippingStage"
+                    p:id="stripAlgNamespace" p:namespace-ref="alg_namespace"/>
+                
+                <bean id="stripInitNamespace" parent="NamespaceStrippingStage"
+                    p:id="stripInitNamespace" p:namespace-ref="init_namespace"/>
+                
+                <bean id="stripArtifactResolutionService" parent="ElementStrippingStage"
+                    p:id="stripArtifactResolutionService"
+                    p:elementName="ArtifactResolutionService"
+                    p:elementNamespace-ref="md_namespace"/>
+                
+                <bean id="stripAttributeAuthorityDescriptor" parent="ElementStrippingStage"
+                    p:id="stripAttributeAuthorityDescriptor"
+                    p:elementName="AttributeAuthorityDescriptor"
+                    p:elementNamespace-ref="md_namespace"/>
+                
+                <bean id="stripAttributeConsumingService" parent="ElementStrippingStage"
+                    p:id="stripAttributeConsumingService"
+                    p:elementName="AttributeConsumingService"
+                    p:elementNamespace-ref="md_namespace"/>
+                
+                <bean id="stripContactPerson" parent="ElementStrippingStage"
+                    p:id="stripContactPerson"
+                    p:elementName="ContactPerson"
+                    p:elementNamespace-ref="md_namespace"/>
+                
+                <bean id="stripKeyDescriptor" parent="ElementStrippingStage"
+                    p:id="stripKeyDescriptor"
+                    p:elementName="KeyDescriptor"
+                    p:elementNamespace-ref="md_namespace"/>
+                
+                <bean id="stripManageNameIDService" parent="ElementStrippingStage"
+                    p:id="stripManageNameIDService"
+                    p:elementName="ManageNameIDService"
+                    p:elementNamespace-ref="md_namespace"/>
+                
+                <bean id="stripNameIDFormat" parent="ElementStrippingStage"
+                    p:id="stripNameIDFormat"
+                    p:elementName="NameIDFormat"
+                    p:elementNamespace-ref="md_namespace"/>
+                
+                <bean id="stripSingleLogoutService" parent="ElementStrippingStage"
+                    p:id="stripSingleLogoutService"
+                    p:elementName="SingleLogoutService"
+                    p:elementNamespace-ref="md_namespace"/>
+                
+                <bean id="stripShibScope" parent="ElementStrippingStage"
+                    p:id="stripShibScope"
+                    p:elementName="Scope"
+                    p:elementNamespace-ref="shibmd_namespace"/>
+                
+                <!-- remove any now-empty Extensions elements -->
+                <ref bean="removeEmptyExtensions"/>
+            </list>
+        </property>
+    </bean>
+    
+    <bean id="CDSFinalise" parent="finalise_parent"
+        p:id="CDSFinalise">
+        <property name="transformParameters">
+            <map>
+                <entry key="extraText"
+                    value="Central Discovery Service metadata; not for end entity use"/>
+                <entry key="publisher" value-ref="uk_federation_uri"/>
+            </map>
+        </property>
+    </bean>
+    
+    <bean id="CDSNormaliseNamespaces" parent="XSLTransformationStage"
+        p:id="CDSNormaliseNamespaces">
+        <property name="xslResource">
+            <bean parent="FilesystemResource">
+                <constructor-arg value="#{ systemProperties['basedir'] }/mdx/uk/ns_norm_cds.xsl"/>
+            </bean>
+        </property>
+    </bean>
+
+    <!--
+        *******************************************
+        ***                                     ***
+        ***   C D S A L L   A G G R E G A T E   ***
+        ***                                     ***
+        *******************************************
+    -->
+    
+    <!--
+        Entities in the CDSALL aggregate are restricted to those entities registered by the
+        UK federation plus all identity providers from whatever source.
+    -->
+    <bean id="CDSAllSelector" class="net.shibboleth.metadata.dom.XPathItemSelectionStrategy">
+        <constructor-arg value="/md:EntityDescriptor[md:IDPSSODescriptor or
+            md:Extensions/mdrpi:RegistrationInfo/@registrationAuthority = 'http://ukfederation.org.uk']"/>
+        <constructor-arg ref="commonNamespaces"/>
+    </bean>
+    
+    <bean id="serializeCDSAllAggregate" parent="SerializationStage"
+        p:id="serializeCDSAllAggregate">
+        <property name="outputFile">
+            <bean class="java.io.File">
+                <constructor-arg value="#{ systemProperties['basedir'] }/xml/ukfederation-cdsall-unsigned.xml"/>
+            </bean>
+        </property>
+    </bean>
+    
+    <bean id="CDSAllPipeline" parent="SimplePipeline"
+        p:id="CDSAllPipeline">
+        <property name="stages">
+            <list>
+                <!-- make an aggregate first so that we're only traversing one item -->
+                <ref bean="uk_assemble"/>
+                
+                <!-- remove many things that the CDS doesn't look at -->
+                <ref bean="CDSStripUnwanted"/>
+                
+                <ref bean="CDSFinalise"/>
+                <ref bean="CDSNormaliseNamespaces"/>
+                
+                <!-- schema validity check MUST pass -->
+                <ref bean="checkSchemas"/>
+                <ref bean="errorTerminatingFilter"/>
+                
+                <ref bean="serializeCDSAllAggregate"/>
+            </list>
+        </property>
+    </bean>
+    
     <!--
         ***********************************************
         ***                                         ***
@@ -598,8 +751,7 @@
                     metadata as closely as possible, so the fork must happen before
                     too many UK-specific transformations are performed.
                 -->
-                <bean id="registrarDemux" parent="stage_parent"
-                    class="net.shibboleth.metadata.pipeline.PipelineDemultiplexerStage"
+                <bean id="registrarDemux" parent="PipelineDemultiplexerStage"
                     p:id="registrarDemux">
                     <property name="pipelineAndSelectionStrategies">
                         <list>
@@ -660,8 +812,7 @@
                 <!--
                     Fork into new pipelines for the production, fallback and WAYF aggregates.
                 -->
-                <bean id="productionDemux" parent="stage_parent"
-                    class="net.shibboleth.metadata.pipeline.PipelineDemultiplexerStage"
+                <bean id="productionDemux" parent="PipelineDemultiplexerStage"
                     p:id="productionDemux">
                     <property name="pipelineAndSelectionStrategies">
                         <list>
@@ -717,6 +868,21 @@
                     and pre-production-status MDX relationships.
                 -->
                 
+                <!--
+                    Fork into new pipelines for additional aggregates.
+                -->
+                <bean id="preProductionDemux" parent="PipelineDemultiplexerStage"
+                    p:id="preProductionDemux">
+                    <property name="pipelineAndSelectionStrategies">
+                        <list>
+                            <bean class="net.shibboleth.utilities.java.support.collection.Pair">
+                                <constructor-arg ref="CDSAllPipeline"/>
+                                <constructor-arg ref="CDSAllSelector"/>
+                            </bean>
+                        </list>
+                    </property>
+                </bean>
+                
                 <!-- pipeline continues to generate test aggregate -->
                 <ref bean="uk_testPipeline"/>
 
diff --git a/mdx/uk/ns_norm_cds.xsl b/mdx/uk/ns_norm_cds.xsl
new file mode 100644
index 00000000..77c06d1e
--- /dev/null
+++ b/mdx/uk/ns_norm_cds.xsl
@@ -0,0 +1,79 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+	ns_norm_cds.xsl
+	
+	Normalise the namespaces in a metadata file for publication to the UKf CDS.
+	
+	Author: Ian A. Young <ian@iay.org.uk>
+
+-->
+<xsl:stylesheet version="1.0"
+	xmlns:alg="urn:oasis:names:tc:SAML:metadata:algsupport"
+	xmlns:ds="http://www.w3.org/2000/09/xmldsig#"
+	xmlns:elab="http://eduserv.org.uk/labels"
+	xmlns:idpdisc="urn:oasis:names:tc:SAML:profiles:SSO:idp-discovery-protocol"
+	xmlns:init="urn:oasis:names:tc:SAML:profiles:SSO:request-init"
+	xmlns:md="urn:oasis:names:tc:SAML:2.0:metadata"
+	xmlns:mdattr="urn:oasis:names:tc:SAML:metadata:attribute"
+	xmlns:mdrpi="urn:oasis:names:tc:SAML:metadata:rpi"
+	xmlns:mdui="urn:oasis:names:tc:SAML:metadata:ui"
+	xmlns:saml="urn:oasis:names:tc:SAML:2.0:assertion"
+	xmlns:shibmd="urn:mace:shibboleth:metadata:1.0"
+	xmlns:ukfedlabel="http://ukfederation.org.uk/2006/11/label"
+	xmlns:wayf="http://sdss.ac.uk/2006/06/WAYF"
+	
+	exclude-result-prefixes="alg ds elab init md mdattr saml shibmd ukfedlabel wayf xsi"
+	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+	xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
+	xmlns="urn:oasis:names:tc:SAML:2.0:metadata">
+
+
+	<!--
+		Import templates for basic normalisation.
+	-->
+	<xsl:import href="../ns_norm.xsl"/>
+	
+
+	<!--
+		Force UTF-8 encoding for the output.
+	-->
+	<xsl:output omit-xml-declaration="no" method="xml" encoding="UTF-8"/>
+	
+
+	<!--
+		*******************************************
+		***                                     ***
+		***   D O C U M E N T   E L E M E N T   ***
+		***                                     ***
+		*******************************************
+	-->
+	
+	
+	<!--
+		We need to handle the document element specially in order to arrange
+		for all appropriate namespace prefix definitions to appear on it.
+		
+		There are only two possible document elements in SAML metadata.
+	-->
+	
+	
+	<!--
+		Document element is <EntityDescriptor>.
+	-->
+	<xsl:template match="/md:EntityDescriptor">
+		<EntityDescriptor>
+			<xsl:apply-templates select="node()|@*"/>
+		</EntityDescriptor>
+	</xsl:template>
+	
+	<!--
+		Document element is <EntitiesDescriptor>.
+	-->
+	<xsl:template match="/md:EntitiesDescriptor">
+		<EntitiesDescriptor>
+			<xsl:apply-templates select="node()|@*"/>
+		</EntitiesDescriptor>
+	</xsl:template>
+	
+</xsl:stylesheet>