From 1cb3ecd9da530e5ee0260d740e23bcf31f1b9ce4 Mon Sep 17 00:00:00 2001
From: Ian Young <ian@iay.org.uk>
Date: Tue, 5 Apr 2011 14:44:19 +0000
Subject: [PATCH] Incremental prototyping of mda-based import from Edugate and
 eduGAIN, and

---
 build.xml               | 122 +++++++++++++++++++++++++++++++-----
 feds/edugain/import.xml | 133 ++++++++++++++++++++++++++++++++++++++++
 feds/edugate/import.xml |  64 +++++++++++++++++++
 feds/uk/collect.xml     |  81 ++++++++++++++++++++++++
 4 files changed, 384 insertions(+), 16 deletions(-)
 create mode 100644 feds/edugain/import.xml
 create mode 100644 feds/edugate/import.xml
 create mode 100644 feds/uk/collect.xml

diff --git a/build.xml b/build.xml
index 778ac4d2..9bd8d07e 100644
--- a/build.xml
+++ b/build.xml
@@ -42,14 +42,15 @@
 	-->
 	<property name="build.dir" value="${basedir}/build"/>
 	<property name="entities.dir" value="entities"/>
-	<property name="feds.dir" value="feds"/>
-	<property name="xml.dir" value="xml"/>
+	<property name="feds.dir" value="${basedir}/feds"/>
+	<property name="xml.dir" value="${basedir}/xml"/>
 
 	<!--
 		Location of externally supplied tool bundles.
 	-->
 	<property name="tools.dir" value="tools"/>
 	<property name="tools.ant" value="${tools.dir}/ant"/>
+	<property name="tools.mda" value="${tools.dir}/mda"/>
 	<property name="tools.mdcheck" value="${tools.dir}/mdcheck"/>
     <property name="tools.mdnorm" value="${tools.dir}/mdnorm"/>
 	<property name="tools.metadatatool" value="${tools.dir}/metadatatool"/>
@@ -486,6 +487,44 @@
             o="${test.prod.unsigned}" />
 	</target>
     
+	<!--
+	    ***************************
+	    ***                     ***
+	    ***   M D A   T O O L   ***
+	    ***                     ***
+	    ***************************
+	-->
+	
+	<!--
+	    Macro to run the mda (metadata aggregator CLI) tool.
+	-->
+    <macrodef name="MDA">
+    	<!-- Spring pipeline configuration file -->
+    	<attribute name="config"/>
+    	<attribute name="out"/>
+        <sequential>
+            <java classname="net.shibboleth.metadata.cli.SimpleCommandLine"
+                fork="true" failonerror="true" maxmemory="${java.max.memory}">
+                <classpath>
+                    <fileset dir="${tools.mda}/lib">
+                        <include name="*.jar"/>
+                    </fileset>
+                	<fileset dir="${tools.dir}/ukf-mda">
+                		<include name="*.jar"/>
+                	</fileset>
+                    <fileset dir="${tools.dir}/ukf-members">
+                        <include name="*.jar"/>
+                    </fileset>
+                </classpath>
+                <jvmarg value="-Djava.endorsed.dirs=${tools.mda}/lib/endorsed"/>
+            	<jvmarg value="-Dlogback.configurationFile=${tools.dir}/ukf-mda/logback.xml"/>
+            	<jvmarg value="-Dbasedir=${basedir}"/>
+            	<arg value="file:@{config}"/>
+            	<arg value="file:@{out}"/>
+            </java>
+        </sequential>
+    </macrodef>
+    
     <!--
         *********************************
         ***                           ***
@@ -598,7 +637,7 @@
 				<args/>
 				<arg value="--validateSchema"/>
 				<arg value="--schemaDirectory"/>
-				<arg value="${xml.dir}"/>
+				<arg value="${xml.dir}"/>
 				<arg value="--inFile"/>
 				<arg value="${xml.dir}/@{i}"/>
 			</java>
@@ -953,12 +992,7 @@
         <get src="${edugate.metadata.location}" dest="${xml.dir}/temp.xml"/>
         
         <!--
-            Verify using metadatatool.
-        -->
-        <!-- <MDT.VFY.uk i="temp.xml"/> -->
-
-        <!--
-            Verify using xmlsectool.
+            Verify signature and schema using xmlsectool.
         -->
         <XMLSECTOOL.VFY.edugate i="temp.xml"/>
         
@@ -966,7 +1000,7 @@
             Run checking rulesets.
         -->
         <CHECK.edugate i="${xml.dir}/temp.xml"/>
-        
+    	
         <!--
             Delete the temporary file.
         -->
@@ -975,6 +1009,22 @@
         <echo>Verification completed.</echo>
     </target>
 
+    <target name="fed.edugate.import">
+        <echo>Importing Edugate metadata from ${edugate.metadata.location}</echo>
+        
+        <!--
+            Run the aggregator CLI to import the metadata.
+        -->
+        <MDA config="${edugate.dir}/import.xml" out="${edugate.dir}/imported.xml"/>
+        
+        <!--
+            Run checking rulesets.
+        -->
+        <CHECK.edugate i="${edugate.dir}/imported.xml"/>
+        
+        <echo>Import completed.</echo>
+    </target>
+    
 	<!--
 	    *************************************************
 	    ***                                           ***
@@ -1010,11 +1060,6 @@
         <delete file="${xml.dir}/temp.xml" quiet="true" verbose="false"/>
         <get src="${incommon.metadata.location}" dest="${xml.dir}/temp.xml"/>
         
-        <!--
-            Verify using metadatatool.
-        -->
-        <!-- <MDT.VFY.uk i="temp.xml"/> -->
-
         <!--
             Verify using xmlsectool.
         -->
@@ -1055,7 +1100,7 @@
                 <args>
                     <arg value="--verifySignature"/>
                     <arg value="--certificate"/>
-                    <arg value="${edugate.metadata.cert}"/>
+                    <arg value="${edugain.metadata.cert}"/>
                     <!--
                     <arg value="- -quiet"/>
                     -->
@@ -1071,8 +1116,13 @@
         
         <!--
             Verify using xmlsectool.
+            
+            Don't do this for now.  Although the signature verifies, the
+            file as a whole is often not schema-valid.
         -->
+        <!--
         <XMLSECTOOL.VFY.edugain i="temp.xml"/>
+        -->
         
         <!--
             Run UK checking rulesets.
@@ -1087,6 +1137,46 @@
         <echo>Verification completed.</echo>
 	</target>
 	
+    <target name="fed.edugain.import">
+        <echo>Importing eduGAIN metadata from ${edugain.metadata.location}</echo>
+        
+        <!--
+            Run the aggregator CLI to import the metadata.
+        -->
+        <MDA config="${edugain.dir}/import.xml" out="${edugain.dir}/imported.xml"/>
+        
+        <!--
+            Run checking rulesets.
+        -->
+        <CHECK.edugate i="${edugain.dir}/imported.xml"/>
+        
+        <echo>Import completed.</echo>
+    </target>
+    
+	<!--
+	    *************************************************
+	    ***                                           ***
+	    ***   U K   F E D E R A T I O N   R E D U X   ***
+	    ***                                           ***
+	    *************************************************
+	-->
+	
+    <target name="fed.uk.collect">
+        <echo>Collecting UK fragment files.</echo>
+        
+        <!--
+            Run the aggregator CLI to collect the metadata.
+        -->
+        <MDA config="${feds.dir}/uk/collect.xml" out="${feds.dir}/uk/collected.xml"/>
+        
+        <!--
+            Run checking rulesets.
+        -->
+        <CHECK.edugate i="${feds.dir}/uk/collected.xml"/>
+        
+        <echo>Collection completed.</echo>
+    </target>
+    
 	<!--
 		*******************************
 		***                         ***
diff --git a/feds/edugain/import.xml b/feds/edugain/import.xml
new file mode 100644
index 00000000..8b95c654
--- /dev/null
+++ b/feds/edugain/import.xml
@@ -0,0 +1,133 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+    Import pipeline for metadata from eduGAIN.
+-->
+<beans xmlns="http://www.springframework.org/schema/beans"
+    xmlns:util="http://www.springframework.org/schema/util"
+    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+    xsi:schemaLocation="
+        http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-3.0.xsd
+        http://www.springframework.org/schema/util http://www.springframework.org/schema/util/spring-util-3.0.xsd">
+    
+    <bean id="fetchEduGainExportAggregate" class="net.shibboleth.metadata.dom.stage.DomHttpSourceStage">
+        <property name="id" value="fetchEduGainExportAggregate"/>
+        <property name="parserPool">
+            <bean class="org.opensaml.util.xml.BasicParserPool" init-method="initialize">
+                <property name="ignoreComments" value="false"/>
+            </bean>
+        </property>
+        <property name="sourceUrl" value="http://mds.edugain.org"/>
+    </bean>
+    
+    <!--
+        eduGAIN signing certificate.
+    -->
+    <bean id="edugainSigningCertificate" class="edu.vt.middleware.crypt.util.CryptReader"
+        factory-method="readCertificate">
+        <constructor-arg>
+            <bean class="java.io.File">
+                <constructor-arg value="#{ systemProperties['basedir'] }/feds/edugain/edugain-signer.crt"/>
+            </bean>
+        </constructor-arg>
+    </bean>
+    
+    <bean id="edugainSigningKey" factory-bean="edugainSigningCertificate" factory-method="getPublicKey"/>
+    
+    <bean id="checkEdugainSignature" class="net.shibboleth.metadata.dom.stage.XMLSignatureValidationStage">
+        <property name="id" value="checkEdugainSignature"/>
+        <property name="verificationKey" ref="edugainSigningKey"/>
+        <property name="postProcessFilter">
+            <bean class="uk.org.ukfederation.mda.ErrorAnnouncingFilteringStrategy"/>
+        </property>
+    </bean>
+    
+    <bean id="disassemble" class="net.shibboleth.metadata.dom.saml.EntitiesDescriptorDisassemblerStage">
+        <property name="id" value="disassemble"/>
+    </bean>
+    
+    <util:list id="schemaResources">
+        <bean class="org.opensaml.util.resource.FilesystemResource">
+            <!-- no dependencies -->
+            <constructor-arg value="#{ systemProperties['basedir'] }/xml/xml.xsd"/>
+        </bean>
+        <bean class="org.opensaml.util.resource.FilesystemResource">
+            <!-- no dependencies -->
+            <constructor-arg value="#{ systemProperties['basedir'] }/xml/xmldsig-core-schema.xsd"/>
+        </bean>
+        <bean class="org.opensaml.util.resource.FilesystemResource">
+            <!-- dependency on xmldsig-core-schema -->
+            <constructor-arg value="#{ systemProperties['basedir'] }/xml/xenc-schema.xsd"/>
+        </bean>
+        <bean class="org.opensaml.util.resource.FilesystemResource">
+            <!-- dependencies on xmldsig-core-schema, xenc-schema -->
+            <constructor-arg value="#{ systemProperties['basedir'] }/xml/saml-schema-assertion-2.0.xsd"/>
+        </bean>
+        <bean class="org.opensaml.util.resource.FilesystemResource">
+            <!-- dependencies on xmldsig-core-schema, xenc-schema, saml-schema-assertion-2.0, xml -->
+            <constructor-arg value="#{ systemProperties['basedir'] }/xml/saml-schema-metadata-2.0.xsd"/>
+        </bean>
+        <bean class="org.opensaml.util.resource.FilesystemResource">
+            <!-- dependencies on xml, saml-schema-metadata-2.0 -->
+            <constructor-arg value="#{ systemProperties['basedir'] }/xml/saml-metadata-rpi-v1.0-wd05.xsd"/>
+        </bean>
+        <bean class="org.opensaml.util.resource.FilesystemResource">
+            <constructor-arg value="#{ systemProperties['basedir'] }/xml/shibboleth-metadata-1.0.xsd"/>
+        </bean>
+        <bean class="org.opensaml.util.resource.FilesystemResource">
+            <constructor-arg value="#{ systemProperties['basedir'] }/xml/sstc-metadata-attr.xsd"/>
+        </bean>
+        <bean class="org.opensaml.util.resource.FilesystemResource">
+            <constructor-arg value="#{ systemProperties['basedir'] }/xml/sstc-request-initiation.xsd"/>
+        </bean>
+        <bean class="org.opensaml.util.resource.FilesystemResource">
+            <constructor-arg value="#{ systemProperties['basedir'] }/xml/sstc-saml-idp-discovery.xsd"/>
+        </bean>
+        <bean class="org.opensaml.util.resource.FilesystemResource">
+            <constructor-arg value="#{ systemProperties['basedir'] }/xml/sstc-saml-metadata-algsupport-v1.0.xsd"/>
+        </bean>
+        <bean class="org.opensaml.util.resource.FilesystemResource">
+            <constructor-arg value="#{ systemProperties['basedir'] }/xml/sstc-saml-metadata-ui-v1.0.xsd"/>
+        </bean>
+        <bean class="org.opensaml.util.resource.FilesystemResource">
+            <constructor-arg value="#{ systemProperties['basedir'] }/xml/uk-fed-label.xsd"/>
+        </bean>
+        <!--
+            These appear to be redundant duplicates...
+            
+        <bean class="org.opensaml.util.resource.FilesystemResource">
+            <constructor-arg value="#{ systemProperties['basedir'] }/xml/sstc-saml-schema-assertion-2.0.xsd"/>
+        </bean>
+        <bean class="org.opensaml.util.resource.FilesystemResource">
+            <constructor-arg value="#{ systemProperties['basedir'] }/xml/sstc-saml-schema-metadata-2.0.xsd"/>
+        </bean>
+        -->
+    </util:list>
+    
+    <bean id="checkSchemas" class="net.shibboleth.metadata.dom.stage.XMLSchemaValidationStage">
+        <property name="id" value="checkSchemas"/>
+        <property name="schemaResources" ref="schemaResources"/>
+        <property name="postProcessFilter">
+            <bean class="uk.org.ukfederation.mda.ErrorAnnouncingFilteringStrategy"/>
+        </property>
+    </bean>
+    
+    <bean id="assemble" class="net.shibboleth.metadata.dom.saml.EntitiesDescriptorAssemblerStage">
+        <property name="id" value="assemble"/>
+    </bean>
+
+    <bean id="pipeline" class="net.shibboleth.metadata.pipeline.SimplePipeline">
+        <property name="id" value="pipeline"/>
+        <property name="stages">
+            <list>
+                <ref bean="fetchEduGainExportAggregate"/>
+                <ref bean="checkEdugainSignature"/>
+                <ref bean="disassemble"/>
+                <ref bean="checkSchemas"/>
+                <ref bean="assemble"/>
+            </list>
+        </property>
+    </bean>
+    
+    <bean id="serializer" class="net.shibboleth.metadata.dom.DomMetadataSerializer" />
+
+</beans>
diff --git a/feds/edugate/import.xml b/feds/edugate/import.xml
new file mode 100644
index 00000000..8b096da0
--- /dev/null
+++ b/feds/edugate/import.xml
@@ -0,0 +1,64 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+    Import pipeline for metadata from Edugate.
+-->
+<beans xmlns="http://www.springframework.org/schema/beans" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+    xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-3.0.xsd"> 
+
+    <!--
+        Fetch the Edugate metadata aggregate.  For the moment, use the production aggregate in lieu
+        of a designated export aggregate.
+    -->
+    <bean id="fetchEdugateExportAggregate" class="net.shibboleth.metadata.dom.stage.DomHttpSourceStage">
+        <property name="id" value="fetchEdugateExportAggregate"/>
+        <property name="parserPool">
+            <bean class="org.opensaml.util.xml.BasicParserPool" init-method="initialize">
+                <property name="ignoreComments" value="false"/>
+                <property name="ignoreElementContentWhitespace" value="false"/>
+            </bean>
+        </property>
+        <property name="sourceUrl" value="https://edugate.heanet.ie/edugate-metadata-signed.xml"/>        
+    </bean>
+    
+    <!--
+        Edugate federation signing certificate.
+    -->
+    <bean id="edugateSigningCertificate" class="edu.vt.middleware.crypt.util.CryptReader"
+        factory-method="readCertificate">
+        <constructor-arg>
+            <bean class="java.io.File">
+                <constructor-arg value="#{ systemProperties['basedir'] }/feds/edugate/metadata-signer.crt"/>
+            </bean>
+        </constructor-arg>
+    </bean>
+    
+    <bean id="edugateSigningKey" factory-bean="edugateSigningCertificate" factory-method="getPublicKey"/>
+    
+    <bean id="checkEdugateSignature" class="net.shibboleth.metadata.dom.stage.XMLSignatureValidationStage">
+        <property name="id" value="checkEdugateSignature"/>
+        <property name="verificationKey" ref="edugateSigningKey"/>
+    </bean>
+    
+    <bean id="disassemble" class="net.shibboleth.metadata.dom.saml.EntitiesDescriptorDisassemblerStage">
+        <property name="id" value="disassemble"/>
+    </bean>
+    
+    <bean id="assemble" class="net.shibboleth.metadata.dom.saml.EntitiesDescriptorAssemblerStage">
+        <property name="id" value="assemble"/>
+    </bean>
+
+    <bean id="pipeline" class="net.shibboleth.metadata.pipeline.SimplePipeline">
+        <property name="id" value="pipeline"/>
+        <property name="stages">
+            <list>
+                <ref bean="fetchEdugateExportAggregate"/>
+                <ref bean="checkEdugateSignature"/>
+                <ref bean="disassemble"/>
+                <ref bean="assemble"/>
+            </list>
+        </property>
+    </bean>
+    
+    <bean id="serializer" class="net.shibboleth.metadata.dom.DomMetadataSerializer" />
+
+</beans>
diff --git a/feds/uk/collect.xml b/feds/uk/collect.xml
new file mode 100644
index 00000000..fdbd6763
--- /dev/null
+++ b/feds/uk/collect.xml
@@ -0,0 +1,81 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+    Pipeline to collect UK-registered entities.
+-->
+<beans xmlns="http://www.springframework.org/schema/beans"
+    xmlns:util="http://www.springframework.org/schema/util"
+    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+    xsi:schemaLocation="
+        http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-3.0.xsd
+        http://www.springframework.org/schema/util http://www.springframework.org/schema/util/spring-util-3.0.xsd">
+
+    <util:map id="commonNamespaces" map-class="java.util.HashMap">
+        <entry key="ds"         value="http://www.w3.org/2000/09/xmldsig#"/>
+        <entry key="elab"       value="http://eduserv.org.uk/labels"/>
+        <entry key="idpdisc"    value="urn:oasis:names:tc:SAML:profiles:SSO:idp-discovery-protocol"/>
+        <entry key="init"       value="urn:oasis:names:tc:SAML:profiles:SSO:request-init"/>
+        <entry key="md"         value="urn:oasis:names:tc:SAML:2.0:metadata"/>
+        <entry key="mdrpi"      value="urn:oasis:names:tc:SAML:metadata:rpi"/>
+        <entry key="mdui"       value="urn:oasis:names:tc:SAML:metadata:ui"/>
+        <entry key="saml"       value="urn:oasis:names:tc:SAML:2.0:assertion"/>
+        <entry key="shibmd"     value="urn:mace:shibboleth:metadata:1.0"/>
+        <entry key="ukfedlabel" value="http://ukfederation.org.uk/2006/11/label"/>
+        <entry key="wayf"       value="http://sdss.ac.uk/2006/06/WAYF"/>
+        <entry key="xenc"       value="http://www.w3.org/2001/04/xmlenc#"/>
+        <entry key="xsi"        value="http://www.w3.org/2001/XMLSchema-instance"/>
+        <entry key="xsl"        value="http://www.w3.org/1999/XSL/Transform"/>
+    </util:map>
+
+    <bean id="filterDeletedEntities" class="uk.org.ukfederation.mda.XPathFilteringStrategy">
+        <constructor-arg name="expression" value="//ukfedlabel:DeletedEntity"/>
+        <constructor-arg name="prefixMappings" ref="commonNamespaces"/>
+    </bean>
+    
+    <bean id="fetchUkFragmentFiles" class="net.shibboleth.metadata.dom.stage.DomFilesystemSourceStage">
+        <property name="id" value="fetchUkFragmentFiles"/>
+        <property name="parserPool">
+            <bean class="org.opensaml.util.xml.BasicParserPool" init-method="initialize">
+                <property name="ignoreComments" value="false"/>
+                <property name="ignoreElementContentWhitespace" value="false"/>
+            </bean>
+        </property>
+        <property name="source">
+            <bean class="java.io.File">
+                <constructor-arg value="#{ systemProperties['basedir'] }/entities"/>
+            </bean>
+        </property>
+        <property name="sourceFileFilter">
+            <bean class="uk.org.ukfederation.mda.RegexFileFilter">
+                <constructor-arg value="uk\d{6}.xml"/>
+            </bean>
+        </property>
+        <property name="postProcessFilter" ref="filterDeletedEntities"/>
+    </bean>
+    
+    <!--
+    <bean id="members" class="uk.org.ukfederation.members.Members">
+        <constructor-arg>
+            <bean class="java.io.File">
+                <constructor-arg value="#{ systemProperties['basedir'] }/xml/members.xml"/>
+            </bean>
+        </constructor-arg>
+    </bean>
+    -->
+
+    <bean id="assemble" class="net.shibboleth.metadata.dom.saml.EntitiesDescriptorAssemblerStage">
+        <property name="id" value="assemble"/>
+    </bean>
+
+    <bean id="pipeline" class="net.shibboleth.metadata.pipeline.SimplePipeline">
+        <property name="id" value="pipeline"/>
+        <property name="stages">
+            <list>
+                <ref bean="fetchUkFragmentFiles"/>
+                <ref bean="assemble"/>
+            </list>
+        </property>
+    </bean>
+    
+    <bean id="serializer" class="net.shibboleth.metadata.dom.DomMetadataSerializer" />
+
+</beans>