New metadata aggregator build:

CLI now takes file path, not URI: switch back to stock CLI CLI now takes pipeline name, not output file: rework all flows to have explicit serialization stages.
InCommon · Jun 21, 2011 · 38f0960 · 38f0960
1 parent d721b0c
commit 38f0960
Showing 14 changed files with 197 additions and 24 deletions.
diff --git a/build.xml b/build.xml
@@ -568,10 +568,10 @@
     <macrodef name="MDA">
     	<!-- Spring pipeline configuration file -->
     	<attribute name="config"/>
-    	<!-- output file -->
-    	<attribute name="out"/>
+    	<!-- pipeline bean ID -->
+    	<attribute name="pipeline"/>
         <sequential>
-            <java classname="uk.org.ukfederation.mda.SimpleCommandLine"
+            <java classname="net.shibboleth.metadata.cli.SimpleCommandLine"
                 fork="true" failonerror="true" maxmemory="${java.max.memory}">
                 <classpath>
                     <fileset dir="${tools.mda}/lib">
@@ -591,7 +591,7 @@
             	<jvmarg value="-Dlogback.configurationFile=${tools.dir}/ukf-mda/logback.xml"/>
             	<jvmarg value="-Dbasedir=${basedir}"/>
             	<arg value="@{config}"/>
-            	<arg value="@{out}"/>
+            	<arg value="@{pipeline}"/>
             </java>
         </sequential>
     </macrodef>
@@ -1047,6 +1047,19 @@
         *********************************************
     -->
 
+	<!--
+	    Aggregator based work is divided into channels, each of
+	    which lives in a directory under /mdx/.  Channels are
+	    mostly named country-source, but there are exceptions
+	    to this rule.  Some of the exceptions may be rationalised
+	    in the future.
+	    
+	    Each channel can have a number of verbs which can be
+	    executed.  The convention is that the verb names a
+	    configuration file, and that the pipeline to execute
+	    is named as channel.pipeline.
+	-->
+
 	<!--
 	    CHANNEL.do
 	    
@@ -1055,11 +1068,10 @@
     <macrodef name="CHANNEL.do">
         <attribute name="channel"/><!-- channel name -->
     	<attribute name="verb"/><!-- verb to perform -->
-    	<attribute name="out"/><!-- output file name -->
         <sequential>
             <echo>Running @{channel} @{verb} flow.</echo>
             <MDA config="${mdx.dir}/@{channel}/@{verb}.xml"
-                out="${mdx.dir}/@{channel}/@{out}.xml"/>
+                pipeline="@{channel}.@{verb}"/>
             <echo>Completed @{channel} @{verb} flow.</echo>
         </sequential>
     </macrodef>
@@ -1072,7 +1084,7 @@
 	<macrodef name="CHANNEL.import">
 		<attribute name="channel"/><!-- channel name -->
 		<sequential>
-			<CHANNEL.do channel="@{channel}" verb="import" out="imported"/>
+			<CHANNEL.do channel="@{channel}" verb="import"/>
 		</sequential>
 	</macrodef>
 
@@ -1113,7 +1125,7 @@
     </target>
 
     <target name="flow.edugain.test.import">
-    	<CHANNEL.do channel="edugain" verb="test-import" out="test-imported"/>
+    	<CHANNEL.do channel="edugain" verb="test-import"/>
     </target>
 
 	<target name="flow.swamid.import">
@@ -1137,7 +1149,7 @@
 	-->
 
     <target name="flow.uk.collect">
-    	<CHANNEL.do channel="uk" verb="collect" out="collected"/>
+    	<CHANNEL.do channel="uk" verb="collect"/>
     </target>
 
 	<target name="flow.uk.pub.production">

diff --git a/mdx/aaf/import.xml b/mdx/aaf/import.xml
@@ -15,7 +15,19 @@
     -->
     <import resource="beans.xml"/>
 
-    <bean id="pipeline" class="net.shibboleth.metadata.pipeline.SimplePipeline">
+    <bean id="aaf.serializeImported" class="net.shibboleth.metadata.pipeline.SerializationStage"
+        init-method="initialize" lazy-init="true">
+        <property name="id" value="aaf.serializeImported"/>
+        <property name="serializer" ref="serializer"/>
+        <property name="outputFile">
+            <bean class="java.io.File">
+                <constructor-arg value="#{ systemProperties['basedir'] }/mdx/aaf/imported.xml"/>
+            </bean>
+        </property>
+    </bean>
+
+    <bean id="aaf.import" class="net.shibboleth.metadata.pipeline.SimplePipeline"
+        init-method="initialize" lazy-init="true">
         <property name="id" value="pipeline"/>
         <property name="stages">
             <list>
@@ -30,6 +42,8 @@
                 <ref bean="standardImportActions"/>
 
                 <ref bean="standardImportTail"/>
+
+                <ref bean="aaf.serializeImported"/>
             </list>
         </property>
     </bean>

diff --git a/mdx/arnes/import.xml b/mdx/arnes/import.xml
@@ -15,7 +15,19 @@
     -->
     <import resource="beans.xml"/>
 
-    <bean id="pipeline" class="net.shibboleth.metadata.pipeline.SimplePipeline">
+    <bean id="arnes.serializeImported" class="net.shibboleth.metadata.pipeline.SerializationStage"
+        init-method="initialize" lazy-init="true">
+        <property name="id" value="arnes.serializeImported"/>
+        <property name="serializer" ref="serializer"/>
+        <property name="outputFile">
+            <bean class="java.io.File">
+                <constructor-arg value="#{ systemProperties['basedir'] }/mdx/arnes/imported.xml"/>
+            </bean>
+        </property>
+    </bean>
+
+    <bean id="arnes.import" class="net.shibboleth.metadata.pipeline.SimplePipeline"
+        init-method="initialize" lazy-init="true">
         <property name="id" value="pipeline"/>
         <property name="stages">
             <list>
@@ -30,6 +42,8 @@
                 <ref bean="standardImportActions"/>
 
                 <ref bean="standardImportTail"/>
+
+                <ref bean="arnes.serializeImported"/>
             </list>
         </property>
     </bean>

diff --git a/mdx/caf/import.xml b/mdx/caf/import.xml
@@ -15,7 +15,19 @@
     -->
     <import resource="beans.xml"/>
 
-    <bean id="pipeline" class="net.shibboleth.metadata.pipeline.SimplePipeline">
+    <bean id="caf.serializeImported" class="net.shibboleth.metadata.pipeline.SerializationStage"
+        init-method="initialize" lazy-init="true">
+        <property name="id" value="caf.serializeImported"/>
+        <property name="serializer" ref="serializer"/>
+        <property name="outputFile">
+            <bean class="java.io.File">
+                <constructor-arg value="#{ systemProperties['basedir'] }/mdx/caf/imported.xml"/>
+            </bean>
+        </property>
+    </bean>
+
+    <bean id="caf.import" class="net.shibboleth.metadata.pipeline.SimplePipeline"
+        init-method="initialize" lazy-init="true">
         <property name="id" value="pipeline"/>
         <property name="stages">
             <list>
@@ -30,6 +42,8 @@
                 <ref bean="standardImportActions"/>
 
                 <ref bean="standardImportTail"/>
+
+                <ref bean="caf.serializeImported"/>
             </list>
         </property>
     </bean>

diff --git a/mdx/cz-eduid/import.xml b/mdx/cz-eduid/import.xml
@@ -15,13 +15,25 @@
     -->
     <import resource="beans.xml"/>
 
-    <bean id="pipeline" class="net.shibboleth.metadata.pipeline.SimplePipeline"
+    <bean id="cz-eduid.serializeImported" class="net.shibboleth.metadata.pipeline.SerializationStage"
+        init-method="initialize" lazy-init="true">
+        <property name="id" value="cz-eduid.serializeImported"/>
+        <property name="serializer" ref="serializer"/>
+        <property name="outputFile">
+            <bean class="java.io.File">
+                <constructor-arg value="#{ systemProperties['basedir'] }/mdx/cz-eduid/imported.xml"/>
+            </bean>
+        </property>
+    </bean>
+
+    <bean id="cz-eduid.import" class="net.shibboleth.metadata.pipeline.SimplePipeline"
         init-method="initialize" lazy-init="true">
         <property name="id" value="pipeline"/>
         <property name="stages">
             <list>                
                 <ref bean="cz-eduid.exportedEntities"/>
                 <ref bean="standardImportTail"/>
+                <ref bean="cz-eduid.serializeImported"/>
             </list>
         </property>
     </bean>

diff --git a/mdx/edugain/import.xml b/mdx/edugain/import.xml
@@ -19,7 +19,19 @@
     -->
     <import resource="beans.xml"/>
 
-    <bean id="pipeline" class="net.shibboleth.metadata.pipeline.SimplePipeline">
+    <bean id="edugain.serializeImported" class="net.shibboleth.metadata.pipeline.SerializationStage"
+        init-method="initialize" lazy-init="true">
+        <property name="id" value="edugain.serializeImported"/>
+        <property name="serializer" ref="serializer"/>
+        <property name="outputFile">
+            <bean class="java.io.File">
+                <constructor-arg value="#{ systemProperties['basedir'] }/mdx/edugain/imported.xml"/>
+            </bean>
+        </property>
+    </bean>
+
+    <bean id="edugain.import" class="net.shibboleth.metadata.pipeline.SimplePipeline"
+        init-method="initialize" lazy-init="true">
         <property name="id" value="pipeline"/>
         <property name="stages">
             <list>
@@ -34,6 +46,8 @@
                 <ref bean="standardImportActions"/>
 
                 <ref bean="standardImportTail"/>
+
+                <ref bean="edugain.serializeImported"/>
             </list>
         </property>
     </bean>

diff --git a/mdx/edugain/test-import.xml b/mdx/edugain/test-import.xml
@@ -19,7 +19,19 @@
     -->
     <import resource="beans.xml"/>
 
-    <bean id="pipeline" class="net.shibboleth.metadata.pipeline.SimplePipeline">
+    <bean id="edugain.serializeTestImported" class="net.shibboleth.metadata.pipeline.SerializationStage"
+        init-method="initialize" lazy-init="true">
+        <property name="id" value="edugain.serializeTestImported"/>
+        <property name="serializer" ref="serializer"/>
+        <property name="outputFile">
+            <bean class="java.io.File">
+                <constructor-arg value="#{ systemProperties['basedir'] }/mdx/edugain/test-imported.xml"/>
+            </bean>
+        </property>
+    </bean>
+
+    <bean id="edugain.test-import" class="net.shibboleth.metadata.pipeline.SimplePipeline"
+        init-method="initialize" lazy-init="true">
         <property name="id" value="pipeline"/>
         <property name="stages">
             <list>
@@ -34,6 +46,8 @@
                 <ref bean="standardImportActions"/>
 
                 <ref bean="standardImportTail"/>
+
+                <ref bean="edugain.serializeTestImported"/>
             </list>
         </property>
     </bean>

diff --git a/mdx/ie-edugate/import.xml b/mdx/ie-edugate/import.xml
@@ -15,13 +15,25 @@
     -->
     <import resource="beans.xml"/>
 
-    <bean id="pipeline" class="net.shibboleth.metadata.pipeline.SimplePipeline"
+    <bean id="ie-edugate.serializeImported" class="net.shibboleth.metadata.pipeline.SerializationStage"
+        init-method="initialize" lazy-init="true">
+        <property name="id" value="ie-edugate.serializeImported"/>
+        <property name="serializer" ref="serializer"/>
+        <property name="outputFile">
+            <bean class="java.io.File">
+                <constructor-arg value="#{ systemProperties['basedir'] }/mdx/ie-edugate/imported.xml"/>
+            </bean>
+        </property>
+    </bean>
+
+    <bean id="ie-edugate.import" class="net.shibboleth.metadata.pipeline.SimplePipeline"
         init-method="initialize" lazy-init="true">
         <property name="id" value="pipeline"/>
         <property name="stages">
             <list>
                 <ref bean="ie-edugate.exportedEntities"/>
                 <ref bean="standardImportTail"/>
+                <ref bean="ie-edugate.serializeImported"/>
             </list>
         </property>
     </bean>

diff --git a/mdx/incommon/import.xml b/mdx/incommon/import.xml
@@ -15,7 +15,19 @@
     -->
     <import resource="beans.xml"/>
 
-    <bean id="pipeline" class="net.shibboleth.metadata.pipeline.SimplePipeline">
+    <bean id="incommon.serializeImported" class="net.shibboleth.metadata.pipeline.SerializationStage"
+        init-method="initialize" lazy-init="true">
+        <property name="id" value="incommon.serializeImported"/>
+        <property name="serializer" ref="serializer"/>
+        <property name="outputFile">
+            <bean class="java.io.File">
+                <constructor-arg value="#{ systemProperties['basedir'] }/mdx/incommon/imported.xml"/>
+            </bean>
+        </property>
+    </bean>
+
+    <bean id="incommon.import" class="net.shibboleth.metadata.pipeline.SimplePipeline"
+        init-method="initialize" lazy-init="true">
         <property name="id" value="pipeline"/>
         <property name="stages">
             <list>
@@ -30,6 +42,8 @@
                 <ref bean="standardImportActions"/>
 
                 <ref bean="standardImportTail"/>
+
+                <ref bean="incommon.serializeImported"/>
             </list>
         </property>
     </bean>

diff --git a/mdx/swamid/import.xml b/mdx/swamid/import.xml
@@ -15,7 +15,19 @@
     -->
     <import resource="beans.xml"/>
 
-    <bean id="pipeline" class="net.shibboleth.metadata.pipeline.SimplePipeline">
+    <bean id="swamid.serializeImported" class="net.shibboleth.metadata.pipeline.SerializationStage"
+        init-method="initialize" lazy-init="true">
+        <property name="id" value="swamid.serializeImported"/>
+        <property name="serializer" ref="serializer"/>
+        <property name="outputFile">
+            <bean class="java.io.File">
+                <constructor-arg value="#{ systemProperties['basedir'] }/mdx/swamid/imported.xml"/>
+            </bean>
+        </property>
+    </bean>
+
+    <bean id="swamid.import" class="net.shibboleth.metadata.pipeline.SimplePipeline"
+        init-method="initialize" lazy-init="true">
         <property name="id" value="pipeline"/>
         <property name="stages">
             <list>
@@ -30,6 +42,8 @@
                 <ref bean="standardImportActions"/>
 
                 <ref bean="standardImportTail"/>
+
+                <ref bean="swamid.serializeImported"/>
             </list>
         </property>
     </bean>

diff --git a/mdx/switch/import.xml b/mdx/switch/import.xml
@@ -15,7 +15,19 @@
     -->
     <import resource="beans.xml"/>
 
-    <bean id="pipeline" class="net.shibboleth.metadata.pipeline.SimplePipeline">
+    <bean id="switch.serializeImported" class="net.shibboleth.metadata.pipeline.SerializationStage"
+        init-method="initialize" lazy-init="true">
+        <property name="id" value="switch.serializeImported"/>
+        <property name="serializer" ref="serializer"/>
+        <property name="outputFile">
+            <bean class="java.io.File">
+                <constructor-arg value="#{ systemProperties['basedir'] }/mdx/switch/imported.xml"/>
+            </bean>
+        </property>
+    </bean>
+
+    <bean id="switch.import" class="net.shibboleth.metadata.pipeline.SimplePipeline"
+        init-method="initialize" lazy-init="true">
         <property name="id" value="pipeline"/>
         <property name="stages">
             <list>
@@ -30,6 +42,8 @@
                 <ref bean="standardImportActions"/>
 
                 <ref bean="standardImportTail"/>
+
+                <ref bean="switch.serializeImported"/>
             </list>
         </property>
     </bean>

diff --git a/mdx/uk-eduserv/import.xml b/mdx/uk-eduserv/import.xml
@@ -15,13 +15,25 @@
     -->
     <import resource="beans.xml"/>
 
-    <bean id="pipeline" class="net.shibboleth.metadata.pipeline.SimplePipeline"
+    <bean id="uk-eduserv.serializeImported" class="net.shibboleth.metadata.pipeline.SerializationStage"
+        init-method="initialize" lazy-init="true">
+        <property name="id" value="uk-eduserv.serializeImported"/>
+        <property name="serializer" ref="serializer"/>
+        <property name="outputFile">
+            <bean class="java.io.File">
+                <constructor-arg value="#{ systemProperties['basedir'] }/mdx/uk-eduserv/imported.xml"/>
+            </bean>
+        </property>
+    </bean>
+
+    <bean id="uk-eduserv.import" class="net.shibboleth.metadata.pipeline.SimplePipeline"
         init-method="initialize" lazy-init="true">
         <property name="id" value="pipeline"/>
         <property name="stages">
             <list>
                 <ref bean="uk-eduserv.exportedEntities"/>
                 <ref bean="standardImportTail"/>
+                <ref bean="uk-eduserv.serializeImported"/>
             </list>
         </property>
     </bean>

diff --git a/mdx/uk/beans.xml b/mdx/uk/beans.xml
@@ -18,7 +18,7 @@
         eligible file matches a particular regular expression.
     -->
     <bean id="uk.fetchFragmentFiles" class="net.shibboleth.metadata.dom.DomFilesystemSourceStage"
-        init-method="initialize">
+        init-method="initialize" lazy-init="true">
         <property name="id" value="fetchUkFragmentFiles"/>
         <property name="parserPool" ref="parserPool"/>
         <property name="source">
@@ -42,7 +42,7 @@
         for collection in the first place.
     -->
     <bean id="uk.filterDeletedEntities" class="net.shibboleth.metadata.dom.XPathFilteringStage"
-        init-method="initialize">
+        init-method="initialize" lazy-init="true">
         <property name="id" value="filterDeletedEntities"/>
         <constructor-arg name="expression" value="//ukfedlabel:DeletedEntity"/>
         <constructor-arg name="prefixMappings" ref="commonNamespaces"/>
@@ -64,7 +64,7 @@
         aggregate EntitiesDescriptor.
     -->
     <bean id="uk.processScopes" class="net.shibboleth.metadata.dom.XSLTransformationStage"
-        init-method="initialize">
+        init-method="initialize" lazy-init="true">
         <property name="id" value="processScopes"/>
         <property name="xslResource">
             <bean class="org.opensaml.util.resource.FilesystemResource">

diff --git a/mdx/uk/collect.xml b/mdx/uk/collect.xml
@@ -19,14 +19,27 @@
     -->
     <import resource="beans.xml"/>
 
-    <bean id="pipeline" class="net.shibboleth.metadata.pipeline.SimplePipeline">
+    <bean id="uk.serializeCollected" class="net.shibboleth.metadata.pipeline.SerializationStage"
+        init-method="initialize" lazy-init="true">
+        <property name="id" value="uk.serializeCollected"/>
+        <property name="serializer" ref="serializer"/>
+        <property name="outputFile">
+            <bean class="java.io.File">
+                <constructor-arg value="#{ systemProperties['basedir'] }/mdx/uk/collected.xml"/>
+            </bean>
+        </property>
+    </bean>
+
+    <bean id="uk.collect" class="net.shibboleth.metadata.pipeline.SimplePipeline"
+        init-method="initialize" lazy-init="true">
         <property name="id" value="pipeline"/>
         <property name="stages">
             <list>
                 <ref bean="uk.registeredEntities"/>
                 <ref bean="uk.assemble"/>
                 <ref bean="uk.addTrustRoots"/>
                 <ref bean="normaliseNamespaces"/>
+                <ref bean="uk.serializeCollected"/>
             </list>
         </property>
     </bean>