Use the new CompositeStage to draw up common import actions into a si…

…ngle stage to simplify import flows. Move all validation steps inside the aggregator using XSLValidationStage. Move to lazy initialization of most Spring beans (which really means lazy _instantiation_). Add explicit init-method settings for most beans, as CompositeStage does not guarantee to do this on the fly (SimplePipeline did, but only as a convenience).
InCommon · May 25, 2011 · 78a1726 · 78a1726
1 parent fa4626d
commit 78a1726
Show file tree

Hide file tree

Showing 14 changed files with 348 additions and 67 deletions.
diff --git a/build.xml b/build.xml
@@ -571,6 +571,12 @@
                 	<fileset dir="${tools.dir}/ukf-mda">
                 		<include name="*.jar"/>
                 	</fileset>
+                	<fileset dir="${tools.dir}/xalan/lib">
+                		<include name="sdss-xalan-md-*.jar"/>
+                	</fileset>
+                    <fileset dir="${tools.dir}/xalan/lib">
+                        <include name="ukf-members-*.jar"/>
+                    </fileset>
                 </classpath>
                 <jvmarg value="-Djava.endorsed.dirs=${tools.mda}/lib/endorsed"/>
             	<jvmarg value="-Dlogback.configurationFile=${tools.dir}/ukf-mda/logback.xml"/>
@@ -1030,16 +1036,6 @@
         *********************************************
     -->
 
-    <macrodef name="CHECK.imported">
-        <attribute name="i"/>
-        <sequential>
-            <CHECK.std i="@{i}">
-                <arg value="${build.dir}/check_future.xsl"/>
-                <arg value="${build.dir}/check_saml2int.xsl"/>
-            </CHECK.std>
-        </sequential>
-    </macrodef>
-
     <!--
         ***************************
         ***                     ***
@@ -1053,7 +1049,6 @@
     <target name="flow.aaf.import">
         <echo>Running AAF import flow.</echo>
         <MDA config="${aaf.dir}/aaf-import.xml" out="${aaf.dir}/imported.xml"/>
-        <CHECK.imported i="${aaf.dir}/imported.xml"/>
         <echo>Completed AAF import flow.</echo>
     </target>
 
@@ -1070,7 +1065,6 @@
     <target name="flow.arnes.import">
         <echo>Running ARNES import flow.</echo>
         <MDA config="${arnes.dir}/arnes-import.xml" out="${arnes.dir}/imported.xml"/>
-        <CHECK.imported i="${arnes.dir}/imported.xml"/>
         <echo>Completed ARNES import flow.</echo>
     </target>
 
@@ -1087,7 +1081,6 @@
     <target name="flow.caf.import">
         <echo>Running CAF import flow.</echo>
         <MDA config="${caf.dir}/caf-import.xml" out="${caf.dir}/imported.xml"/>
-        <CHECK.imported i="${caf.dir}/imported.xml"/>
         <echo>Completed CAF import flow.</echo>
     </target>
 
@@ -1104,7 +1097,6 @@
 	<target name="flow.edugate.import">
         <echo>Running Edugate import flow.</echo>
         <MDA config="${edugate.dir}/edugate-import.xml" out="${edugate.dir}/imported.xml"/>
-        <CHECK.imported i="${edugate.dir}/imported.xml"/>
         <echo>Completed Edugate import flow.</echo>
     </target>
 
@@ -1121,7 +1113,6 @@
     <target name="flow.incommon.import">
         <echo>Running InCommon import flow.</echo>
         <MDA config="${incommon.dir}/incommon-import.xml" out="${incommon.dir}/imported.xml"/>
-        <CHECK.imported i="${incommon.dir}/imported.xml"/>
         <echo>Completed InCommon import flow.</echo>
     </target>
 
@@ -1138,7 +1129,6 @@
     <target name="flow.edugain.import">
         <echo>Running eduGAIN import flow.</echo>
         <MDA config="${edugain.dir}/edugain-import.xml" out="${edugain.dir}/imported.xml"/>
-        <CHECK.imported i="${edugain.dir}/imported.xml"/>
         <echo>Completed eduGAIN import flow.</echo>
     </target>
 
@@ -1155,7 +1145,6 @@
 	<target name="flow.swamid.import">
 		<echo>Running SWAMID import flow.</echo>
 		<MDA config="${swamid.dir}/swamid-import.xml" out="${swamid.dir}/imported.xml"/>
-		<CHECK.imported i="${swamid.dir}/imported.xml"/>
 		<echo>Completed SWAMID import flow.</echo>
 	</target>
 
@@ -1172,7 +1161,6 @@
     <target name="flow.switch.import">
         <echo>Running SWITCH import flow.</echo>
         <MDA config="${switch.dir}/switch-import.xml" out="${switch.dir}/imported.xml"/>
-        <CHECK.imported i="${switch.dir}/imported.xml"/>
         <echo>Completed SWITCH import flow.</echo>
     </target>
 
@@ -1186,17 +1174,7 @@
 
     <target name="flow.uk.collect">
         <echo>Collecting UK fragment files.</echo>
-
-        <!--
-            Run the aggregator CLI to collect the metadata.
-        -->
         <MDA config="${mdx.dir}/uk/uk-collect.xml" out="${mdx.dir}/uk/collected.xml"/>
-
-        <!--
-            Run checking rulesets.
-        -->
-        <CHECK i="${xml.dir}/${uk.master.file}" s="check.xsl"/>
-
         <echo>Collection completed.</echo>
     </target>
 

diff --git a/mdx/aaf/aaf-import.xml b/mdx/aaf/aaf-import.xml
@@ -22,10 +22,10 @@
                 <ref bean="fetchAafExportAggregate"/>
                 <ref bean="checkAafSignature"/>
                 <ref bean="errorAnnouncingFilter"/>
-                <ref bean="cleanImport"/>
                 <ref bean="disassemble"/>
-                <ref bean="checkSchemas"/>
-                <ref bean="errorAnnouncingFilter"/>
+
+                <ref bean="standardImportActions"/>
+
                 <ref bean="assemble"/>
             </list>
         </property>

diff --git a/mdx/arnes/arnes-import.xml b/mdx/arnes/arnes-import.xml
@@ -22,10 +22,10 @@
                 <ref bean="fetchArnesExportAggregate"/>
                 <ref bean="checkArnesSignature"/>
                 <ref bean="errorAnnouncingFilter"/>
-                <ref bean="cleanImport"/>
                 <ref bean="disassemble"/>
-                <ref bean="checkSchemas"/>
-                <ref bean="errorAnnouncingFilter"/>
+
+                <ref bean="standardImportActions"/>
+
                 <ref bean="assemble"/>
             </list>
         </property>

diff --git a/mdx/caf/caf-import.xml b/mdx/caf/caf-import.xml
@@ -22,10 +22,10 @@
                 <ref bean="fetchCafExportAggregate"/>
                 <ref bean="checkCafSignature"/>
                 <ref bean="errorAnnouncingFilter"/>
-                <ref bean="cleanImport"/>
                 <ref bean="disassemble"/>
-                <ref bean="checkSchemas"/>
-                <ref bean="errorAnnouncingFilter"/>
+
+                <ref bean="standardImportActions"/>
+
                 <ref bean="assemble"/>
             </list>
         </property>

diff --git a/mdx/common-beans.xml b/mdx/common-beans.xml
@@ -9,12 +9,18 @@
         http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-3.0.xsd
         http://www.springframework.org/schema/util http://www.springframework.org/schema/util/spring-util-3.0.xsd">
 
+    <!--
+        Import beans related to metadata validation.
+    -->
+    <import resource="validation-beans.xml"/>
+
     <!--
         parserPool
         
         A pre-configured parser pool for use by source stages.
     -->
-    <bean id="parserPool" class="org.opensaml.util.xml.BasicParserPool" init-method="initialize">
+    <bean id="parserPool" class="org.opensaml.util.xml.BasicParserPool"
+        init-method="initialize" lazy-init="true">
         <property name="ignoreComments" value="false"/>
         <property name="ignoreElementContentWhitespace" value="false"/>
     </bean>
@@ -101,7 +107,8 @@
         
         A pipeline stage that checks against all the common schemas, as above.
     -->
-    <bean id="checkSchemas" class="net.shibboleth.metadata.dom.XMLSchemaValidationStage">
+    <bean id="checkSchemas" class="net.shibboleth.metadata.dom.XMLSchemaValidationStage"
+        init-method="initialize" lazy-init="true">
         <property name="id" value="checkSchemas"/>
         <property name="schemaResources" ref="schemaResources"/>
     </bean>
@@ -112,7 +119,8 @@
         A pipeline stage that checks for any errors, and then announces
         and filters any encountered.
     -->
-    <bean id="errorAnnouncingFilter" class="uk.org.ukfederation.mda.ErrorAnnouncingFilteringStage">
+    <bean id="errorAnnouncingFilter" class="uk.org.ukfederation.mda.ErrorAnnouncingFilteringStage"
+        init-method="initialize" lazy-init="true">
         <property name="id" value="errorAnnouncingFilter"/>
     </bean>
 
@@ -123,7 +131,8 @@
         presented, for example by removing redundant attributes or elements which only have
         meaning when added by the UK federation registrar.
     -->
-    <bean id="cleanImport" class="net.shibboleth.metadata.dom.XSLTStage">
+    <bean id="cleanImport" class="net.shibboleth.metadata.dom.XSLTransformationStage"
+        init-method="initialize" lazy-init="true">
         <property name="id" value="cleanImport"/>
         <property name="xslResource">
             <bean class="org.opensaml.util.resource.FilesystemResource">
@@ -135,14 +144,45 @@
     <!--
         Basic EntitiesDescriptor disassembler pipeline stage.
     -->
-    <bean id="disassemble" class="net.shibboleth.metadata.dom.saml.EntitiesDescriptorDisassemblerStage">
+    <bean id="disassemble" class="net.shibboleth.metadata.dom.saml.EntitiesDescriptorDisassemblerStage"
+        init-method="initialize" lazy-init="true">
         <property name="id" value="disassemble"/>
     </bean>
 
+    <!--
+        Populate ItemId values from entities.
+    -->
+    <bean id="populateItemIds" class="net.shibboleth.metadata.dom.saml.EntityDescriptorItemIdPopulationStage"
+        init-method="initialize" lazy-init="true">
+        <property name="id" value="populateItemIds"/>
+    </bean>
+
+    <!--
+        standardImportActions
+        
+        Standard actions performed on any metadata import channel.  Assumes that the
+        collection has been acquired, had its signature validated, and disassembled into
+        individual entities.
+    -->
+    <bean id="standardImportActions" class="net.shibboleth.metadata.pipeline.CompositeStage"
+        init-method="initialize" lazy-init="true">
+        <property name="id" value="standardImportActions"/>
+        <property name="composedStages">
+            <list>
+                <ref bean="populateItemIds"/>
+                <ref bean="cleanImport"/>
+                <ref bean="checkSchemas"/>
+                <ref bean="CHECK.imported"/>
+                <ref bean="errorAnnouncingFilter"/>
+            </list>
+        </property>
+    </bean>
+
     <!--
         Basic EntitiesDescriptor assembler pipeline stage.
     -->
-    <bean id="assemble" class="net.shibboleth.metadata.dom.saml.EntitiesDescriptorAssemblerStage">
+    <bean id="assemble" class="net.shibboleth.metadata.dom.saml.EntitiesDescriptorAssemblerStage"
+        init-method="initialize" lazy-init="true">
         <property name="id" value="assemble"/>
     </bean>
 

diff --git a/mdx/edugain/edugain-import.xml b/mdx/edugain/edugain-import.xml
@@ -26,10 +26,10 @@
                 <ref bean="fetchEduGainExportAggregate"/>
                 <ref bean="checkEdugainSignature"/>
                 <ref bean="errorAnnouncingFilter"/>
-                <ref bean="cleanImport"/>
                 <ref bean="disassemble"/>
-                <ref bean="checkSchemas"/>
-                <ref bean="errorAnnouncingFilter"/>
+
+                <ref bean="standardImportActions"/>
+
                 <ref bean="assemble"/>
             </list>
         </property>

diff --git a/mdx/edugate/edugate-import.xml b/mdx/edugate/edugate-import.xml
@@ -22,10 +22,10 @@
                 <ref bean="fetchEdugateExportAggregate"/>
                 <ref bean="checkEdugateSignature"/>
                 <ref bean="errorAnnouncingFilter"/>
-                <ref bean="cleanImport"/>
                 <ref bean="disassemble"/>
-                <ref bean="checkSchemas"/>
-                <ref bean="errorAnnouncingFilter"/>
+
+                <ref bean="standardImportActions"/>
+
                 <ref bean="assemble"/>
             </list>
         </property>

diff --git a/mdx/incommon/incommon-import.xml b/mdx/incommon/incommon-import.xml
@@ -22,10 +22,10 @@
                 <ref bean="fetchInCommonExportAggregate"/>
                 <ref bean="checkInCommonSignature"/>
                 <ref bean="errorAnnouncingFilter"/>
-                <ref bean="cleanImport"/>
                 <ref bean="disassemble"/>
-                <ref bean="checkSchemas"/>
-                <ref bean="errorAnnouncingFilter"/>
+
+                <ref bean="standardImportActions"/>
+
                 <ref bean="assemble"/>
             </list>
         </property>

diff --git a/mdx/swamid/swamid-import.xml b/mdx/swamid/swamid-import.xml
@@ -22,10 +22,10 @@
                 <ref bean="fetchSwamidExportAggregate"/>
                 <ref bean="checkSwamidSignature"/>
                 <ref bean="errorAnnouncingFilter"/>
-                <ref bean="cleanImport"/>
                 <ref bean="disassemble"/>
-                <ref bean="checkSchemas"/>
-                <ref bean="errorAnnouncingFilter"/>
+
+                <ref bean="standardImportActions"/>
+
                 <ref bean="assemble"/>
             </list>
         </property>

diff --git a/mdx/switch/switch-import.xml b/mdx/switch/switch-import.xml
@@ -22,10 +22,10 @@
                 <ref bean="fetchSwitchExportAggregate"/>
                 <ref bean="checkSwitchSignature"/>
                 <ref bean="errorAnnouncingFilter"/>
-                <ref bean="cleanImport"/>
                 <ref bean="disassemble"/>
-                <ref bean="checkSchemas"/>
-                <ref bean="errorAnnouncingFilter"/>
+
+                <ref bean="standardImportActions"/>
+
                 <ref bean="assemble"/>
             </list>
         </property>

diff --git a/mdx/uk/uk-beans.xml b/mdx/uk/uk-beans.xml
@@ -10,13 +10,16 @@
         http://www.springframework.org/schema/util http://www.springframework.org/schema/util/spring-util-3.0.xsd">
 
     <!--
-        <bean id="members" class="uk.org.ukfederation.members.Members">
+        members
+        
+        This bean implements an API for access to the contents of the members.xml document.
+    -->
+    <bean id="members" class="uk.org.ukfederation.members.Members" lazy-init="true">
         <constructor-arg>
-        <bean class="java.io.File">
-        <constructor-arg value="#{ systemProperties['basedir'] }/xml/members.xml"/>
-        </bean>
+            <bean class="java.io.File">
+                <constructor-arg value="#{ systemProperties['basedir'] }/xml/members.xml"/>
+            </bean>
         </constructor-arg>
-        </bean>
-    -->
+    </bean>
 
 </beans>
diff --git a/mdx/uk/uk-collect.xml b/mdx/uk/uk-collect.xml
@@ -18,14 +18,23 @@
         Import UK-specific beans.
     -->
     <import resource="uk-beans.xml"/>
+
+    <!--
+        filterDeletedEntities
         
-    <bean id="filterDeletedEntities" class="uk.org.ukfederation.mda.XPathFilteringStage">
+        Remove entity items which have the ukfedlabel:DeletedEntity label on them.
+        This convention will be replaced at some point by the files not being available
+        for collection in the first place.
+    -->
+    <bean id="filterDeletedEntities" class="uk.org.ukfederation.mda.XPathFilteringStage"
+        init-method="initialize" lazy-init="true">
         <property name="id" value="filterDeletedEntities"/>
         <constructor-arg name="expression" value="//ukfedlabel:DeletedEntity"/>
         <constructor-arg name="prefixMappings" ref="commonNamespaces"/>
     </bean>
 
-    <bean id="fetchUkFragmentFiles" class="net.shibboleth.metadata.dom.DomFilesystemSourceStage">
+    <bean id="fetchUkFragmentFiles" class="net.shibboleth.metadata.dom.DomFilesystemSourceStage"
+        init-method="initialize" lazy-init="true">
         <property name="id" value="fetchUkFragmentFiles"/>
         <property name="parserPool" ref="parserPool"/>
         <property name="source">
@@ -47,8 +56,17 @@
                 <ref bean="fetchUkFragmentFiles"/>
                 <ref bean="filterDeletedEntities"/>
                 <ref bean="checkSchemas"/>
-                <ref bean="errorAnnouncingFilter"/>
+                <ref bean="CHECK.std"/>
+                <ref bean="errorAnnouncingFilter"/><!-- should be fatal -->
                 <ref bean="assemble"/>
+
+                <!--
+                    It's much faster to run this check on the EntitiesDescriptor because it
+                    loads up a copy of members.xml every time it is used.  This should be
+                    fixed to use a parameter.
+                -->
+                <ref bean="check_check"/>
+                <ref bean="errorAnnouncingFilter"/><!-- should be fatal -->
             </list>
         </property>
     </bean>