From d5156ab3027439813025900f220c40d70f2dddef Mon Sep 17 00:00:00 2001
From: David Shafer <dshafer@internet2.edu>
Date: Wed, 6 Feb 2019 16:24:19 -0600
Subject: [PATCH] Create new pipeline to produce all MDQ metadata

---
 build.xml                              |  68 +++----
 mdx/incommon/mdq-all-cloudhsm.xml      |  46 +++++
 mdx/incommon/mdq-all-localkey.xml      |  46 +++++
 mdx/incommon/mdq-all.xml               | 245 +++++++++++++++++++++++++
 mdx/incommon/mdq-combined-cloudhsm.xml | 224 ----------------------
 mdx/incommon/mdq-combined-localkey.xml | 223 ----------------------
 6 files changed, 367 insertions(+), 485 deletions(-)
 create mode 100644 mdx/incommon/mdq-all-cloudhsm.xml
 create mode 100644 mdx/incommon/mdq-all-localkey.xml
 create mode 100644 mdx/incommon/mdq-all.xml
 delete mode 100644 mdx/incommon/mdq-combined-cloudhsm.xml
 delete mode 100644 mdx/incommon/mdq-combined-localkey.xml

diff --git a/build.xml b/build.xml
index ba49a694..083c6411 100644
--- a/build.xml
+++ b/build.xml
@@ -2656,13 +2656,10 @@
     </target>
 
     <!--
-        inc.mdq-combined.localkey
+        inc.mdq.generate.all.cloudhsm
 
-        Generate all required metadata for the InCommon MDQ service
-        using the localkey configuration:
-
-          * Per-entity metadata
-          * InCommon IdP-only aggregate
+        Generate all metadata required for the InCommon federation MDQ service
+        using the AWS CloudHSM configuration.
 
         Properties:
 
@@ -2670,27 +2667,27 @@
         identifier) to set the password to access the key resource. If not set,
         this will be prompted for.
     -->
-    <target name="inc.mdq.combined.localkey" depends="get.sign.uk.keyPassword">
-      <property name="mda.mdq.input" value="${mda.inc.imported.xml}"/>
-      <property name="mda.sign.keyPassword" value="${sign.uk.keyPassword}"/>
-      <echo>Generating per-entity metadata in ${mda.mdq.output}</echo>
-      <echo>    (IdP-only aggregate in ${mda.mdq.output}/idps.xml)</echo>
-      <echo>    from unsigned aggregate in ${mda.mdq.input}</echo>
-      <delete dir="${mdq.output.dir}" quiet="true"/>
-      <mkdir dir="${mdq.output.dir}"/>
-      <mkdir dir="${mdq.output.dir}/idps"/>
-      <CHANNEL.do channel="incommon" verb="mdq-combined-localkey"/>
-      <echo>Generation complete.</echo>
+    <target name="inc.mdq.generate.all.cloudhsm" depends="get.sign.uk.keyPassword">
+        <property name="mda.classpath.extra" value="inc-mda-cloudhsm"/>
+        <property name="mda.jni.path" value="/opt/cloudhsm/lib"/>
+		<property name="mda.mdq.input" value="${mda.inc.imported.xml}"/>
+        <property name="mda.sign.keyHandle" value="${sign.keyHandle}"/>
+		<property name="mda.sign.keyPassword" value="${sign.uk.keyPassword}"/>
+        <property name="mda.sign.keyUser" value="${sign.keyUser}"/>
+		<echo>Generating MDQ metadata in ${mda.mdq.output}</echo>
+		<echo>    from unsigned aggregate in ${mda.mdq.input}</echo>
+		<delete dir="${mdq.output.dir}" quiet="true"/>
+		<mkdir dir="${mdq.output.dir}"/>
+		<mkdir dir="${mdq.output.dir}/idps"/>
+		<CHANNEL.do channel="incommon" verb="mdq-all-cloudhsm"/>
+		<echo>Generation complete.</echo>
     </target>
 
     <!--
-        inc.mdq.combined.cloudhsm
-
-        Generate all required metadata for the InCommon MDQ service
-        using the cloudhsm configuration:
+        inc.mdq.generate.all.localkey
 
-          * Per-entity metadata
-          * InCommon IdP-only aggregate
+        Generate all metadata required for the InCommon federation MDQ service
+        using the local key configuration.
 
         Properties:
 
@@ -2698,21 +2695,16 @@
         identifier) to set the password to access the key resource. If not set,
         this will be prompted for.
     -->
-    <target name="inc.mdq.combined.cloudhsm" depends="get.sign.uk.keyPassword">
-      <property name="mda.mdq.input" value="${mda.inc.imported.xml}"/>
-      <property name="mda.sign.keyHandle" value="${sign.keyHandle}"/>
-      <property name="mda.sign.keyUser" value="${sign.keyUser}"/>
-      <property name="mda.sign.keyPassword" value="${sign.keyPassword}"/>
-      <property name="mda.classpath.extra" value="inc-mda-cloudhsm"/>
-      <property name="mda.jni.path" value="/opt/cloudhsm/lib"/>
-      <echo>Generating per-entity metadata in ${mda.mdq.output}</echo>
-      <echo>    (IdP-only aggregate in ${mda.mdq.output}/idps.xml)</echo>
-      <echo>    from unsigned aggregate in ${mda.mdq.input}</echo>
-      <delete dir="${mdq.output.dir}" quiet="true"/>
-      <mkdir dir="${mdq.output.dir}"/>
-      <mkdir dir="${mdq.output.dir}/idps"/>
-      <CHANNEL.do channel="incommon" verb="mdq-combined-cloudhsm"/>
-      <echo>Generation complete.</echo>
+	<target name="inc.mdq.generate.all.localkey" depends="get.sign.uk.keyPassword">
+		<property name="mda.mdq.input" value="${mda.inc.imported.xml}"/>
+		<property name="mda.sign.keyPassword" value="${sign.uk.keyPassword}"/>
+		<echo>Generating MDQ metadata in ${mda.mdq.output}</echo>
+		<echo>    from unsigned aggregate in ${mda.mdq.input}</echo>
+		<delete dir="${mdq.output.dir}" quiet="true"/>
+		<mkdir dir="${mdq.output.dir}"/>
+		<mkdir dir="${mdq.output.dir}/idps"/>
+		<CHANNEL.do channel="incommon" verb="mdq-all-localkey"/>
+		<echo>Generation complete.</echo>
     </target>
 
     <!--
diff --git a/mdx/incommon/mdq-all-cloudhsm.xml b/mdx/incommon/mdq-all-cloudhsm.xml
new file mode 100644
index 00000000..e54e711a
--- /dev/null
+++ b/mdx/incommon/mdq-all-cloudhsm.xml
@@ -0,0 +1,46 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<beans xmlns="http://www.springframework.org/schema/beans" default-lazy-init="true"
+    xmlns:c="http://www.springframework.org/schema/c" xmlns:p="http://www.springframework.org/schema/p"
+    xmlns:util="http://www.springframework.org/schema/util" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+    xsi:schemaLocation="
+        http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd
+        http://www.springframework.org/schema/util http://www.springframework.org/schema/util/spring-util.xsd">
+
+    <!-- Import commonly used beans -->
+    <import resource="classpath:common-beans.xml" />
+
+    <!-- Import inc-mda beans (from import.xml) -->
+    <import resource="classpath:uk/org/iay/incommon/mda/beans.xml" />
+
+    <!-- Import channel-specific beans -->
+    <import resource="classpath:incommon/beans.xml" />
+
+    <!-- Import mdq-all beans -->
+    <import resource="classpath:incommon/mdq-all.xml" />
+
+    <!-- This bean MUST be called "conversionService" to work properly -->
+    <bean id="conversionService" class="org.springframework.context.support.ConversionServiceFactoryBean">
+        <property name="converters">
+            <set>
+                <bean class="net.shibboleth.ext.spring.config.DurationToLongConverter" />
+                <bean class="net.shibboleth.ext.spring.config.StringToIPRangeConverter" />
+                <bean class="net.shibboleth.ext.spring.config.BooleanToPredicateConverter" />
+                <bean class="net.shibboleth.ext.spring.config.StringBooleanToPredicateConverter" />
+                <bean class="net.shibboleth.ext.spring.config.StringToResourceConverter" />
+            </set>
+        </property>
+    </bean>
+
+    <bean id="privateKeyFactory" class="uk.org.iay.incommon.mda.cloudhsm.CaviumPrivateKeyFactoryBean"
+        p:username="${sign.keyUser}" p:password="${sign.keyPassword}" p:keyHandle="${sign.keyHandle}" />
+
+    <bean id="mdq-all-cloudhsm" parent="mda.SimplePipeline">
+        <property name="stages">
+            <list>
+                <ref bean="generateAll" />
+            </list>
+        </property>
+    </bean>
+
+</beans>
diff --git a/mdx/incommon/mdq-all-localkey.xml b/mdx/incommon/mdq-all-localkey.xml
new file mode 100644
index 00000000..d329dc75
--- /dev/null
+++ b/mdx/incommon/mdq-all-localkey.xml
@@ -0,0 +1,46 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<beans xmlns="http://www.springframework.org/schema/beans" default-lazy-init="true"
+    xmlns:c="http://www.springframework.org/schema/c" xmlns:p="http://www.springframework.org/schema/p"
+    xmlns:util="http://www.springframework.org/schema/util" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+    xsi:schemaLocation="
+        http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd
+        http://www.springframework.org/schema/util http://www.springframework.org/schema/util/spring-util.xsd">
+
+    <!-- Import commonly used beans -->
+    <import resource="classpath:common-beans.xml" />
+
+    <!-- Import inc-mda beans (from import.xml) -->
+    <import resource="classpath:uk/org/iay/incommon/mda/beans.xml" />
+
+    <!-- Import channel-specific beans -->
+    <import resource="classpath:incommon/beans.xml" />
+
+    <!-- Import mdq-all beans -->
+    <import resource="classpath:incommon/mdq-all.xml" />
+
+    <!-- This bean MUST be called "conversionService" to work properly -->
+    <bean id="conversionService" class="org.springframework.context.support.ConversionServiceFactoryBean">
+        <property name="converters">
+            <set>
+                <bean class="net.shibboleth.ext.spring.config.DurationToLongConverter" />
+                <bean class="net.shibboleth.ext.spring.config.StringToIPRangeConverter" />
+                <bean class="net.shibboleth.ext.spring.config.BooleanToPredicateConverter" />
+                <bean class="net.shibboleth.ext.spring.config.StringBooleanToPredicateConverter" />
+                <bean class="net.shibboleth.ext.spring.config.StringToResourceConverter" />
+            </set>
+        </property>
+    </bean>
+
+    <bean id="privateKeyFactory" parent="PrivateKeyFactoryBean" p:resource="${sign.keyResource}"
+        p:privateKeyPassword="${sign.keyPassword}" />
+
+    <bean id="mdq-all-localkey" parent="mda.SimplePipeline">
+        <property name="stages">
+            <list>
+                <ref bean="generateAll" />
+            </list>
+        </property>
+    </bean>
+
+</beans>
diff --git a/mdx/incommon/mdq-all.xml b/mdx/incommon/mdq-all.xml
new file mode 100644
index 00000000..ca85098a
--- /dev/null
+++ b/mdx/incommon/mdq-all.xml
@@ -0,0 +1,245 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<beans xmlns="http://www.springframework.org/schema/beans" default-lazy-init="true"
+    xmlns:c="http://www.springframework.org/schema/c" xmlns:p="http://www.springframework.org/schema/p"
+    xmlns:if="ant:if" xmlns:unless="ant:unless" xmlns:util="http://www.springframework.org/schema/util"
+    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+    xsi:schemaLocation="
+        http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd
+        http://www.springframework.org/schema/util http://www.springframework.org/schema/util/spring-util.xsd">
+
+    <!-- Import commonly used beans -->
+    <import resource="classpath:common-beans.xml" />
+
+    <!-- Import inc-mda beans (from import.xml) -->
+    <import resource="classpath:uk/org/iay/incommon/mda/beans.xml" />
+
+    <!-- Import channel-specific beans -->
+    <import resource="classpath:incommon/beans.xml" />
+
+    <!-- This bean MUST be called "conversionService" to work properly -->
+    <bean id="conversionService" class="org.springframework.context.support.ConversionServiceFactoryBean">
+        <property name="converters">
+            <set>
+                <bean class="net.shibboleth.ext.spring.config.DurationToLongConverter" />
+                <bean class="net.shibboleth.ext.spring.config.StringToIPRangeConverter" />
+                <bean class="net.shibboleth.ext.spring.config.BooleanToPredicateConverter" />
+                <bean class="net.shibboleth.ext.spring.config.StringBooleanToPredicateConverter" />
+                <bean class="net.shibboleth.ext.spring.config.StringToResourceConverter" />
+            </set>
+        </property>
+    </bean>
+
+    <!-- Predicate for matching everything -->
+    <bean id="matchEverything" class="com.google.common.base.Predicates" factory-method="alwaysTrue" />
+
+    <!-- Generate all required metadata for the InCommon MDQ service -->
+    <bean id="generateAll" parent="mda.CompositeStage">
+        <property name="composedStages">
+            <list>
+
+                <!-- Begin with the published aggregate -->
+                <ref bean="commonSourceStage" />
+
+                <!-- Fork pipelines for creating signed aggregates -->
+                <bean id="demux" parent="mda.PipelineDemultiplexerStage">
+
+                    <property name="pipelineAndSelectionStrategies">
+                        <list>
+
+                            <!-- Create the signed aggregate -->
+                            <bean class="net.shibboleth.utilities.java.support.collection.Pair">
+                                <constructor-arg ref="aggregatePipeline" />
+                                <constructor-arg ref="matchEverything" />
+                            </bean>
+
+                            <!-- Create the signed IdP-only aggregate -->
+                            <bean class="net.shibboleth.utilities.java.support.collection.Pair">
+                                <constructor-arg ref="idpPipeline" />
+                                <constructor-arg ref="idpSelectionStrategy" />
+                            </bean>
+
+                        </list>
+                    </property>
+
+                    <property name="waitingForPipelines" value="true" />
+
+                </bean>
+
+                <!-- Create per-entity metadata -->
+                <ref bean="perEntityStage" />
+
+            </list>
+        </property>
+    </bean>
+
+    <!-- Pipeline that produces the signed aggregate from a collection of entities -->
+    <bean id="aggregatePipeline" parent="mda.SimplePipeline">
+        <property name="stages">
+            <list>
+                <ref bean="aggregateStage" />
+            </list>
+        </property>
+    </bean>
+
+    <!-- Stage that produces the signed aggregate from a collection of entities -->
+    <bean id="aggregateStage" parent="mda.CompositeStage">
+        <property name="composedStages">
+            <list>
+
+                <!-- Perform common steps in constructing an output aggregate -->
+                <ref bean="commonOutputStage" />
+
+                <!-- Write the resulting aggregate to a file -->
+                <ref bean="aggregateSerializationStage" />
+
+            </list>
+        </property>
+    </bean>
+
+    <!-- Stage that writes the aggregate to a file -->
+    <bean id="aggregateSerializationStage" parent="mda.SerializationStage">
+        <property name="serializer" ref="serializer" />
+        <property name="outputFile">
+            <bean parent="File">
+                <constructor-arg value="${mdq.output}/all.xml" />
+            </bean>
+        </property>
+    </bean>
+
+    <!-- Output stage used by all MDQ aggregate pipelines -->
+    <bean id="commonOutputStage" parent="mda.CompositeStage">
+        <property name="composedStages">
+            <list>
+                <!-- Construct a new aggregate from the collection of entities -->
+                <bean id="inc.assemble" parent="mda.EntitiesDescriptorAssemblerStage">
+                    <property name="itemOrderingStrategy">
+                        <bean parent="inc.InCommonEntityOrderingStrategy" c:_-ref="us_incommon_registrar" />
+                    </property>
+                </bean>
+
+                <!-- Apply final tweaks to the aggregate -->
+                <bean id="finalise" parent="incommon_finalise_parent">
+                    <property name="transformParameters">
+                        <map>
+                            <entry key="extraText" value="Contains InCommon and eduGAIN metadata" />
+                            <entry key="publisher" value-ref="us_incommon_registrar" />
+                        </map>
+                    </property>
+                </bean>
+
+                <!-- Normalise the use of namespace prefixes in the resulting XML document. -->
+                <bean id="normalise" parent="mda.XSLTransformationStage"
+                    p:XSLResource="classpath:incommon/ns_norm_import.xsl" />
+
+                <!-- Sign the aggregate -->
+                <ref bean="signItems" />
+
+            </list>
+        </property>
+    </bean>
+
+    <!-- Source stage used by all MDQ pipelines -->
+    <bean id="commonSourceStage" parent="mda.CompositeStage">
+        <property name="composedStages">
+            <list>
+
+                <!-- Start with the signed production aggregate -->
+                <bean id="production_aggregate" parent="mda.DOMResourceSourceStage">
+                    <property name="parserPool" ref="parserPool" />
+                    <property name="DOMResource">
+                        <bean parent="FileSystemResource" c:_0="${mdq.input}" />
+                    </property>
+                </bean>
+
+                <!-- TODO: Check signature on source aggregate -->
+
+                <!-- Break down into individual entities. -->
+                <ref bean="disassemble" />
+
+                <!-- Include a default registrationAuthority for each entity. -->
+                <ref bean="us_incommon_default_regauth" />
+
+                <!-- Populate identifiers for future actions. -->
+                <ref bean="populateItemIds" />
+                <ref bean="populateRegistrationAuthorities" />
+
+                <!-- Discard entities in the InCommon entity blacklist. -->
+                <ref bean="remove_blacklisted_incommon_entities" />
+
+            </list>
+        </property>
+    </bean>
+
+    <!-- Pipeline that produces the IdP-only aggregate from a collection of IdP entities -->
+    <bean id="idpPipeline" parent="mda.SimplePipeline">
+        <property name="stages">
+            <list>
+
+                <!-- Perform common steps in constructing an output aggregate. -->
+                <ref bean="commonOutputStage" />
+
+                <!-- Write the resulting aggregate out to a file. -->
+                <ref bean="idpSerializationStage" />
+            </list>
+        </property>
+    </bean>
+
+    <!-- Selection strategy that selects IdP entities for the IdP-only aggregate -->
+    <bean id="idpSelectionStrategy" parent="mda.XPathItemSelectionStrategy">
+        <constructor-arg value="/md:EntityDescriptor[md:IDPSSODescriptor]" />
+        <constructor-arg ref="commonNamespaces" />
+    </bean>
+
+    <!-- Serialization stage that writes the IdP-only aggregate to a file -->
+    <bean id="idpSerializationStage" parent="mda.SerializationStage">
+        <property name="serializer" ref="serializer" />
+        <property name="outputFile">
+            <bean parent="File">
+                <constructor-arg value="${mdq.output}/idps/all.xml" />
+            </bean>
+        </property>
+    </bean>
+
+    <!-- Stage to produce signed per-entity metadata -->
+    <bean id="perEntityStage" parent="mda.CompositeStage">
+        <property name="composedStages">
+            <list>
+
+                <!-- Set ID, cacheDuration and validUntil attributes. -->
+                <bean parent="mda.GenerateIdStage" />
+                <bean parent="mda.SetValidUntilStage" p:validityDuration="P14D" />
+
+                <!-- Identity transform fixes signing issues. -->
+                <bean parent="mda.XSLTransformationStage" p:XSLResource="classpath:identity.xsl" />
+
+                <!-- Sign each item. -->
+                <ref bean="signItems" />
+
+                <!-- Write individual entity documents to files. -->
+                <bean id="write.perentity" parent="mda.MultiOutputSerializationStage">
+                    <property name="serializer" ref="serializer" />
+                    <property name="outputStrategy">
+                        <bean parent="mda.FilesInDirectoryMultiOutputStrategy" p:nameSuffix=".xml">
+                            <property name="directory">
+                                <bean parent="File" c:_="${mdq.output}" />
+                            </property>
+                            <property name="nameTransformer">
+                                <bean parent="mda.PathSegmentStringTransformer" />
+                            </property>
+                        </bean>
+                    </property>
+                </bean>
+
+            </list>
+        </property>
+    </bean>
+
+    <!-- Signs items using a privateKeyFactory defined elsewhere -->
+    <bean id="signItems" parent="mda.XMLSignatureSigningStage">
+        <property name="privateKey">
+            <ref bean="privateKeyFactory" />
+        </property>
+    </bean>
+
+</beans>
diff --git a/mdx/incommon/mdq-combined-cloudhsm.xml b/mdx/incommon/mdq-combined-cloudhsm.xml
deleted file mode 100644
index 6a54503d..00000000
--- a/mdx/incommon/mdq-combined-cloudhsm.xml
+++ /dev/null
@@ -1,224 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-    Generate and sign per-entity metadata using a local private key resource.
--->
-<beans xmlns="http://www.springframework.org/schema/beans"
-    default-lazy-init="true"
-    xmlns:c="http://www.springframework.org/schema/c"
-    xmlns:p="http://www.springframework.org/schema/p"
-    xmlns:util="http://www.springframework.org/schema/util"
-    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-    xsi:schemaLocation="
-        http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd
-        http://www.springframework.org/schema/util http://www.springframework.org/schema/util/spring-util.xsd">
-
-    <!--
-        Import commonly used beans.
-    -->
-    <import resource="classpath:common-beans.xml"/>
-
-    <!--
-        Import inc-mda beans. (from import.xml)
-    -->
-    <import resource="classpath:uk/org/iay/incommon/mda/beans.xml"/>
-
-    <!--
-        Import channel-specific beans.
-    -->
-    <import resource="classpath:incommon/beans.xml"/>
-
-    <!-- This bean MUST be called "conversionService" to work properly. -->
-    <bean id="conversionService" class="org.springframework.context.support.ConversionServiceFactoryBean">
-        <property name="converters">
-            <set>
-                <bean class="net.shibboleth.ext.spring.config.DurationToLongConverter" />
-                <bean class="net.shibboleth.ext.spring.config.StringToIPRangeConverter" />
-                <bean class="net.shibboleth.ext.spring.config.BooleanToPredicateConverter" />
-                <bean class="net.shibboleth.ext.spring.config.StringBooleanToPredicateConverter" />
-                <bean class="net.shibboleth.ext.spring.config.StringToResourceConverter" />
-            </set>
-        </property>
-    </bean>
-
-    <!--
-        ***********************************************************
-        ***                                                     ***
-        ***   C O M M O N   O U T P U T   P R O C E S S I N G   ***
-        ***                                                     ***
-        ***********************************************************
-    -->
-
-    <bean id="common.output" parent="mda.CompositeStage">
-        <property name="composedStages">
-            <list>
-                <!-- Construct a new aggregate from the collection of entities. -->
-                <bean id="inc.assemble" parent="mda.EntitiesDescriptorAssemblerStage">
-                    <property name="itemOrderingStrategy">
-                        <bean parent="inc.InCommonEntityOrderingStrategy"
-                            c:_-ref="us_incommon_registrar"/>
-                    </property>
-                </bean>
-
-                <!-- Apply final tweaks to the aggregate. -->
-                <bean id="finalise" parent="incommon_finalise_parent">
-                    <property name="transformParameters">
-                        <map>
-                            <entry key="extraText" value="Contains InCommon and eduGAIN metadata"/>
-                            <entry key="publisher" value-ref="us_incommon_registrar"/>
-                        </map>
-                    </property>
-                </bean>
-
-                <!-- Normalise the use of namespace prefixes in the resulting XML document. -->
-                <bean id="normalise" parent="mda.XSLTransformationStage"
-                    p:XSLResource="classpath:incommon/ns_norm_import.xsl"/>
-            </list>
-        </property>
-    </bean>
-
-    <!--
-        *****************************************
-        ***                                   ***
-        ***   I D P - O N L Y   O U T P U T   ***
-        ***                                   ***
-        *****************************************
-    -->
-
-    <!--
-        idp.serialize
-
-        Writes the IdP-only aggregate out to a file.
-    -->
-    <bean id="idp.serialize" parent="mda.SerializationStage">
-        <property name="serializer" ref="serializer"/>
-        <property name="outputFile">
-            <bean parent="File">
-                <constructor-arg value="${mdq.output}/idps/all.xml"/>
-            </bean>
-        </property>
-    </bean>
-
-    <!--
-        idp.selector
-
-        Entities in the IdP-only aggregate are restricted to identity providers.
-    -->
-    <bean id="idp.selector" parent="mda.XPathItemSelectionStrategy">
-        <constructor-arg value="/md:EntityDescriptor[md:IDPSSODescriptor]"/>
-        <constructor-arg ref="commonNamespaces"/>
-    </bean>
-
-    <!--
-        idp.pipeline
-
-        Generates the IdP-only aggregate. The selector has already taken care
-        of removing non-IdP entities.
-    -->
-    <bean id="idp.pipeline" parent="mda.SimplePipeline">
-        <property name="stages">
-            <list>
-                <!-- Perform common steps in constructing an output aggregate. -->
-                <ref bean="common.output"/>
-
-                <!-- Write the resulting aggregate out to a file. -->
-                <ref bean="idp.serialize"/>
-            </list>
-        </property>
-    </bean>
-
-    <!--
-        Generate all required metadata for the InCommon MDQ service
-        using the localkey configuration:
-
-          * Per-entity metadata
-          * InCommon aggregate
-          * InCommon IdP-only aggregate
-
-        Properties:
-
-        Set sign.uk.keyPassword (note the slightly anomalous "uk" in that
-        identifier) to set the password to access the key resource. If not set,
-        this will be prompted for.
-    -->
-    <bean id="mdq-combined-cloudhsm" parent="mda.SimplePipeline">
-        <property name="stages">
-            <list>
-                <!--
-                    Start with the unsigned production aggregate.
-                    TODO: (MDQ-177) For now, we're actually starting with a signed aggregate;
-                      this pipeline isn't configured to check the signature
-                -->
-                <bean id="production_aggregate" parent="mda.DOMResourceSourceStage">
-                    <property name="parserPool" ref="parserPool"/>
-                    <property name="DOMResource">
-                        <bean parent="FileSystemResource" c:_0="${mdq.input}"/>
-                    </property>
-                </bean>
-
-                <!-- Break down into individual entities. -->
-                <ref bean="disassemble"/>
-
-                <!-- Include a default registrationAuthority for each entity. -->
-                <ref bean="us_incommon_default_regauth"/>
-
-                <!-- Populate identifiers for future actions. -->
-                <ref bean="populateItemIds"/>
-                <ref bean="populateRegistrationAuthorities"/>
-
-                <!-- Discard entities in the InCommon entity blacklist. -->
-                <ref bean="remove_blacklisted_incommon_entities"/>
-
-                <!-- Fork new pipelines to generate special aggregates. -->
-                <bean id="demux" parent="mda.PipelineDemultiplexerStage">
-                    <property name="pipelineAndSelectionStrategies">
-                        <list>
-                            <!-- IdP-only aggregate. -->
-                            <bean class="net.shibboleth.utilities.java.support.collection.Pair">
-                                <constructor-arg ref="idp.pipeline"/>
-                                <constructor-arg ref="idp.selector"/>
-                            </bean>
-                        </list>
-                    </property>
-                    <property name="waitingForPipelines" value="true"/>
-                </bean>
-
-                <!-- Set ID, cacheDuration and validUntil attributes. -->
-                <bean parent="mda.GenerateIdStage"/>
-                <bean parent="mda.SetValidUntilStage" p:validityDuration="P14D"/>
-
-                <!-- Identity transform fixes signing issues. -->
-                <bean parent="mda.XSLTransformationStage"
-                    p:XSLResource="classpath:identity.xsl"/>
-
-                <!-- Sign each item. -->
-                <bean id="perform.signature" parent="mda.XMLSignatureSigningStage">
-                    <property name="privateKey">
-                      <bean class="uk.org.iay.incommon.mda.cloudhsm.CaviumPrivateKeyFactoryBean"
-                          p:username="${sign.keyUser}"
-                          p:password="${sign.keyPassword}"
-                          p:keyHandle="${sign.keyHandle}"
-                      />
-                    </property>
-                </bean>
-
-                <!-- Write individual entity documents to files. -->
-                <bean id="write.perentity" parent="mda.MultiOutputSerializationStage">
-                    <property name="serializer" ref="serializer"/>
-                    <property name="outputStrategy">
-                        <bean parent="mda.FilesInDirectoryMultiOutputStrategy" p:nameSuffix=".xml">
-                            <property name="directory">
-                                <bean parent="File" c:_="${mdq.output}"/>
-                            </property>
-                            <property name="nameTransformer">
-                                <bean parent="mda.PathSegmentStringTransformer"/>
-                            </property>
-                        </bean>
-                    </property>
-                </bean>
-
-            </list>
-        </property>
-
-    </bean>
-
-</beans>
diff --git a/mdx/incommon/mdq-combined-localkey.xml b/mdx/incommon/mdq-combined-localkey.xml
deleted file mode 100644
index 5e686953..00000000
--- a/mdx/incommon/mdq-combined-localkey.xml
+++ /dev/null
@@ -1,223 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-    Generate and sign per-entity metadata using a local private key resource.
--->
-<beans xmlns="http://www.springframework.org/schema/beans"
-    default-lazy-init="true"
-    xmlns:c="http://www.springframework.org/schema/c"
-    xmlns:p="http://www.springframework.org/schema/p"
-    xmlns:util="http://www.springframework.org/schema/util"
-    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-    xsi:schemaLocation="
-        http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd
-        http://www.springframework.org/schema/util http://www.springframework.org/schema/util/spring-util.xsd">
-
-    <!--
-        Import commonly used beans.
-    -->
-    <import resource="classpath:common-beans.xml"/>
-
-    <!--
-        Import inc-mda beans. (from import.xml)
-    -->
-    <import resource="classpath:uk/org/iay/incommon/mda/beans.xml"/>
-
-    <!--
-        Import channel-specific beans.
-    -->
-    <import resource="classpath:incommon/beans.xml"/>
-
-    <!-- This bean MUST be called "conversionService" to work properly. -->
-    <bean id="conversionService" class="org.springframework.context.support.ConversionServiceFactoryBean">
-        <property name="converters">
-            <set>
-                <bean class="net.shibboleth.ext.spring.config.DurationToLongConverter" />
-                <bean class="net.shibboleth.ext.spring.config.StringToIPRangeConverter" />
-                <bean class="net.shibboleth.ext.spring.config.BooleanToPredicateConverter" />
-                <bean class="net.shibboleth.ext.spring.config.StringBooleanToPredicateConverter" />
-                <bean class="net.shibboleth.ext.spring.config.StringToResourceConverter" />
-            </set>
-        </property>
-    </bean>
-
-    <!--
-        ***********************************************************
-        ***                                                     ***
-        ***   C O M M O N   O U T P U T   P R O C E S S I N G   ***
-        ***                                                     ***
-        ***********************************************************
-    -->
-
-    <bean id="common.output" parent="mda.CompositeStage">
-        <property name="composedStages">
-            <list>
-                <!-- Construct a new aggregate from the collection of entities. -->
-                <bean id="inc.assemble" parent="mda.EntitiesDescriptorAssemblerStage">
-                    <property name="itemOrderingStrategy">
-                        <bean parent="inc.InCommonEntityOrderingStrategy"
-                            c:_-ref="us_incommon_registrar"/>
-                    </property>
-                </bean>
-
-                <!-- Apply final tweaks to the aggregate. -->
-                <bean id="finalise" parent="incommon_finalise_parent">
-                    <property name="transformParameters">
-                        <map>
-                            <entry key="extraText" value="Contains InCommon and eduGAIN metadata"/>
-                            <entry key="publisher" value-ref="us_incommon_registrar"/>
-                        </map>
-                    </property>
-                </bean>
-
-                <!-- Normalise the use of namespace prefixes in the resulting XML document. -->
-                <bean id="normalise" parent="mda.XSLTransformationStage"
-                    p:XSLResource="classpath:incommon/ns_norm_import.xsl"/>
-            </list>
-        </property>
-    </bean>
-
-    <!--
-        *****************************************
-        ***                                   ***
-        ***   I D P - O N L Y   O U T P U T   ***
-        ***                                   ***
-        *****************************************
-    -->
-
-    <!--
-        idp.serialize
-
-        Writes the IdP-only aggregate out to a file.
-    -->
-    <bean id="idp.serialize" parent="mda.SerializationStage">
-        <property name="serializer" ref="serializer"/>
-        <property name="outputFile">
-            <bean parent="File">
-                <constructor-arg value="${mdq.output}/idps/all.xml"/>
-            </bean>
-        </property>
-    </bean>
-
-    <!--
-        idp.selector
-
-        Entities in the IdP-only aggregate are restricted to identity providers.
-    -->
-    <bean id="idp.selector" parent="mda.XPathItemSelectionStrategy">
-        <constructor-arg value="/md:EntityDescriptor[md:IDPSSODescriptor]"/>
-        <constructor-arg ref="commonNamespaces"/>
-    </bean>
-
-    <!--
-        idp.pipeline
-
-        Generates the IdP-only aggregate. The selector has already taken care
-        of removing non-IdP entities.
-    -->
-    <bean id="idp.pipeline" parent="mda.SimplePipeline">
-        <property name="stages">
-            <list>
-                <!-- Perform common steps in constructing an output aggregate. -->
-                <ref bean="common.output"/>
-
-                <!-- Write the resulting aggregate out to a file. -->
-                <ref bean="idp.serialize"/>
-            </list>
-        </property>
-    </bean>
-
-    <!--
-        Generate all required metadata for the InCommon MDQ service
-        using the localkey configuration:
-
-          * Per-entity metadata
-          * InCommon aggregate
-          * InCommon IdP-only aggregate
-
-        Properties:
-
-        Set sign.uk.keyPassword (note the slightly anomalous "uk" in that
-        identifier) to set the password to access the key resource. If not set,
-        this will be prompted for.
-    -->
-    <bean id="mdq-combined-localkey" parent="mda.SimplePipeline">
-        <property name="stages">
-            <list>
-                <!--
-                    Start with the unsigned production aggregate.
-                    TODO: (MDQ-177) For now, we're actually starting with a signed aggregate;
-                      this pipeline isn't configured to check the signature
-                -->
-                <bean id="production_aggregate" parent="mda.DOMResourceSourceStage">
-                    <property name="parserPool" ref="parserPool"/>
-                    <property name="DOMResource">
-                        <bean parent="FileSystemResource" c:_0="${mdq.input}"/>
-                    </property>
-                </bean>
-
-                <!-- Break down into individual entities. -->
-                <ref bean="disassemble"/>
-
-                <!-- Include a default registrationAuthority for each entity. -->
-                <ref bean="us_incommon_default_regauth"/>
-
-                <!-- Populate identifiers for future actions. -->
-                <ref bean="populateItemIds"/>
-                <ref bean="populateRegistrationAuthorities"/>
-
-                <!-- Discard entities in the InCommon entity blacklist. -->
-                <ref bean="remove_blacklisted_incommon_entities"/>
-
-                <!-- Fork new pipelines to generate special aggregates. -->
-                <bean id="demux" parent="mda.PipelineDemultiplexerStage">
-                    <property name="pipelineAndSelectionStrategies">
-                        <list>
-                            <!-- IdP-only aggregate. -->
-                            <bean class="net.shibboleth.utilities.java.support.collection.Pair">
-                                <constructor-arg ref="idp.pipeline"/>
-                                <constructor-arg ref="idp.selector"/>
-                            </bean>
-                        </list>
-                    </property>
-                    <property name="waitingForPipelines" value="true"/>
-                </bean>
-
-                <!-- Set ID, cacheDuration and validUntil attributes. -->
-                <bean parent="mda.GenerateIdStage"/>
-                <bean parent="mda.SetValidUntilStage" p:validityDuration="P14D"/>
-
-                <!-- Identity transform fixes signing issues. -->
-                <bean parent="mda.XSLTransformationStage"
-                    p:XSLResource="classpath:identity.xsl"/>
-
-                <!-- Sign each item. -->
-                <bean id="perform.signature" parent="mda.XMLSignatureSigningStage">
-                    <property name="privateKey">
-                        <bean parent="PrivateKeyFactoryBean"
-                            p:resource="${sign.keyResource}"
-                            p:privateKeyPassword="${sign.keyPassword}"
-                        />
-                    </property>
-                </bean>
-
-                <!-- Write individual entity documents to files. -->
-                <bean id="write.perentity" parent="mda.MultiOutputSerializationStage">
-                    <property name="serializer" ref="serializer"/>
-                    <property name="outputStrategy">
-                        <bean parent="mda.FilesInDirectoryMultiOutputStrategy" p:nameSuffix=".xml">
-                            <property name="directory">
-                                <bean parent="File" c:_="${mdq.output}"/>
-                            </property>
-                            <property name="nameTransformer">
-                                <bean parent="mda.PathSegmentStringTransformer"/>
-                            </property>
-                        </bean>
-                    </property>
-                </bean>
-
-            </list>
-        </property>
-
-    </bean>
-
-</beans>