From 6ad0a86e5951f0fd97db5e20eaa98a738c5102b2 Mon Sep 17 00:00:00 2001
From: David Shafer <dshafer@internet2.edu>
Date: Mon, 28 Jan 2019 16:11:50 -0600
Subject: [PATCH 01/19] Create the IdP-only aggregate (#1)

* Create new pipeline for MDQ and IdP-only aggregate
* Move IdP-only aggregate to idps/all.xml
* Rename mdq-new-* to mdq-combined-*
* Reference JIRA issues in TODOs
---
 build.xml                              |  60 +++++++
 mdx/incommon/mdq-combined-cloudhsm.xml | 224 +++++++++++++++++++++++++
 mdx/incommon/mdq-combined-localkey.xml | 223 ++++++++++++++++++++++++
 3 files changed, 507 insertions(+)
 create mode 100644 mdx/incommon/mdq-combined-cloudhsm.xml
 create mode 100644 mdx/incommon/mdq-combined-localkey.xml

diff --git a/build.xml b/build.xml
index 1c8348cf..ba49a694 100644
--- a/build.xml
+++ b/build.xml
@@ -2655,6 +2655,66 @@
         <echo>Generation complete.</echo>
     </target>
 
+    <!--
+        inc.mdq-combined.localkey
+
+        Generate all required metadata for the InCommon MDQ service
+        using the localkey configuration:
+
+          * Per-entity metadata
+          * InCommon IdP-only aggregate
+
+        Properties:
+
+        Set sign.uk.keyPassword (note the slightly anomalous "uk" in that
+        identifier) to set the password to access the key resource. If not set,
+        this will be prompted for.
+    -->
+    <target name="inc.mdq.combined.localkey" depends="get.sign.uk.keyPassword">
+      <property name="mda.mdq.input" value="${mda.inc.imported.xml}"/>
+      <property name="mda.sign.keyPassword" value="${sign.uk.keyPassword}"/>
+      <echo>Generating per-entity metadata in ${mda.mdq.output}</echo>
+      <echo>    (IdP-only aggregate in ${mda.mdq.output}/idps.xml)</echo>
+      <echo>    from unsigned aggregate in ${mda.mdq.input}</echo>
+      <delete dir="${mdq.output.dir}" quiet="true"/>
+      <mkdir dir="${mdq.output.dir}"/>
+      <mkdir dir="${mdq.output.dir}/idps"/>
+      <CHANNEL.do channel="incommon" verb="mdq-combined-localkey"/>
+      <echo>Generation complete.</echo>
+    </target>
+
+    <!--
+        inc.mdq.combined.cloudhsm
+
+        Generate all required metadata for the InCommon MDQ service
+        using the cloudhsm configuration:
+
+          * Per-entity metadata
+          * InCommon IdP-only aggregate
+
+        Properties:
+
+        Set sign.uk.keyPassword (note the slightly anomalous "uk" in that
+        identifier) to set the password to access the key resource. If not set,
+        this will be prompted for.
+    -->
+    <target name="inc.mdq.combined.cloudhsm" depends="get.sign.uk.keyPassword">
+      <property name="mda.mdq.input" value="${mda.inc.imported.xml}"/>
+      <property name="mda.sign.keyHandle" value="${sign.keyHandle}"/>
+      <property name="mda.sign.keyUser" value="${sign.keyUser}"/>
+      <property name="mda.sign.keyPassword" value="${sign.keyPassword}"/>
+      <property name="mda.classpath.extra" value="inc-mda-cloudhsm"/>
+      <property name="mda.jni.path" value="/opt/cloudhsm/lib"/>
+      <echo>Generating per-entity metadata in ${mda.mdq.output}</echo>
+      <echo>    (IdP-only aggregate in ${mda.mdq.output}/idps.xml)</echo>
+      <echo>    from unsigned aggregate in ${mda.mdq.input}</echo>
+      <delete dir="${mdq.output.dir}" quiet="true"/>
+      <mkdir dir="${mdq.output.dir}"/>
+      <mkdir dir="${mdq.output.dir}/idps"/>
+      <CHANNEL.do channel="incommon" verb="mdq-combined-cloudhsm"/>
+      <echo>Generation complete.</echo>
+    </target>
+
     <!--
         inc.mdq.generate.localkey
 
diff --git a/mdx/incommon/mdq-combined-cloudhsm.xml b/mdx/incommon/mdq-combined-cloudhsm.xml
new file mode 100644
index 00000000..6a54503d
--- /dev/null
+++ b/mdx/incommon/mdq-combined-cloudhsm.xml
@@ -0,0 +1,224 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+    Generate and sign per-entity metadata using a local private key resource.
+-->
+<beans xmlns="http://www.springframework.org/schema/beans"
+    default-lazy-init="true"
+    xmlns:c="http://www.springframework.org/schema/c"
+    xmlns:p="http://www.springframework.org/schema/p"
+    xmlns:util="http://www.springframework.org/schema/util"
+    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+    xsi:schemaLocation="
+        http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd
+        http://www.springframework.org/schema/util http://www.springframework.org/schema/util/spring-util.xsd">
+
+    <!--
+        Import commonly used beans.
+    -->
+    <import resource="classpath:common-beans.xml"/>
+
+    <!--
+        Import inc-mda beans. (from import.xml)
+    -->
+    <import resource="classpath:uk/org/iay/incommon/mda/beans.xml"/>
+
+    <!--
+        Import channel-specific beans.
+    -->
+    <import resource="classpath:incommon/beans.xml"/>
+
+    <!-- This bean MUST be called "conversionService" to work properly. -->
+    <bean id="conversionService" class="org.springframework.context.support.ConversionServiceFactoryBean">
+        <property name="converters">
+            <set>
+                <bean class="net.shibboleth.ext.spring.config.DurationToLongConverter" />
+                <bean class="net.shibboleth.ext.spring.config.StringToIPRangeConverter" />
+                <bean class="net.shibboleth.ext.spring.config.BooleanToPredicateConverter" />
+                <bean class="net.shibboleth.ext.spring.config.StringBooleanToPredicateConverter" />
+                <bean class="net.shibboleth.ext.spring.config.StringToResourceConverter" />
+            </set>
+        </property>
+    </bean>
+
+    <!--
+        ***********************************************************
+        ***                                                     ***
+        ***   C O M M O N   O U T P U T   P R O C E S S I N G   ***
+        ***                                                     ***
+        ***********************************************************
+    -->
+
+    <bean id="common.output" parent="mda.CompositeStage">
+        <property name="composedStages">
+            <list>
+                <!-- Construct a new aggregate from the collection of entities. -->
+                <bean id="inc.assemble" parent="mda.EntitiesDescriptorAssemblerStage">
+                    <property name="itemOrderingStrategy">
+                        <bean parent="inc.InCommonEntityOrderingStrategy"
+                            c:_-ref="us_incommon_registrar"/>
+                    </property>
+                </bean>
+
+                <!-- Apply final tweaks to the aggregate. -->
+                <bean id="finalise" parent="incommon_finalise_parent">
+                    <property name="transformParameters">
+                        <map>
+                            <entry key="extraText" value="Contains InCommon and eduGAIN metadata"/>
+                            <entry key="publisher" value-ref="us_incommon_registrar"/>
+                        </map>
+                    </property>
+                </bean>
+
+                <!-- Normalise the use of namespace prefixes in the resulting XML document. -->
+                <bean id="normalise" parent="mda.XSLTransformationStage"
+                    p:XSLResource="classpath:incommon/ns_norm_import.xsl"/>
+            </list>
+        </property>
+    </bean>
+
+    <!--
+        *****************************************
+        ***                                   ***
+        ***   I D P - O N L Y   O U T P U T   ***
+        ***                                   ***
+        *****************************************
+    -->
+
+    <!--
+        idp.serialize
+
+        Writes the IdP-only aggregate out to a file.
+    -->
+    <bean id="idp.serialize" parent="mda.SerializationStage">
+        <property name="serializer" ref="serializer"/>
+        <property name="outputFile">
+            <bean parent="File">
+                <constructor-arg value="${mdq.output}/idps/all.xml"/>
+            </bean>
+        </property>
+    </bean>
+
+    <!--
+        idp.selector
+
+        Entities in the IdP-only aggregate are restricted to identity providers.
+    -->
+    <bean id="idp.selector" parent="mda.XPathItemSelectionStrategy">
+        <constructor-arg value="/md:EntityDescriptor[md:IDPSSODescriptor]"/>
+        <constructor-arg ref="commonNamespaces"/>
+    </bean>
+
+    <!--
+        idp.pipeline
+
+        Generates the IdP-only aggregate. The selector has already taken care
+        of removing non-IdP entities.
+    -->
+    <bean id="idp.pipeline" parent="mda.SimplePipeline">
+        <property name="stages">
+            <list>
+                <!-- Perform common steps in constructing an output aggregate. -->
+                <ref bean="common.output"/>
+
+                <!-- Write the resulting aggregate out to a file. -->
+                <ref bean="idp.serialize"/>
+            </list>
+        </property>
+    </bean>
+
+    <!--
+        Generate all required metadata for the InCommon MDQ service
+        using the localkey configuration:
+
+          * Per-entity metadata
+          * InCommon aggregate
+          * InCommon IdP-only aggregate
+
+        Properties:
+
+        Set sign.uk.keyPassword (note the slightly anomalous "uk" in that
+        identifier) to set the password to access the key resource. If not set,
+        this will be prompted for.
+    -->
+    <bean id="mdq-combined-cloudhsm" parent="mda.SimplePipeline">
+        <property name="stages">
+            <list>
+                <!--
+                    Start with the unsigned production aggregate.
+                    TODO: (MDQ-177) For now, we're actually starting with a signed aggregate;
+                      this pipeline isn't configured to check the signature
+                -->
+                <bean id="production_aggregate" parent="mda.DOMResourceSourceStage">
+                    <property name="parserPool" ref="parserPool"/>
+                    <property name="DOMResource">
+                        <bean parent="FileSystemResource" c:_0="${mdq.input}"/>
+                    </property>
+                </bean>
+
+                <!-- Break down into individual entities. -->
+                <ref bean="disassemble"/>
+
+                <!-- Include a default registrationAuthority for each entity. -->
+                <ref bean="us_incommon_default_regauth"/>
+
+                <!-- Populate identifiers for future actions. -->
+                <ref bean="populateItemIds"/>
+                <ref bean="populateRegistrationAuthorities"/>
+
+                <!-- Discard entities in the InCommon entity blacklist. -->
+                <ref bean="remove_blacklisted_incommon_entities"/>
+
+                <!-- Fork new pipelines to generate special aggregates. -->
+                <bean id="demux" parent="mda.PipelineDemultiplexerStage">
+                    <property name="pipelineAndSelectionStrategies">
+                        <list>
+                            <!-- IdP-only aggregate. -->
+                            <bean class="net.shibboleth.utilities.java.support.collection.Pair">
+                                <constructor-arg ref="idp.pipeline"/>
+                                <constructor-arg ref="idp.selector"/>
+                            </bean>
+                        </list>
+                    </property>
+                    <property name="waitingForPipelines" value="true"/>
+                </bean>
+
+                <!-- Set ID, cacheDuration and validUntil attributes. -->
+                <bean parent="mda.GenerateIdStage"/>
+                <bean parent="mda.SetValidUntilStage" p:validityDuration="P14D"/>
+
+                <!-- Identity transform fixes signing issues. -->
+                <bean parent="mda.XSLTransformationStage"
+                    p:XSLResource="classpath:identity.xsl"/>
+
+                <!-- Sign each item. -->
+                <bean id="perform.signature" parent="mda.XMLSignatureSigningStage">
+                    <property name="privateKey">
+                      <bean class="uk.org.iay.incommon.mda.cloudhsm.CaviumPrivateKeyFactoryBean"
+                          p:username="${sign.keyUser}"
+                          p:password="${sign.keyPassword}"
+                          p:keyHandle="${sign.keyHandle}"
+                      />
+                    </property>
+                </bean>
+
+                <!-- Write individual entity documents to files. -->
+                <bean id="write.perentity" parent="mda.MultiOutputSerializationStage">
+                    <property name="serializer" ref="serializer"/>
+                    <property name="outputStrategy">
+                        <bean parent="mda.FilesInDirectoryMultiOutputStrategy" p:nameSuffix=".xml">
+                            <property name="directory">
+                                <bean parent="File" c:_="${mdq.output}"/>
+                            </property>
+                            <property name="nameTransformer">
+                                <bean parent="mda.PathSegmentStringTransformer"/>
+                            </property>
+                        </bean>
+                    </property>
+                </bean>
+
+            </list>
+        </property>
+
+    </bean>
+
+</beans>
diff --git a/mdx/incommon/mdq-combined-localkey.xml b/mdx/incommon/mdq-combined-localkey.xml
new file mode 100644
index 00000000..5e686953
--- /dev/null
+++ b/mdx/incommon/mdq-combined-localkey.xml
@@ -0,0 +1,223 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+    Generate and sign per-entity metadata using a local private key resource.
+-->
+<beans xmlns="http://www.springframework.org/schema/beans"
+    default-lazy-init="true"
+    xmlns:c="http://www.springframework.org/schema/c"
+    xmlns:p="http://www.springframework.org/schema/p"
+    xmlns:util="http://www.springframework.org/schema/util"
+    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+    xsi:schemaLocation="
+        http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd
+        http://www.springframework.org/schema/util http://www.springframework.org/schema/util/spring-util.xsd">
+
+    <!--
+        Import commonly used beans.
+    -->
+    <import resource="classpath:common-beans.xml"/>
+
+    <!--
+        Import inc-mda beans. (from import.xml)
+    -->
+    <import resource="classpath:uk/org/iay/incommon/mda/beans.xml"/>
+
+    <!--
+        Import channel-specific beans.
+    -->
+    <import resource="classpath:incommon/beans.xml"/>
+
+    <!-- This bean MUST be called "conversionService" to work properly. -->
+    <bean id="conversionService" class="org.springframework.context.support.ConversionServiceFactoryBean">
+        <property name="converters">
+            <set>
+                <bean class="net.shibboleth.ext.spring.config.DurationToLongConverter" />
+                <bean class="net.shibboleth.ext.spring.config.StringToIPRangeConverter" />
+                <bean class="net.shibboleth.ext.spring.config.BooleanToPredicateConverter" />
+                <bean class="net.shibboleth.ext.spring.config.StringBooleanToPredicateConverter" />
+                <bean class="net.shibboleth.ext.spring.config.StringToResourceConverter" />
+            </set>
+        </property>
+    </bean>
+
+    <!--
+        ***********************************************************
+        ***                                                     ***
+        ***   C O M M O N   O U T P U T   P R O C E S S I N G   ***
+        ***                                                     ***
+        ***********************************************************
+    -->
+
+    <bean id="common.output" parent="mda.CompositeStage">
+        <property name="composedStages">
+            <list>
+                <!-- Construct a new aggregate from the collection of entities. -->
+                <bean id="inc.assemble" parent="mda.EntitiesDescriptorAssemblerStage">
+                    <property name="itemOrderingStrategy">
+                        <bean parent="inc.InCommonEntityOrderingStrategy"
+                            c:_-ref="us_incommon_registrar"/>
+                    </property>
+                </bean>
+
+                <!-- Apply final tweaks to the aggregate. -->
+                <bean id="finalise" parent="incommon_finalise_parent">
+                    <property name="transformParameters">
+                        <map>
+                            <entry key="extraText" value="Contains InCommon and eduGAIN metadata"/>
+                            <entry key="publisher" value-ref="us_incommon_registrar"/>
+                        </map>
+                    </property>
+                </bean>
+
+                <!-- Normalise the use of namespace prefixes in the resulting XML document. -->
+                <bean id="normalise" parent="mda.XSLTransformationStage"
+                    p:XSLResource="classpath:incommon/ns_norm_import.xsl"/>
+            </list>
+        </property>
+    </bean>
+
+    <!--
+        *****************************************
+        ***                                   ***
+        ***   I D P - O N L Y   O U T P U T   ***
+        ***                                   ***
+        *****************************************
+    -->
+
+    <!--
+        idp.serialize
+
+        Writes the IdP-only aggregate out to a file.
+    -->
+    <bean id="idp.serialize" parent="mda.SerializationStage">
+        <property name="serializer" ref="serializer"/>
+        <property name="outputFile">
+            <bean parent="File">
+                <constructor-arg value="${mdq.output}/idps/all.xml"/>
+            </bean>
+        </property>
+    </bean>
+
+    <!--
+        idp.selector
+
+        Entities in the IdP-only aggregate are restricted to identity providers.
+    -->
+    <bean id="idp.selector" parent="mda.XPathItemSelectionStrategy">
+        <constructor-arg value="/md:EntityDescriptor[md:IDPSSODescriptor]"/>
+        <constructor-arg ref="commonNamespaces"/>
+    </bean>
+
+    <!--
+        idp.pipeline
+
+        Generates the IdP-only aggregate. The selector has already taken care
+        of removing non-IdP entities.
+    -->
+    <bean id="idp.pipeline" parent="mda.SimplePipeline">
+        <property name="stages">
+            <list>
+                <!-- Perform common steps in constructing an output aggregate. -->
+                <ref bean="common.output"/>
+
+                <!-- Write the resulting aggregate out to a file. -->
+                <ref bean="idp.serialize"/>
+            </list>
+        </property>
+    </bean>
+
+    <!--
+        Generate all required metadata for the InCommon MDQ service
+        using the localkey configuration:
+
+          * Per-entity metadata
+          * InCommon aggregate
+          * InCommon IdP-only aggregate
+
+        Properties:
+
+        Set sign.uk.keyPassword (note the slightly anomalous "uk" in that
+        identifier) to set the password to access the key resource. If not set,
+        this will be prompted for.
+    -->
+    <bean id="mdq-combined-localkey" parent="mda.SimplePipeline">
+        <property name="stages">
+            <list>
+                <!--
+                    Start with the unsigned production aggregate.
+                    TODO: (MDQ-177) For now, we're actually starting with a signed aggregate;
+                      this pipeline isn't configured to check the signature
+                -->
+                <bean id="production_aggregate" parent="mda.DOMResourceSourceStage">
+                    <property name="parserPool" ref="parserPool"/>
+                    <property name="DOMResource">
+                        <bean parent="FileSystemResource" c:_0="${mdq.input}"/>
+                    </property>
+                </bean>
+
+                <!-- Break down into individual entities. -->
+                <ref bean="disassemble"/>
+
+                <!-- Include a default registrationAuthority for each entity. -->
+                <ref bean="us_incommon_default_regauth"/>
+
+                <!-- Populate identifiers for future actions. -->
+                <ref bean="populateItemIds"/>
+                <ref bean="populateRegistrationAuthorities"/>
+
+                <!-- Discard entities in the InCommon entity blacklist. -->
+                <ref bean="remove_blacklisted_incommon_entities"/>
+
+                <!-- Fork new pipelines to generate special aggregates. -->
+                <bean id="demux" parent="mda.PipelineDemultiplexerStage">
+                    <property name="pipelineAndSelectionStrategies">
+                        <list>
+                            <!-- IdP-only aggregate. -->
+                            <bean class="net.shibboleth.utilities.java.support.collection.Pair">
+                                <constructor-arg ref="idp.pipeline"/>
+                                <constructor-arg ref="idp.selector"/>
+                            </bean>
+                        </list>
+                    </property>
+                    <property name="waitingForPipelines" value="true"/>
+                </bean>
+
+                <!-- Set ID, cacheDuration and validUntil attributes. -->
+                <bean parent="mda.GenerateIdStage"/>
+                <bean parent="mda.SetValidUntilStage" p:validityDuration="P14D"/>
+
+                <!-- Identity transform fixes signing issues. -->
+                <bean parent="mda.XSLTransformationStage"
+                    p:XSLResource="classpath:identity.xsl"/>
+
+                <!-- Sign each item. -->
+                <bean id="perform.signature" parent="mda.XMLSignatureSigningStage">
+                    <property name="privateKey">
+                        <bean parent="PrivateKeyFactoryBean"
+                            p:resource="${sign.keyResource}"
+                            p:privateKeyPassword="${sign.keyPassword}"
+                        />
+                    </property>
+                </bean>
+
+                <!-- Write individual entity documents to files. -->
+                <bean id="write.perentity" parent="mda.MultiOutputSerializationStage">
+                    <property name="serializer" ref="serializer"/>
+                    <property name="outputStrategy">
+                        <bean parent="mda.FilesInDirectoryMultiOutputStrategy" p:nameSuffix=".xml">
+                            <property name="directory">
+                                <bean parent="File" c:_="${mdq.output}"/>
+                            </property>
+                            <property name="nameTransformer">
+                                <bean parent="mda.PathSegmentStringTransformer"/>
+                            </property>
+                        </bean>
+                    </property>
+                </bean>
+
+            </list>
+        </property>
+
+    </bean>
+
+</beans>

From d5156ab3027439813025900f220c40d70f2dddef Mon Sep 17 00:00:00 2001
From: David Shafer <dshafer@internet2.edu>
Date: Wed, 6 Feb 2019 16:24:19 -0600
Subject: [PATCH 02/19] Create new pipeline to produce all MDQ metadata

---
 build.xml                              |  68 +++----
 mdx/incommon/mdq-all-cloudhsm.xml      |  46 +++++
 mdx/incommon/mdq-all-localkey.xml      |  46 +++++
 mdx/incommon/mdq-all.xml               | 245 +++++++++++++++++++++++++
 mdx/incommon/mdq-combined-cloudhsm.xml | 224 ----------------------
 mdx/incommon/mdq-combined-localkey.xml | 223 ----------------------
 6 files changed, 367 insertions(+), 485 deletions(-)
 create mode 100644 mdx/incommon/mdq-all-cloudhsm.xml
 create mode 100644 mdx/incommon/mdq-all-localkey.xml
 create mode 100644 mdx/incommon/mdq-all.xml
 delete mode 100644 mdx/incommon/mdq-combined-cloudhsm.xml
 delete mode 100644 mdx/incommon/mdq-combined-localkey.xml

diff --git a/build.xml b/build.xml
index ba49a694..083c6411 100644
--- a/build.xml
+++ b/build.xml
@@ -2656,13 +2656,10 @@
     </target>
 
     <!--
-        inc.mdq-combined.localkey
+        inc.mdq.generate.all.cloudhsm
 
-        Generate all required metadata for the InCommon MDQ service
-        using the localkey configuration:
-
-          * Per-entity metadata
-          * InCommon IdP-only aggregate
+        Generate all metadata required for the InCommon federation MDQ service
+        using the AWS CloudHSM configuration.
 
         Properties:
 
@@ -2670,27 +2667,27 @@
         identifier) to set the password to access the key resource. If not set,
         this will be prompted for.
     -->
-    <target name="inc.mdq.combined.localkey" depends="get.sign.uk.keyPassword">
-      <property name="mda.mdq.input" value="${mda.inc.imported.xml}"/>
-      <property name="mda.sign.keyPassword" value="${sign.uk.keyPassword}"/>
-      <echo>Generating per-entity metadata in ${mda.mdq.output}</echo>
-      <echo>    (IdP-only aggregate in ${mda.mdq.output}/idps.xml)</echo>
-      <echo>    from unsigned aggregate in ${mda.mdq.input}</echo>
-      <delete dir="${mdq.output.dir}" quiet="true"/>
-      <mkdir dir="${mdq.output.dir}"/>
-      <mkdir dir="${mdq.output.dir}/idps"/>
-      <CHANNEL.do channel="incommon" verb="mdq-combined-localkey"/>
-      <echo>Generation complete.</echo>
+    <target name="inc.mdq.generate.all.cloudhsm" depends="get.sign.uk.keyPassword">
+        <property name="mda.classpath.extra" value="inc-mda-cloudhsm"/>
+        <property name="mda.jni.path" value="/opt/cloudhsm/lib"/>
+		<property name="mda.mdq.input" value="${mda.inc.imported.xml}"/>
+        <property name="mda.sign.keyHandle" value="${sign.keyHandle}"/>
+		<property name="mda.sign.keyPassword" value="${sign.uk.keyPassword}"/>
+        <property name="mda.sign.keyUser" value="${sign.keyUser}"/>
+		<echo>Generating MDQ metadata in ${mda.mdq.output}</echo>
+		<echo>    from unsigned aggregate in ${mda.mdq.input}</echo>
+		<delete dir="${mdq.output.dir}" quiet="true"/>
+		<mkdir dir="${mdq.output.dir}"/>
+		<mkdir dir="${mdq.output.dir}/idps"/>
+		<CHANNEL.do channel="incommon" verb="mdq-all-cloudhsm"/>
+		<echo>Generation complete.</echo>
     </target>
 
     <!--
-        inc.mdq.combined.cloudhsm
-
-        Generate all required metadata for the InCommon MDQ service
-        using the cloudhsm configuration:
+        inc.mdq.generate.all.localkey
 
-          * Per-entity metadata
-          * InCommon IdP-only aggregate
+        Generate all metadata required for the InCommon federation MDQ service
+        using the local key configuration.
 
         Properties:
 
@@ -2698,21 +2695,16 @@
         identifier) to set the password to access the key resource. If not set,
         this will be prompted for.
     -->
-    <target name="inc.mdq.combined.cloudhsm" depends="get.sign.uk.keyPassword">
-      <property name="mda.mdq.input" value="${mda.inc.imported.xml}"/>
-      <property name="mda.sign.keyHandle" value="${sign.keyHandle}"/>
-      <property name="mda.sign.keyUser" value="${sign.keyUser}"/>
-      <property name="mda.sign.keyPassword" value="${sign.keyPassword}"/>
-      <property name="mda.classpath.extra" value="inc-mda-cloudhsm"/>
-      <property name="mda.jni.path" value="/opt/cloudhsm/lib"/>
-      <echo>Generating per-entity metadata in ${mda.mdq.output}</echo>
-      <echo>    (IdP-only aggregate in ${mda.mdq.output}/idps.xml)</echo>
-      <echo>    from unsigned aggregate in ${mda.mdq.input}</echo>
-      <delete dir="${mdq.output.dir}" quiet="true"/>
-      <mkdir dir="${mdq.output.dir}"/>
-      <mkdir dir="${mdq.output.dir}/idps"/>
-      <CHANNEL.do channel="incommon" verb="mdq-combined-cloudhsm"/>
-      <echo>Generation complete.</echo>
+	<target name="inc.mdq.generate.all.localkey" depends="get.sign.uk.keyPassword">
+		<property name="mda.mdq.input" value="${mda.inc.imported.xml}"/>
+		<property name="mda.sign.keyPassword" value="${sign.uk.keyPassword}"/>
+		<echo>Generating MDQ metadata in ${mda.mdq.output}</echo>
+		<echo>    from unsigned aggregate in ${mda.mdq.input}</echo>
+		<delete dir="${mdq.output.dir}" quiet="true"/>
+		<mkdir dir="${mdq.output.dir}"/>
+		<mkdir dir="${mdq.output.dir}/idps"/>
+		<CHANNEL.do channel="incommon" verb="mdq-all-localkey"/>
+		<echo>Generation complete.</echo>
     </target>
 
     <!--
diff --git a/mdx/incommon/mdq-all-cloudhsm.xml b/mdx/incommon/mdq-all-cloudhsm.xml
new file mode 100644
index 00000000..e54e711a
--- /dev/null
+++ b/mdx/incommon/mdq-all-cloudhsm.xml
@@ -0,0 +1,46 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<beans xmlns="http://www.springframework.org/schema/beans" default-lazy-init="true"
+    xmlns:c="http://www.springframework.org/schema/c" xmlns:p="http://www.springframework.org/schema/p"
+    xmlns:util="http://www.springframework.org/schema/util" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+    xsi:schemaLocation="
+        http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd
+        http://www.springframework.org/schema/util http://www.springframework.org/schema/util/spring-util.xsd">
+
+    <!-- Import commonly used beans -->
+    <import resource="classpath:common-beans.xml" />
+
+    <!-- Import inc-mda beans (from import.xml) -->
+    <import resource="classpath:uk/org/iay/incommon/mda/beans.xml" />
+
+    <!-- Import channel-specific beans -->
+    <import resource="classpath:incommon/beans.xml" />
+
+    <!-- Import mdq-all beans -->
+    <import resource="classpath:incommon/mdq-all.xml" />
+
+    <!-- This bean MUST be called "conversionService" to work properly -->
+    <bean id="conversionService" class="org.springframework.context.support.ConversionServiceFactoryBean">
+        <property name="converters">
+            <set>
+                <bean class="net.shibboleth.ext.spring.config.DurationToLongConverter" />
+                <bean class="net.shibboleth.ext.spring.config.StringToIPRangeConverter" />
+                <bean class="net.shibboleth.ext.spring.config.BooleanToPredicateConverter" />
+                <bean class="net.shibboleth.ext.spring.config.StringBooleanToPredicateConverter" />
+                <bean class="net.shibboleth.ext.spring.config.StringToResourceConverter" />
+            </set>
+        </property>
+    </bean>
+
+    <bean id="privateKeyFactory" class="uk.org.iay.incommon.mda.cloudhsm.CaviumPrivateKeyFactoryBean"
+        p:username="${sign.keyUser}" p:password="${sign.keyPassword}" p:keyHandle="${sign.keyHandle}" />
+
+    <bean id="mdq-all-cloudhsm" parent="mda.SimplePipeline">
+        <property name="stages">
+            <list>
+                <ref bean="generateAll" />
+            </list>
+        </property>
+    </bean>
+
+</beans>
diff --git a/mdx/incommon/mdq-all-localkey.xml b/mdx/incommon/mdq-all-localkey.xml
new file mode 100644
index 00000000..d329dc75
--- /dev/null
+++ b/mdx/incommon/mdq-all-localkey.xml
@@ -0,0 +1,46 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<beans xmlns="http://www.springframework.org/schema/beans" default-lazy-init="true"
+    xmlns:c="http://www.springframework.org/schema/c" xmlns:p="http://www.springframework.org/schema/p"
+    xmlns:util="http://www.springframework.org/schema/util" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+    xsi:schemaLocation="
+        http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd
+        http://www.springframework.org/schema/util http://www.springframework.org/schema/util/spring-util.xsd">
+
+    <!-- Import commonly used beans -->
+    <import resource="classpath:common-beans.xml" />
+
+    <!-- Import inc-mda beans (from import.xml) -->
+    <import resource="classpath:uk/org/iay/incommon/mda/beans.xml" />
+
+    <!-- Import channel-specific beans -->
+    <import resource="classpath:incommon/beans.xml" />
+
+    <!-- Import mdq-all beans -->
+    <import resource="classpath:incommon/mdq-all.xml" />
+
+    <!-- This bean MUST be called "conversionService" to work properly -->
+    <bean id="conversionService" class="org.springframework.context.support.ConversionServiceFactoryBean">
+        <property name="converters">
+            <set>
+                <bean class="net.shibboleth.ext.spring.config.DurationToLongConverter" />
+                <bean class="net.shibboleth.ext.spring.config.StringToIPRangeConverter" />
+                <bean class="net.shibboleth.ext.spring.config.BooleanToPredicateConverter" />
+                <bean class="net.shibboleth.ext.spring.config.StringBooleanToPredicateConverter" />
+                <bean class="net.shibboleth.ext.spring.config.StringToResourceConverter" />
+            </set>
+        </property>
+    </bean>
+
+    <bean id="privateKeyFactory" parent="PrivateKeyFactoryBean" p:resource="${sign.keyResource}"
+        p:privateKeyPassword="${sign.keyPassword}" />
+
+    <bean id="mdq-all-localkey" parent="mda.SimplePipeline">
+        <property name="stages">
+            <list>
+                <ref bean="generateAll" />
+            </list>
+        </property>
+    </bean>
+
+</beans>
diff --git a/mdx/incommon/mdq-all.xml b/mdx/incommon/mdq-all.xml
new file mode 100644
index 00000000..ca85098a
--- /dev/null
+++ b/mdx/incommon/mdq-all.xml
@@ -0,0 +1,245 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<beans xmlns="http://www.springframework.org/schema/beans" default-lazy-init="true"
+    xmlns:c="http://www.springframework.org/schema/c" xmlns:p="http://www.springframework.org/schema/p"
+    xmlns:if="ant:if" xmlns:unless="ant:unless" xmlns:util="http://www.springframework.org/schema/util"
+    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+    xsi:schemaLocation="
+        http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd
+        http://www.springframework.org/schema/util http://www.springframework.org/schema/util/spring-util.xsd">
+
+    <!-- Import commonly used beans -->
+    <import resource="classpath:common-beans.xml" />
+
+    <!-- Import inc-mda beans (from import.xml) -->
+    <import resource="classpath:uk/org/iay/incommon/mda/beans.xml" />
+
+    <!-- Import channel-specific beans -->
+    <import resource="classpath:incommon/beans.xml" />
+
+    <!-- This bean MUST be called "conversionService" to work properly -->
+    <bean id="conversionService" class="org.springframework.context.support.ConversionServiceFactoryBean">
+        <property name="converters">
+            <set>
+                <bean class="net.shibboleth.ext.spring.config.DurationToLongConverter" />
+                <bean class="net.shibboleth.ext.spring.config.StringToIPRangeConverter" />
+                <bean class="net.shibboleth.ext.spring.config.BooleanToPredicateConverter" />
+                <bean class="net.shibboleth.ext.spring.config.StringBooleanToPredicateConverter" />
+                <bean class="net.shibboleth.ext.spring.config.StringToResourceConverter" />
+            </set>
+        </property>
+    </bean>
+
+    <!-- Predicate for matching everything -->
+    <bean id="matchEverything" class="com.google.common.base.Predicates" factory-method="alwaysTrue" />
+
+    <!-- Generate all required metadata for the InCommon MDQ service -->
+    <bean id="generateAll" parent="mda.CompositeStage">
+        <property name="composedStages">
+            <list>
+
+                <!-- Begin with the published aggregate -->
+                <ref bean="commonSourceStage" />
+
+                <!-- Fork pipelines for creating signed aggregates -->
+                <bean id="demux" parent="mda.PipelineDemultiplexerStage">
+
+                    <property name="pipelineAndSelectionStrategies">
+                        <list>
+
+                            <!-- Create the signed aggregate -->
+                            <bean class="net.shibboleth.utilities.java.support.collection.Pair">
+                                <constructor-arg ref="aggregatePipeline" />
+                                <constructor-arg ref="matchEverything" />
+                            </bean>
+
+                            <!-- Create the signed IdP-only aggregate -->
+                            <bean class="net.shibboleth.utilities.java.support.collection.Pair">
+                                <constructor-arg ref="idpPipeline" />
+                                <constructor-arg ref="idpSelectionStrategy" />
+                            </bean>
+
+                        </list>
+                    </property>
+
+                    <property name="waitingForPipelines" value="true" />
+
+                </bean>
+
+                <!-- Create per-entity metadata -->
+                <ref bean="perEntityStage" />
+
+            </list>
+        </property>
+    </bean>
+
+    <!-- Pipeline that produces the signed aggregate from a collection of entities -->
+    <bean id="aggregatePipeline" parent="mda.SimplePipeline">
+        <property name="stages">
+            <list>
+                <ref bean="aggregateStage" />
+            </list>
+        </property>
+    </bean>
+
+    <!-- Stage that produces the signed aggregate from a collection of entities -->
+    <bean id="aggregateStage" parent="mda.CompositeStage">
+        <property name="composedStages">
+            <list>
+
+                <!-- Perform common steps in constructing an output aggregate -->
+                <ref bean="commonOutputStage" />
+
+                <!-- Write the resulting aggregate to a file -->
+                <ref bean="aggregateSerializationStage" />
+
+            </list>
+        </property>
+    </bean>
+
+    <!-- Stage that writes the aggregate to a file -->
+    <bean id="aggregateSerializationStage" parent="mda.SerializationStage">
+        <property name="serializer" ref="serializer" />
+        <property name="outputFile">
+            <bean parent="File">
+                <constructor-arg value="${mdq.output}/all.xml" />
+            </bean>
+        </property>
+    </bean>
+
+    <!-- Output stage used by all MDQ aggregate pipelines -->
+    <bean id="commonOutputStage" parent="mda.CompositeStage">
+        <property name="composedStages">
+            <list>
+                <!-- Construct a new aggregate from the collection of entities -->
+                <bean id="inc.assemble" parent="mda.EntitiesDescriptorAssemblerStage">
+                    <property name="itemOrderingStrategy">
+                        <bean parent="inc.InCommonEntityOrderingStrategy" c:_-ref="us_incommon_registrar" />
+                    </property>
+                </bean>
+
+                <!-- Apply final tweaks to the aggregate -->
+                <bean id="finalise" parent="incommon_finalise_parent">
+                    <property name="transformParameters">
+                        <map>
+                            <entry key="extraText" value="Contains InCommon and eduGAIN metadata" />
+                            <entry key="publisher" value-ref="us_incommon_registrar" />
+                        </map>
+                    </property>
+                </bean>
+
+                <!-- Normalise the use of namespace prefixes in the resulting XML document. -->
+                <bean id="normalise" parent="mda.XSLTransformationStage"
+                    p:XSLResource="classpath:incommon/ns_norm_import.xsl" />
+
+                <!-- Sign the aggregate -->
+                <ref bean="signItems" />
+
+            </list>
+        </property>
+    </bean>
+
+    <!-- Source stage used by all MDQ pipelines -->
+    <bean id="commonSourceStage" parent="mda.CompositeStage">
+        <property name="composedStages">
+            <list>
+
+                <!-- Start with the signed production aggregate -->
+                <bean id="production_aggregate" parent="mda.DOMResourceSourceStage">
+                    <property name="parserPool" ref="parserPool" />
+                    <property name="DOMResource">
+                        <bean parent="FileSystemResource" c:_0="${mdq.input}" />
+                    </property>
+                </bean>
+
+                <!-- TODO: Check signature on source aggregate -->
+
+                <!-- Break down into individual entities. -->
+                <ref bean="disassemble" />
+
+                <!-- Include a default registrationAuthority for each entity. -->
+                <ref bean="us_incommon_default_regauth" />
+
+                <!-- Populate identifiers for future actions. -->
+                <ref bean="populateItemIds" />
+                <ref bean="populateRegistrationAuthorities" />
+
+                <!-- Discard entities in the InCommon entity blacklist. -->
+                <ref bean="remove_blacklisted_incommon_entities" />
+
+            </list>
+        </property>
+    </bean>
+
+    <!-- Pipeline that produces the IdP-only aggregate from a collection of IdP entities -->
+    <bean id="idpPipeline" parent="mda.SimplePipeline">
+        <property name="stages">
+            <list>
+
+                <!-- Perform common steps in constructing an output aggregate. -->
+                <ref bean="commonOutputStage" />
+
+                <!-- Write the resulting aggregate out to a file. -->
+                <ref bean="idpSerializationStage" />
+            </list>
+        </property>
+    </bean>
+
+    <!-- Selection strategy that selects IdP entities for the IdP-only aggregate -->
+    <bean id="idpSelectionStrategy" parent="mda.XPathItemSelectionStrategy">
+        <constructor-arg value="/md:EntityDescriptor[md:IDPSSODescriptor]" />
+        <constructor-arg ref="commonNamespaces" />
+    </bean>
+
+    <!-- Serialization stage that writes the IdP-only aggregate to a file -->
+    <bean id="idpSerializationStage" parent="mda.SerializationStage">
+        <property name="serializer" ref="serializer" />
+        <property name="outputFile">
+            <bean parent="File">
+                <constructor-arg value="${mdq.output}/idps/all.xml" />
+            </bean>
+        </property>
+    </bean>
+
+    <!-- Stage to produce signed per-entity metadata -->
+    <bean id="perEntityStage" parent="mda.CompositeStage">
+        <property name="composedStages">
+            <list>
+
+                <!-- Set ID, cacheDuration and validUntil attributes. -->
+                <bean parent="mda.GenerateIdStage" />
+                <bean parent="mda.SetValidUntilStage" p:validityDuration="P14D" />
+
+                <!-- Identity transform fixes signing issues. -->
+                <bean parent="mda.XSLTransformationStage" p:XSLResource="classpath:identity.xsl" />
+
+                <!-- Sign each item. -->
+                <ref bean="signItems" />
+
+                <!-- Write individual entity documents to files. -->
+                <bean id="write.perentity" parent="mda.MultiOutputSerializationStage">
+                    <property name="serializer" ref="serializer" />
+                    <property name="outputStrategy">
+                        <bean parent="mda.FilesInDirectoryMultiOutputStrategy" p:nameSuffix=".xml">
+                            <property name="directory">
+                                <bean parent="File" c:_="${mdq.output}" />
+                            </property>
+                            <property name="nameTransformer">
+                                <bean parent="mda.PathSegmentStringTransformer" />
+                            </property>
+                        </bean>
+                    </property>
+                </bean>
+
+            </list>
+        </property>
+    </bean>
+
+    <!-- Signs items using a privateKeyFactory defined elsewhere -->
+    <bean id="signItems" parent="mda.XMLSignatureSigningStage">
+        <property name="privateKey">
+            <ref bean="privateKeyFactory" />
+        </property>
+    </bean>
+
+</beans>
diff --git a/mdx/incommon/mdq-combined-cloudhsm.xml b/mdx/incommon/mdq-combined-cloudhsm.xml
deleted file mode 100644
index 6a54503d..00000000
--- a/mdx/incommon/mdq-combined-cloudhsm.xml
+++ /dev/null
@@ -1,224 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-    Generate and sign per-entity metadata using a local private key resource.
--->
-<beans xmlns="http://www.springframework.org/schema/beans"
-    default-lazy-init="true"
-    xmlns:c="http://www.springframework.org/schema/c"
-    xmlns:p="http://www.springframework.org/schema/p"
-    xmlns:util="http://www.springframework.org/schema/util"
-    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-    xsi:schemaLocation="
-        http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd
-        http://www.springframework.org/schema/util http://www.springframework.org/schema/util/spring-util.xsd">
-
-    <!--
-        Import commonly used beans.
-    -->
-    <import resource="classpath:common-beans.xml"/>
-
-    <!--
-        Import inc-mda beans. (from import.xml)
-    -->
-    <import resource="classpath:uk/org/iay/incommon/mda/beans.xml"/>
-
-    <!--
-        Import channel-specific beans.
-    -->
-    <import resource="classpath:incommon/beans.xml"/>
-
-    <!-- This bean MUST be called "conversionService" to work properly. -->
-    <bean id="conversionService" class="org.springframework.context.support.ConversionServiceFactoryBean">
-        <property name="converters">
-            <set>
-                <bean class="net.shibboleth.ext.spring.config.DurationToLongConverter" />
-                <bean class="net.shibboleth.ext.spring.config.StringToIPRangeConverter" />
-                <bean class="net.shibboleth.ext.spring.config.BooleanToPredicateConverter" />
-                <bean class="net.shibboleth.ext.spring.config.StringBooleanToPredicateConverter" />
-                <bean class="net.shibboleth.ext.spring.config.StringToResourceConverter" />
-            </set>
-        </property>
-    </bean>
-
-    <!--
-        ***********************************************************
-        ***                                                     ***
-        ***   C O M M O N   O U T P U T   P R O C E S S I N G   ***
-        ***                                                     ***
-        ***********************************************************
-    -->
-
-    <bean id="common.output" parent="mda.CompositeStage">
-        <property name="composedStages">
-            <list>
-                <!-- Construct a new aggregate from the collection of entities. -->
-                <bean id="inc.assemble" parent="mda.EntitiesDescriptorAssemblerStage">
-                    <property name="itemOrderingStrategy">
-                        <bean parent="inc.InCommonEntityOrderingStrategy"
-                            c:_-ref="us_incommon_registrar"/>
-                    </property>
-                </bean>
-
-                <!-- Apply final tweaks to the aggregate. -->
-                <bean id="finalise" parent="incommon_finalise_parent">
-                    <property name="transformParameters">
-                        <map>
-                            <entry key="extraText" value="Contains InCommon and eduGAIN metadata"/>
-                            <entry key="publisher" value-ref="us_incommon_registrar"/>
-                        </map>
-                    </property>
-                </bean>
-
-                <!-- Normalise the use of namespace prefixes in the resulting XML document. -->
-                <bean id="normalise" parent="mda.XSLTransformationStage"
-                    p:XSLResource="classpath:incommon/ns_norm_import.xsl"/>
-            </list>
-        </property>
-    </bean>
-
-    <!--
-        *****************************************
-        ***                                   ***
-        ***   I D P - O N L Y   O U T P U T   ***
-        ***                                   ***
-        *****************************************
-    -->
-
-    <!--
-        idp.serialize
-
-        Writes the IdP-only aggregate out to a file.
-    -->
-    <bean id="idp.serialize" parent="mda.SerializationStage">
-        <property name="serializer" ref="serializer"/>
-        <property name="outputFile">
-            <bean parent="File">
-                <constructor-arg value="${mdq.output}/idps/all.xml"/>
-            </bean>
-        </property>
-    </bean>
-
-    <!--
-        idp.selector
-
-        Entities in the IdP-only aggregate are restricted to identity providers.
-    -->
-    <bean id="idp.selector" parent="mda.XPathItemSelectionStrategy">
-        <constructor-arg value="/md:EntityDescriptor[md:IDPSSODescriptor]"/>
-        <constructor-arg ref="commonNamespaces"/>
-    </bean>
-
-    <!--
-        idp.pipeline
-
-        Generates the IdP-only aggregate. The selector has already taken care
-        of removing non-IdP entities.
-    -->
-    <bean id="idp.pipeline" parent="mda.SimplePipeline">
-        <property name="stages">
-            <list>
-                <!-- Perform common steps in constructing an output aggregate. -->
-                <ref bean="common.output"/>
-
-                <!-- Write the resulting aggregate out to a file. -->
-                <ref bean="idp.serialize"/>
-            </list>
-        </property>
-    </bean>
-
-    <!--
-        Generate all required metadata for the InCommon MDQ service
-        using the localkey configuration:
-
-          * Per-entity metadata
-          * InCommon aggregate
-          * InCommon IdP-only aggregate
-
-        Properties:
-
-        Set sign.uk.keyPassword (note the slightly anomalous "uk" in that
-        identifier) to set the password to access the key resource. If not set,
-        this will be prompted for.
-    -->
-    <bean id="mdq-combined-cloudhsm" parent="mda.SimplePipeline">
-        <property name="stages">
-            <list>
-                <!--
-                    Start with the unsigned production aggregate.
-                    TODO: (MDQ-177) For now, we're actually starting with a signed aggregate;
-                      this pipeline isn't configured to check the signature
-                -->
-                <bean id="production_aggregate" parent="mda.DOMResourceSourceStage">
-                    <property name="parserPool" ref="parserPool"/>
-                    <property name="DOMResource">
-                        <bean parent="FileSystemResource" c:_0="${mdq.input}"/>
-                    </property>
-                </bean>
-
-                <!-- Break down into individual entities. -->
-                <ref bean="disassemble"/>
-
-                <!-- Include a default registrationAuthority for each entity. -->
-                <ref bean="us_incommon_default_regauth"/>
-
-                <!-- Populate identifiers for future actions. -->
-                <ref bean="populateItemIds"/>
-                <ref bean="populateRegistrationAuthorities"/>
-
-                <!-- Discard entities in the InCommon entity blacklist. -->
-                <ref bean="remove_blacklisted_incommon_entities"/>
-
-                <!-- Fork new pipelines to generate special aggregates. -->
-                <bean id="demux" parent="mda.PipelineDemultiplexerStage">
-                    <property name="pipelineAndSelectionStrategies">
-                        <list>
-                            <!-- IdP-only aggregate. -->
-                            <bean class="net.shibboleth.utilities.java.support.collection.Pair">
-                                <constructor-arg ref="idp.pipeline"/>
-                                <constructor-arg ref="idp.selector"/>
-                            </bean>
-                        </list>
-                    </property>
-                    <property name="waitingForPipelines" value="true"/>
-                </bean>
-
-                <!-- Set ID, cacheDuration and validUntil attributes. -->
-                <bean parent="mda.GenerateIdStage"/>
-                <bean parent="mda.SetValidUntilStage" p:validityDuration="P14D"/>
-
-                <!-- Identity transform fixes signing issues. -->
-                <bean parent="mda.XSLTransformationStage"
-                    p:XSLResource="classpath:identity.xsl"/>
-
-                <!-- Sign each item. -->
-                <bean id="perform.signature" parent="mda.XMLSignatureSigningStage">
-                    <property name="privateKey">
-                      <bean class="uk.org.iay.incommon.mda.cloudhsm.CaviumPrivateKeyFactoryBean"
-                          p:username="${sign.keyUser}"
-                          p:password="${sign.keyPassword}"
-                          p:keyHandle="${sign.keyHandle}"
-                      />
-                    </property>
-                </bean>
-
-                <!-- Write individual entity documents to files. -->
-                <bean id="write.perentity" parent="mda.MultiOutputSerializationStage">
-                    <property name="serializer" ref="serializer"/>
-                    <property name="outputStrategy">
-                        <bean parent="mda.FilesInDirectoryMultiOutputStrategy" p:nameSuffix=".xml">
-                            <property name="directory">
-                                <bean parent="File" c:_="${mdq.output}"/>
-                            </property>
-                            <property name="nameTransformer">
-                                <bean parent="mda.PathSegmentStringTransformer"/>
-                            </property>
-                        </bean>
-                    </property>
-                </bean>
-
-            </list>
-        </property>
-
-    </bean>
-
-</beans>
diff --git a/mdx/incommon/mdq-combined-localkey.xml b/mdx/incommon/mdq-combined-localkey.xml
deleted file mode 100644
index 5e686953..00000000
--- a/mdx/incommon/mdq-combined-localkey.xml
+++ /dev/null
@@ -1,223 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-    Generate and sign per-entity metadata using a local private key resource.
--->
-<beans xmlns="http://www.springframework.org/schema/beans"
-    default-lazy-init="true"
-    xmlns:c="http://www.springframework.org/schema/c"
-    xmlns:p="http://www.springframework.org/schema/p"
-    xmlns:util="http://www.springframework.org/schema/util"
-    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-    xsi:schemaLocation="
-        http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd
-        http://www.springframework.org/schema/util http://www.springframework.org/schema/util/spring-util.xsd">
-
-    <!--
-        Import commonly used beans.
-    -->
-    <import resource="classpath:common-beans.xml"/>
-
-    <!--
-        Import inc-mda beans. (from import.xml)
-    -->
-    <import resource="classpath:uk/org/iay/incommon/mda/beans.xml"/>
-
-    <!--
-        Import channel-specific beans.
-    -->
-    <import resource="classpath:incommon/beans.xml"/>
-
-    <!-- This bean MUST be called "conversionService" to work properly. -->
-    <bean id="conversionService" class="org.springframework.context.support.ConversionServiceFactoryBean">
-        <property name="converters">
-            <set>
-                <bean class="net.shibboleth.ext.spring.config.DurationToLongConverter" />
-                <bean class="net.shibboleth.ext.spring.config.StringToIPRangeConverter" />
-                <bean class="net.shibboleth.ext.spring.config.BooleanToPredicateConverter" />
-                <bean class="net.shibboleth.ext.spring.config.StringBooleanToPredicateConverter" />
-                <bean class="net.shibboleth.ext.spring.config.StringToResourceConverter" />
-            </set>
-        </property>
-    </bean>
-
-    <!--
-        ***********************************************************
-        ***                                                     ***
-        ***   C O M M O N   O U T P U T   P R O C E S S I N G   ***
-        ***                                                     ***
-        ***********************************************************
-    -->
-
-    <bean id="common.output" parent="mda.CompositeStage">
-        <property name="composedStages">
-            <list>
-                <!-- Construct a new aggregate from the collection of entities. -->
-                <bean id="inc.assemble" parent="mda.EntitiesDescriptorAssemblerStage">
-                    <property name="itemOrderingStrategy">
-                        <bean parent="inc.InCommonEntityOrderingStrategy"
-                            c:_-ref="us_incommon_registrar"/>
-                    </property>
-                </bean>
-
-                <!-- Apply final tweaks to the aggregate. -->
-                <bean id="finalise" parent="incommon_finalise_parent">
-                    <property name="transformParameters">
-                        <map>
-                            <entry key="extraText" value="Contains InCommon and eduGAIN metadata"/>
-                            <entry key="publisher" value-ref="us_incommon_registrar"/>
-                        </map>
-                    </property>
-                </bean>
-
-                <!-- Normalise the use of namespace prefixes in the resulting XML document. -->
-                <bean id="normalise" parent="mda.XSLTransformationStage"
-                    p:XSLResource="classpath:incommon/ns_norm_import.xsl"/>
-            </list>
-        </property>
-    </bean>
-
-    <!--
-        *****************************************
-        ***                                   ***
-        ***   I D P - O N L Y   O U T P U T   ***
-        ***                                   ***
-        *****************************************
-    -->
-
-    <!--
-        idp.serialize
-
-        Writes the IdP-only aggregate out to a file.
-    -->
-    <bean id="idp.serialize" parent="mda.SerializationStage">
-        <property name="serializer" ref="serializer"/>
-        <property name="outputFile">
-            <bean parent="File">
-                <constructor-arg value="${mdq.output}/idps/all.xml"/>
-            </bean>
-        </property>
-    </bean>
-
-    <!--
-        idp.selector
-
-        Entities in the IdP-only aggregate are restricted to identity providers.
-    -->
-    <bean id="idp.selector" parent="mda.XPathItemSelectionStrategy">
-        <constructor-arg value="/md:EntityDescriptor[md:IDPSSODescriptor]"/>
-        <constructor-arg ref="commonNamespaces"/>
-    </bean>
-
-    <!--
-        idp.pipeline
-
-        Generates the IdP-only aggregate. The selector has already taken care
-        of removing non-IdP entities.
-    -->
-    <bean id="idp.pipeline" parent="mda.SimplePipeline">
-        <property name="stages">
-            <list>
-                <!-- Perform common steps in constructing an output aggregate. -->
-                <ref bean="common.output"/>
-
-                <!-- Write the resulting aggregate out to a file. -->
-                <ref bean="idp.serialize"/>
-            </list>
-        </property>
-    </bean>
-
-    <!--
-        Generate all required metadata for the InCommon MDQ service
-        using the localkey configuration:
-
-          * Per-entity metadata
-          * InCommon aggregate
-          * InCommon IdP-only aggregate
-
-        Properties:
-
-        Set sign.uk.keyPassword (note the slightly anomalous "uk" in that
-        identifier) to set the password to access the key resource. If not set,
-        this will be prompted for.
-    -->
-    <bean id="mdq-combined-localkey" parent="mda.SimplePipeline">
-        <property name="stages">
-            <list>
-                <!--
-                    Start with the unsigned production aggregate.
-                    TODO: (MDQ-177) For now, we're actually starting with a signed aggregate;
-                      this pipeline isn't configured to check the signature
-                -->
-                <bean id="production_aggregate" parent="mda.DOMResourceSourceStage">
-                    <property name="parserPool" ref="parserPool"/>
-                    <property name="DOMResource">
-                        <bean parent="FileSystemResource" c:_0="${mdq.input}"/>
-                    </property>
-                </bean>
-
-                <!-- Break down into individual entities. -->
-                <ref bean="disassemble"/>
-
-                <!-- Include a default registrationAuthority for each entity. -->
-                <ref bean="us_incommon_default_regauth"/>
-
-                <!-- Populate identifiers for future actions. -->
-                <ref bean="populateItemIds"/>
-                <ref bean="populateRegistrationAuthorities"/>
-
-                <!-- Discard entities in the InCommon entity blacklist. -->
-                <ref bean="remove_blacklisted_incommon_entities"/>
-
-                <!-- Fork new pipelines to generate special aggregates. -->
-                <bean id="demux" parent="mda.PipelineDemultiplexerStage">
-                    <property name="pipelineAndSelectionStrategies">
-                        <list>
-                            <!-- IdP-only aggregate. -->
-                            <bean class="net.shibboleth.utilities.java.support.collection.Pair">
-                                <constructor-arg ref="idp.pipeline"/>
-                                <constructor-arg ref="idp.selector"/>
-                            </bean>
-                        </list>
-                    </property>
-                    <property name="waitingForPipelines" value="true"/>
-                </bean>
-
-                <!-- Set ID, cacheDuration and validUntil attributes. -->
-                <bean parent="mda.GenerateIdStage"/>
-                <bean parent="mda.SetValidUntilStage" p:validityDuration="P14D"/>
-
-                <!-- Identity transform fixes signing issues. -->
-                <bean parent="mda.XSLTransformationStage"
-                    p:XSLResource="classpath:identity.xsl"/>
-
-                <!-- Sign each item. -->
-                <bean id="perform.signature" parent="mda.XMLSignatureSigningStage">
-                    <property name="privateKey">
-                        <bean parent="PrivateKeyFactoryBean"
-                            p:resource="${sign.keyResource}"
-                            p:privateKeyPassword="${sign.keyPassword}"
-                        />
-                    </property>
-                </bean>
-
-                <!-- Write individual entity documents to files. -->
-                <bean id="write.perentity" parent="mda.MultiOutputSerializationStage">
-                    <property name="serializer" ref="serializer"/>
-                    <property name="outputStrategy">
-                        <bean parent="mda.FilesInDirectoryMultiOutputStrategy" p:nameSuffix=".xml">
-                            <property name="directory">
-                                <bean parent="File" c:_="${mdq.output}"/>
-                            </property>
-                            <property name="nameTransformer">
-                                <bean parent="mda.PathSegmentStringTransformer"/>
-                            </property>
-                        </bean>
-                    </property>
-                </bean>
-
-            </list>
-        </property>
-
-    </bean>
-
-</beans>

From d45c6a4570feef2f4b8a92e8e2d8586f43078ea8 Mon Sep 17 00:00:00 2001
From: David Shafer <dshafer@internet2.edu>
Date: Fri, 8 Feb 2019 15:09:26 -0600
Subject: [PATCH 03/19] Comment conversionService

---
 mdx/incommon/mdq-all-cloudhsm.xml | 24 ++++++++++++------------
 mdx/incommon/mdq-all-localkey.xml | 24 ++++++++++++------------
 2 files changed, 24 insertions(+), 24 deletions(-)

diff --git a/mdx/incommon/mdq-all-cloudhsm.xml b/mdx/incommon/mdq-all-cloudhsm.xml
index e54e711a..2ddcb105 100644
--- a/mdx/incommon/mdq-all-cloudhsm.xml
+++ b/mdx/incommon/mdq-all-cloudhsm.xml
@@ -19,18 +19,18 @@
     <!-- Import mdq-all beans -->
     <import resource="classpath:incommon/mdq-all.xml" />
 
-    <!-- This bean MUST be called "conversionService" to work properly -->
-    <bean id="conversionService" class="org.springframework.context.support.ConversionServiceFactoryBean">
-        <property name="converters">
-            <set>
-                <bean class="net.shibboleth.ext.spring.config.DurationToLongConverter" />
-                <bean class="net.shibboleth.ext.spring.config.StringToIPRangeConverter" />
-                <bean class="net.shibboleth.ext.spring.config.BooleanToPredicateConverter" />
-                <bean class="net.shibboleth.ext.spring.config.StringBooleanToPredicateConverter" />
-                <bean class="net.shibboleth.ext.spring.config.StringToResourceConverter" />
-            </set>
-        </property>
-    </bean>
+<!--     This bean MUST be called "conversionService" to work properly -->
+<!--     <bean id="conversionService" class="org.springframework.context.support.ConversionServiceFactoryBean"> -->
+<!--         <property name="converters"> -->
+<!--             <set> -->
+<!--                 <bean class="net.shibboleth.ext.spring.config.DurationToLongConverter" /> -->
+<!--                 <bean class="net.shibboleth.ext.spring.config.StringToIPRangeConverter" /> -->
+<!--                 <bean class="net.shibboleth.ext.spring.config.BooleanToPredicateConverter" /> -->
+<!--                 <bean class="net.shibboleth.ext.spring.config.StringBooleanToPredicateConverter" /> -->
+<!--                 <bean class="net.shibboleth.ext.spring.config.StringToResourceConverter" /> -->
+<!--             </set> -->
+<!--         </property> -->
+<!--     </bean> -->
 
     <bean id="privateKeyFactory" class="uk.org.iay.incommon.mda.cloudhsm.CaviumPrivateKeyFactoryBean"
         p:username="${sign.keyUser}" p:password="${sign.keyPassword}" p:keyHandle="${sign.keyHandle}" />
diff --git a/mdx/incommon/mdq-all-localkey.xml b/mdx/incommon/mdq-all-localkey.xml
index d329dc75..f9a87501 100644
--- a/mdx/incommon/mdq-all-localkey.xml
+++ b/mdx/incommon/mdq-all-localkey.xml
@@ -19,18 +19,18 @@
     <!-- Import mdq-all beans -->
     <import resource="classpath:incommon/mdq-all.xml" />
 
-    <!-- This bean MUST be called "conversionService" to work properly -->
-    <bean id="conversionService" class="org.springframework.context.support.ConversionServiceFactoryBean">
-        <property name="converters">
-            <set>
-                <bean class="net.shibboleth.ext.spring.config.DurationToLongConverter" />
-                <bean class="net.shibboleth.ext.spring.config.StringToIPRangeConverter" />
-                <bean class="net.shibboleth.ext.spring.config.BooleanToPredicateConverter" />
-                <bean class="net.shibboleth.ext.spring.config.StringBooleanToPredicateConverter" />
-                <bean class="net.shibboleth.ext.spring.config.StringToResourceConverter" />
-            </set>
-        </property>
-    </bean>
+<!--     This bean MUST be called "conversionService" to work properly -->
+<!--     <bean id="conversionService" class="org.springframework.context.support.ConversionServiceFactoryBean"> -->
+<!--         <property name="converters"> -->
+<!--             <set> -->
+<!--                 <bean class="net.shibboleth.ext.spring.config.DurationToLongConverter" /> -->
+<!--                 <bean class="net.shibboleth.ext.spring.config.StringToIPRangeConverter" /> -->
+<!--                 <bean class="net.shibboleth.ext.spring.config.BooleanToPredicateConverter" /> -->
+<!--                 <bean class="net.shibboleth.ext.spring.config.StringBooleanToPredicateConverter" /> -->
+<!--                 <bean class="net.shibboleth.ext.spring.config.StringToResourceConverter" /> -->
+<!--             </set> -->
+<!--         </property> -->
+<!--     </bean> -->
 
     <bean id="privateKeyFactory" parent="PrivateKeyFactoryBean" p:resource="${sign.keyResource}"
         p:privateKeyPassword="${sign.keyPassword}" />

From 1b39bae420ad2bf77ac0abf64b8a144c4b2031f8 Mon Sep 17 00:00:00 2001
From: David Shafer <dshafer@internet2.edu>
Date: Fri, 8 Feb 2019 15:43:18 -0600
Subject: [PATCH 04/19] Finish rule updates

---
 build.xml                         |  2 +-
 mdx/incommon/mdq-all-cloudhsm.xml | 13 -------------
 mdx/incommon/mdq-all-localkey.xml | 13 -------------
 3 files changed, 1 insertion(+), 27 deletions(-)

diff --git a/build.xml b/build.xml
index 083c6411..8e10af4c 100644
--- a/build.xml
+++ b/build.xml
@@ -2685,7 +2685,7 @@
 
     <!--
         inc.mdq.generate.all.localkey
-
+    
         Generate all metadata required for the InCommon federation MDQ service
         using the local key configuration.
 
diff --git a/mdx/incommon/mdq-all-cloudhsm.xml b/mdx/incommon/mdq-all-cloudhsm.xml
index 2ddcb105..35a917d7 100644
--- a/mdx/incommon/mdq-all-cloudhsm.xml
+++ b/mdx/incommon/mdq-all-cloudhsm.xml
@@ -19,19 +19,6 @@
     <!-- Import mdq-all beans -->
     <import resource="classpath:incommon/mdq-all.xml" />
 
-<!--     This bean MUST be called "conversionService" to work properly -->
-<!--     <bean id="conversionService" class="org.springframework.context.support.ConversionServiceFactoryBean"> -->
-<!--         <property name="converters"> -->
-<!--             <set> -->
-<!--                 <bean class="net.shibboleth.ext.spring.config.DurationToLongConverter" /> -->
-<!--                 <bean class="net.shibboleth.ext.spring.config.StringToIPRangeConverter" /> -->
-<!--                 <bean class="net.shibboleth.ext.spring.config.BooleanToPredicateConverter" /> -->
-<!--                 <bean class="net.shibboleth.ext.spring.config.StringBooleanToPredicateConverter" /> -->
-<!--                 <bean class="net.shibboleth.ext.spring.config.StringToResourceConverter" /> -->
-<!--             </set> -->
-<!--         </property> -->
-<!--     </bean> -->
-
     <bean id="privateKeyFactory" class="uk.org.iay.incommon.mda.cloudhsm.CaviumPrivateKeyFactoryBean"
         p:username="${sign.keyUser}" p:password="${sign.keyPassword}" p:keyHandle="${sign.keyHandle}" />
 
diff --git a/mdx/incommon/mdq-all-localkey.xml b/mdx/incommon/mdq-all-localkey.xml
index f9a87501..01d1b22c 100644
--- a/mdx/incommon/mdq-all-localkey.xml
+++ b/mdx/incommon/mdq-all-localkey.xml
@@ -19,19 +19,6 @@
     <!-- Import mdq-all beans -->
     <import resource="classpath:incommon/mdq-all.xml" />
 
-<!--     This bean MUST be called "conversionService" to work properly -->
-<!--     <bean id="conversionService" class="org.springframework.context.support.ConversionServiceFactoryBean"> -->
-<!--         <property name="converters"> -->
-<!--             <set> -->
-<!--                 <bean class="net.shibboleth.ext.spring.config.DurationToLongConverter" /> -->
-<!--                 <bean class="net.shibboleth.ext.spring.config.StringToIPRangeConverter" /> -->
-<!--                 <bean class="net.shibboleth.ext.spring.config.BooleanToPredicateConverter" /> -->
-<!--                 <bean class="net.shibboleth.ext.spring.config.StringBooleanToPredicateConverter" /> -->
-<!--                 <bean class="net.shibboleth.ext.spring.config.StringToResourceConverter" /> -->
-<!--             </set> -->
-<!--         </property> -->
-<!--     </bean> -->
-
     <bean id="privateKeyFactory" parent="PrivateKeyFactoryBean" p:resource="${sign.keyResource}"
         p:privateKeyPassword="${sign.keyPassword}" />
 

From 9f3b91e3a9ec06fec108998c9836794682a15bd8 Mon Sep 17 00:00:00 2001
From: David Shafer <dshafer@internet2.edu>
Date: Fri, 8 Feb 2019 16:20:43 -0600
Subject: [PATCH 05/19] Add comments

---
 mdx/incommon/mdq-all-cloudhsm.xml | 14 ++++----------
 mdx/incommon/mdq-all-localkey.xml | 14 ++++----------
 2 files changed, 8 insertions(+), 20 deletions(-)

diff --git a/mdx/incommon/mdq-all-cloudhsm.xml b/mdx/incommon/mdq-all-cloudhsm.xml
index 35a917d7..db3e85db 100644
--- a/mdx/incommon/mdq-all-cloudhsm.xml
+++ b/mdx/incommon/mdq-all-cloudhsm.xml
@@ -1,5 +1,6 @@
 <?xml version="1.0" encoding="UTF-8"?>
 
+<!-- Generate and sign (using AWS CloudHSM) all metadata required by the InCommon MDQ service -->
 <beans xmlns="http://www.springframework.org/schema/beans" default-lazy-init="true"
     xmlns:c="http://www.springframework.org/schema/c" xmlns:p="http://www.springframework.org/schema/p"
     xmlns:util="http://www.springframework.org/schema/util" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
@@ -7,21 +8,14 @@
         http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd
         http://www.springframework.org/schema/util http://www.springframework.org/schema/util/spring-util.xsd">
 
-    <!-- Import commonly used beans -->
-    <import resource="classpath:common-beans.xml" />
-
-    <!-- Import inc-mda beans (from import.xml) -->
-    <import resource="classpath:uk/org/iay/incommon/mda/beans.xml" />
-
-    <!-- Import channel-specific beans -->
-    <import resource="classpath:incommon/beans.xml" />
-
-    <!-- Import mdq-all beans -->
+    <!-- Import generateAll and supporting beans -->
     <import resource="classpath:incommon/mdq-all.xml" />
 
+    <!-- Define a private key factory (based on AWS CloudHSM) to be used by the generateAll stage -->
     <bean id="privateKeyFactory" class="uk.org.iay.incommon.mda.cloudhsm.CaviumPrivateKeyFactoryBean"
         p:username="${sign.keyUser}" p:password="${sign.keyPassword}" p:keyHandle="${sign.keyHandle}" />
 
+    <!-- Generate all signed metadata required by the InCommon MDQ service  -->
     <bean id="mdq-all-cloudhsm" parent="mda.SimplePipeline">
         <property name="stages">
             <list>
diff --git a/mdx/incommon/mdq-all-localkey.xml b/mdx/incommon/mdq-all-localkey.xml
index 01d1b22c..c2617cb6 100644
--- a/mdx/incommon/mdq-all-localkey.xml
+++ b/mdx/incommon/mdq-all-localkey.xml
@@ -1,5 +1,6 @@
 <?xml version="1.0" encoding="UTF-8"?>
 
+<!-- Generate and sign (using a local key) all metadata required by the InCommon MDQ service -->
 <beans xmlns="http://www.springframework.org/schema/beans" default-lazy-init="true"
     xmlns:c="http://www.springframework.org/schema/c" xmlns:p="http://www.springframework.org/schema/p"
     xmlns:util="http://www.springframework.org/schema/util" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
@@ -7,21 +8,14 @@
         http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd
         http://www.springframework.org/schema/util http://www.springframework.org/schema/util/spring-util.xsd">
 
-    <!-- Import commonly used beans -->
-    <import resource="classpath:common-beans.xml" />
-
-    <!-- Import inc-mda beans (from import.xml) -->
-    <import resource="classpath:uk/org/iay/incommon/mda/beans.xml" />
-
-    <!-- Import channel-specific beans -->
-    <import resource="classpath:incommon/beans.xml" />
-
-    <!-- Import mdq-all beans -->
+    <!-- Import generateAll and supporting beans -->
     <import resource="classpath:incommon/mdq-all.xml" />
 
+    <!-- Define a private key factory (based on a local key) to be used by the generateAll stage -->
     <bean id="privateKeyFactory" parent="PrivateKeyFactoryBean" p:resource="${sign.keyResource}"
         p:privateKeyPassword="${sign.keyPassword}" />
 
+    <!-- Generate all signed metadata required by the InCommon MDQ service  -->
     <bean id="mdq-all-localkey" parent="mda.SimplePipeline">
         <property name="stages">
             <list>

From 1d9f7646524165da9aac8c533c0f3a7a7246d13b Mon Sep 17 00:00:00 2001
From: David Shafer <dshafer@internet2.edu>
Date: Tue, 12 Feb 2019 12:01:08 -0600
Subject: [PATCH 06/19] Update properties

---
 build.xml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/build.xml b/build.xml
index 8e10af4c..3b9727cc 100644
--- a/build.xml
+++ b/build.xml
@@ -2672,7 +2672,7 @@
         <property name="mda.jni.path" value="/opt/cloudhsm/lib"/>
 		<property name="mda.mdq.input" value="${mda.inc.imported.xml}"/>
         <property name="mda.sign.keyHandle" value="${sign.keyHandle}"/>
-		<property name="mda.sign.keyPassword" value="${sign.uk.keyPassword}"/>
+		<property name="mda.sign.keyPassword" value="${sign.keyPassword}"/>
         <property name="mda.sign.keyUser" value="${sign.keyUser}"/>
 		<echo>Generating MDQ metadata in ${mda.mdq.output}</echo>
 		<echo>    from unsigned aggregate in ${mda.mdq.input}</echo>
@@ -2697,7 +2697,7 @@
     -->
 	<target name="inc.mdq.generate.all.localkey" depends="get.sign.uk.keyPassword">
 		<property name="mda.mdq.input" value="${mda.inc.imported.xml}"/>
-		<property name="mda.sign.keyPassword" value="${sign.uk.keyPassword}"/>
+		<property name="mda.sign.keyPassword" value="${sign.keyPassword}"/>
 		<echo>Generating MDQ metadata in ${mda.mdq.output}</echo>
 		<echo>    from unsigned aggregate in ${mda.mdq.input}</echo>
 		<delete dir="${mdq.output.dir}" quiet="true"/>

From 3a5c0c0021178ee2cc7227c0bb7dd5d6dc442194 Mon Sep 17 00:00:00 2001
From: David Shafer <dshafer@internet2.edu>
Date: Tue, 12 Mar 2019 18:39:16 -0500
Subject: [PATCH 07/19] Remove spaces from empty line

---
 build.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build.xml b/build.xml
index 3b9727cc..aa493fca 100644
--- a/build.xml
+++ b/build.xml
@@ -2685,7 +2685,7 @@
 
     <!--
         inc.mdq.generate.all.localkey
-    
+
         Generate all metadata required for the InCommon federation MDQ service
         using the local key configuration.
 

From 8a77022e3c4447d69737650e5891ac4d22818fd6 Mon Sep 17 00:00:00 2001
From: David Shafer <dshafer@internet2.edu>
Date: Tue, 12 Mar 2019 18:55:50 -0500
Subject: [PATCH 08/19] Change tabs to spaces

---
 build.xml | 38 +++++++++++++++++++-------------------
 1 file changed, 19 insertions(+), 19 deletions(-)

diff --git a/build.xml b/build.xml
index aa493fca..e062dc83 100644
--- a/build.xml
+++ b/build.xml
@@ -2670,17 +2670,17 @@
     <target name="inc.mdq.generate.all.cloudhsm" depends="get.sign.uk.keyPassword">
         <property name="mda.classpath.extra" value="inc-mda-cloudhsm"/>
         <property name="mda.jni.path" value="/opt/cloudhsm/lib"/>
-		<property name="mda.mdq.input" value="${mda.inc.imported.xml}"/>
+        <property name="mda.mdq.input" value="${mda.inc.imported.xml}"/>
         <property name="mda.sign.keyHandle" value="${sign.keyHandle}"/>
-		<property name="mda.sign.keyPassword" value="${sign.keyPassword}"/>
+        <property name="mda.sign.keyPassword" value="${sign.keyPassword}"/>
         <property name="mda.sign.keyUser" value="${sign.keyUser}"/>
-		<echo>Generating MDQ metadata in ${mda.mdq.output}</echo>
-		<echo>    from unsigned aggregate in ${mda.mdq.input}</echo>
-		<delete dir="${mdq.output.dir}" quiet="true"/>
-		<mkdir dir="${mdq.output.dir}"/>
-		<mkdir dir="${mdq.output.dir}/idps"/>
-		<CHANNEL.do channel="incommon" verb="mdq-all-cloudhsm"/>
-		<echo>Generation complete.</echo>
+        <echo>Generating MDQ metadata in ${mda.mdq.output}</echo>
+        <echo>    from unsigned aggregate in ${mda.mdq.input}</echo>
+        <delete dir="${mdq.output.dir}" quiet="true"/>
+        <mkdir dir="${mdq.output.dir}"/>
+        <mkdir dir="${mdq.output.dir}/idps"/>
+        <CHANNEL.do channel="incommon" verb="mdq-all-cloudhsm"/>
+        <echo>Generation complete.</echo>
     </target>
 
     <!--
@@ -2695,16 +2695,16 @@
         identifier) to set the password to access the key resource. If not set,
         this will be prompted for.
     -->
-	<target name="inc.mdq.generate.all.localkey" depends="get.sign.uk.keyPassword">
-		<property name="mda.mdq.input" value="${mda.inc.imported.xml}"/>
-		<property name="mda.sign.keyPassword" value="${sign.keyPassword}"/>
-		<echo>Generating MDQ metadata in ${mda.mdq.output}</echo>
-		<echo>    from unsigned aggregate in ${mda.mdq.input}</echo>
-		<delete dir="${mdq.output.dir}" quiet="true"/>
-		<mkdir dir="${mdq.output.dir}"/>
-		<mkdir dir="${mdq.output.dir}/idps"/>
-		<CHANNEL.do channel="incommon" verb="mdq-all-localkey"/>
-		<echo>Generation complete.</echo>
+    <target name="inc.mdq.generate.all.localkey" depends="get.sign.uk.keyPassword">
+        <property name="mda.mdq.input" value="${mda.inc.imported.xml}"/>
+        <property name="mda.sign.keyPassword" value="${sign.keyPassword}"/>
+        <echo>Generating MDQ metadata in ${mda.mdq.output}</echo>
+        <echo>    from unsigned aggregate in ${mda.mdq.input}</echo>
+        <delete dir="${mdq.output.dir}" quiet="true"/>
+        <mkdir dir="${mdq.output.dir}"/>
+        <mkdir dir="${mdq.output.dir}/idps"/>
+        <CHANNEL.do channel="incommon" verb="mdq-all-localkey"/>
+        <echo>Generation complete.</echo>
     </target>
 
     <!--

From d855821658aed1c0044263c2493b68abecd6fe8a Mon Sep 17 00:00:00 2001
From: David Shafer <dshafer@internet2.edu>
Date: Tue, 12 Mar 2019 19:02:43 -0500
Subject: [PATCH 09/19] Reorganize targets

---
 build.xml | 72 ++++++++++++++++++++++++++++---------------------------
 1 file changed, 37 insertions(+), 35 deletions(-)

diff --git a/build.xml b/build.xml
index e062dc83..ae465411 100644
--- a/build.xml
+++ b/build.xml
@@ -2656,10 +2656,10 @@
     </target>
 
     <!--
-        inc.mdq.generate.all.cloudhsm
+        inc.mdq.generate.localkey
 
-        Generate all metadata required for the InCommon federation MDQ service
-        using the AWS CloudHSM configuration.
+        Generate per-entity metadata for the InCommon federation
+        using the localkey configuration.
 
         Properties:
 
@@ -2667,22 +2667,18 @@
         identifier) to set the password to access the key resource. If not set,
         this will be prompted for.
     -->
-    <target name="inc.mdq.generate.all.cloudhsm" depends="get.sign.uk.keyPassword">
-        <property name="mda.classpath.extra" value="inc-mda-cloudhsm"/>
-        <property name="mda.jni.path" value="/opt/cloudhsm/lib"/>
+    <target name="inc.mdq.generate.localkey" depends="get.sign.uk.keyPassword">
         <property name="mda.mdq.input" value="${mda.inc.imported.xml}"/>
-        <property name="mda.sign.keyHandle" value="${sign.keyHandle}"/>
-        <property name="mda.sign.keyPassword" value="${sign.keyPassword}"/>
-        <property name="mda.sign.keyUser" value="${sign.keyUser}"/>
-        <echo>Generating MDQ metadata in ${mda.mdq.output}</echo>
+        <property name="mda.sign.keyPassword" value="${sign.uk.keyPassword}"/>
+        <echo>Generating per-entity metadata in ${mda.mdq.output}</echo>
         <echo>    from unsigned aggregate in ${mda.mdq.input}</echo>
         <delete dir="${mdq.output.dir}" quiet="true"/>
         <mkdir dir="${mdq.output.dir}"/>
-        <mkdir dir="${mdq.output.dir}/idps"/>
-        <CHANNEL.do channel="incommon" verb="mdq-all-cloudhsm"/>
+        <CHANNEL.do channel="incommon" verb="mdq-multisign-localkey"/>
         <echo>Generation complete.</echo>
     </target>
 
+
     <!--
         inc.mdq.generate.all.localkey
 
@@ -2707,29 +2703,7 @@
         <echo>Generation complete.</echo>
     </target>
 
-    <!--
-        inc.mdq.generate.localkey
-
-        Generate per-entity metadata for the InCommon federation
-        using the localkey configuration.
-
-        Properties:
-
-        Set sign.uk.keyPassword (note the slightly anomalous "uk" in that
-        identifier) to set the password to access the key resource. If not set,
-        this will be prompted for.
-    -->
-    <target name="inc.mdq.generate.localkey" depends="get.sign.uk.keyPassword">
-        <property name="mda.mdq.input" value="${mda.inc.imported.xml}"/>
-        <property name="mda.sign.keyPassword" value="${sign.uk.keyPassword}"/>
-        <echo>Generating per-entity metadata in ${mda.mdq.output}</echo>
-        <echo>    from unsigned aggregate in ${mda.mdq.input}</echo>
-        <delete dir="${mdq.output.dir}" quiet="true"/>
-        <mkdir dir="${mdq.output.dir}"/>
-        <CHANNEL.do channel="incommon" verb="mdq-multisign-localkey"/>
-        <echo>Generation complete.</echo>
-    </target>
-
+    
     <!--
         inc.generate.mdq
 
@@ -2766,6 +2740,34 @@
         <CHANNEL.do channel="incommon" verb="mdq-multisign-cloudhsm"/>
         <echo>Generation complete.</echo>
     </target>
+    
+    <!--
+        inc.mdq.generate.all.cloudhsm
+
+        Generate all metadata required for the InCommon federation MDQ service
+        using the AWS CloudHSM configuration.
+
+        Properties:
+
+        Set sign.uk.keyPassword (note the slightly anomalous "uk" in that
+        identifier) to set the password to access the key resource. If not set,
+        this will be prompted for.
+    -->
+    <target name="inc.mdq.generate.all.cloudhsm" depends="get.sign.uk.keyPassword">
+        <property name="mda.classpath.extra" value="inc-mda-cloudhsm"/>
+        <property name="mda.jni.path" value="/opt/cloudhsm/lib"/>
+        <property name="mda.mdq.input" value="${mda.inc.imported.xml}"/>
+        <property name="mda.sign.keyHandle" value="${sign.keyHandle}"/>
+        <property name="mda.sign.keyPassword" value="${sign.keyPassword}"/>
+        <property name="mda.sign.keyUser" value="${sign.keyUser}"/>
+        <echo>Generating MDQ metadata in ${mda.mdq.output}</echo>
+        <echo>    from unsigned aggregate in ${mda.mdq.input}</echo>
+        <delete dir="${mdq.output.dir}" quiet="true"/>
+        <mkdir dir="${mdq.output.dir}"/>
+        <mkdir dir="${mdq.output.dir}/idps"/>
+        <CHANNEL.do channel="incommon" verb="mdq-all-cloudhsm"/>
+        <echo>Generation complete.</echo>
+    </target>
 
     <!--
         inc.edugain.report

From 4fdd82475e9b79218cd671121ecfb1200369f0c6 Mon Sep 17 00:00:00 2001
From: David Shafer <dshafer@internet2.edu>
Date: Mon, 28 Jan 2019 16:11:50 -0600
Subject: [PATCH 10/19] Create the IdP-only aggregate (#1)

* Create new pipeline for MDQ and IdP-only aggregate
* Move IdP-only aggregate to idps/all.xml
* Rename mdq-new-* to mdq-combined-*
* Reference JIRA issues in TODOs
---
 build.xml                              |  60 +++++++
 mdx/incommon/mdq-combined-cloudhsm.xml | 224 +++++++++++++++++++++++++
 mdx/incommon/mdq-combined-localkey.xml | 223 ++++++++++++++++++++++++
 3 files changed, 507 insertions(+)
 create mode 100644 mdx/incommon/mdq-combined-cloudhsm.xml
 create mode 100644 mdx/incommon/mdq-combined-localkey.xml

diff --git a/build.xml b/build.xml
index 83bf53f2..bdfb205e 100644
--- a/build.xml
+++ b/build.xml
@@ -2664,6 +2664,66 @@
         <echo>Generation complete.</echo>
     </target>
 
+    <!--
+        inc.mdq-combined.localkey
+
+        Generate all required metadata for the InCommon MDQ service
+        using the localkey configuration:
+
+          * Per-entity metadata
+          * InCommon IdP-only aggregate
+
+        Properties:
+
+        Set sign.uk.keyPassword (note the slightly anomalous "uk" in that
+        identifier) to set the password to access the key resource. If not set,
+        this will be prompted for.
+    -->
+    <target name="inc.mdq.combined.localkey" depends="get.sign.uk.keyPassword">
+      <property name="mda.mdq.input" value="${mda.inc.imported.xml}"/>
+      <property name="mda.sign.keyPassword" value="${sign.uk.keyPassword}"/>
+      <echo>Generating per-entity metadata in ${mda.mdq.output}</echo>
+      <echo>    (IdP-only aggregate in ${mda.mdq.output}/idps.xml)</echo>
+      <echo>    from unsigned aggregate in ${mda.mdq.input}</echo>
+      <delete dir="${mdq.output.dir}" quiet="true"/>
+      <mkdir dir="${mdq.output.dir}"/>
+      <mkdir dir="${mdq.output.dir}/idps"/>
+      <CHANNEL.do channel="incommon" verb="mdq-combined-localkey"/>
+      <echo>Generation complete.</echo>
+    </target>
+
+    <!--
+        inc.mdq.combined.cloudhsm
+
+        Generate all required metadata for the InCommon MDQ service
+        using the cloudhsm configuration:
+
+          * Per-entity metadata
+          * InCommon IdP-only aggregate
+
+        Properties:
+
+        Set sign.uk.keyPassword (note the slightly anomalous "uk" in that
+        identifier) to set the password to access the key resource. If not set,
+        this will be prompted for.
+    -->
+    <target name="inc.mdq.combined.cloudhsm" depends="get.sign.uk.keyPassword">
+      <property name="mda.mdq.input" value="${mda.inc.imported.xml}"/>
+      <property name="mda.sign.keyHandle" value="${sign.keyHandle}"/>
+      <property name="mda.sign.keyUser" value="${sign.keyUser}"/>
+      <property name="mda.sign.keyPassword" value="${sign.keyPassword}"/>
+      <property name="mda.classpath.extra" value="inc-mda-cloudhsm"/>
+      <property name="mda.jni.path" value="/opt/cloudhsm/lib"/>
+      <echo>Generating per-entity metadata in ${mda.mdq.output}</echo>
+      <echo>    (IdP-only aggregate in ${mda.mdq.output}/idps.xml)</echo>
+      <echo>    from unsigned aggregate in ${mda.mdq.input}</echo>
+      <delete dir="${mdq.output.dir}" quiet="true"/>
+      <mkdir dir="${mdq.output.dir}"/>
+      <mkdir dir="${mdq.output.dir}/idps"/>
+      <CHANNEL.do channel="incommon" verb="mdq-combined-cloudhsm"/>
+      <echo>Generation complete.</echo>
+    </target>
+
     <!--
         inc.mdq.generate.localkey
 
diff --git a/mdx/incommon/mdq-combined-cloudhsm.xml b/mdx/incommon/mdq-combined-cloudhsm.xml
new file mode 100644
index 00000000..6a54503d
--- /dev/null
+++ b/mdx/incommon/mdq-combined-cloudhsm.xml
@@ -0,0 +1,224 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+    Generate and sign per-entity metadata using a local private key resource.
+-->
+<beans xmlns="http://www.springframework.org/schema/beans"
+    default-lazy-init="true"
+    xmlns:c="http://www.springframework.org/schema/c"
+    xmlns:p="http://www.springframework.org/schema/p"
+    xmlns:util="http://www.springframework.org/schema/util"
+    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+    xsi:schemaLocation="
+        http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd
+        http://www.springframework.org/schema/util http://www.springframework.org/schema/util/spring-util.xsd">
+
+    <!--
+        Import commonly used beans.
+    -->
+    <import resource="classpath:common-beans.xml"/>
+
+    <!--
+        Import inc-mda beans. (from import.xml)
+    -->
+    <import resource="classpath:uk/org/iay/incommon/mda/beans.xml"/>
+
+    <!--
+        Import channel-specific beans.
+    -->
+    <import resource="classpath:incommon/beans.xml"/>
+
+    <!-- This bean MUST be called "conversionService" to work properly. -->
+    <bean id="conversionService" class="org.springframework.context.support.ConversionServiceFactoryBean">
+        <property name="converters">
+            <set>
+                <bean class="net.shibboleth.ext.spring.config.DurationToLongConverter" />
+                <bean class="net.shibboleth.ext.spring.config.StringToIPRangeConverter" />
+                <bean class="net.shibboleth.ext.spring.config.BooleanToPredicateConverter" />
+                <bean class="net.shibboleth.ext.spring.config.StringBooleanToPredicateConverter" />
+                <bean class="net.shibboleth.ext.spring.config.StringToResourceConverter" />
+            </set>
+        </property>
+    </bean>
+
+    <!--
+        ***********************************************************
+        ***                                                     ***
+        ***   C O M M O N   O U T P U T   P R O C E S S I N G   ***
+        ***                                                     ***
+        ***********************************************************
+    -->
+
+    <bean id="common.output" parent="mda.CompositeStage">
+        <property name="composedStages">
+            <list>
+                <!-- Construct a new aggregate from the collection of entities. -->
+                <bean id="inc.assemble" parent="mda.EntitiesDescriptorAssemblerStage">
+                    <property name="itemOrderingStrategy">
+                        <bean parent="inc.InCommonEntityOrderingStrategy"
+                            c:_-ref="us_incommon_registrar"/>
+                    </property>
+                </bean>
+
+                <!-- Apply final tweaks to the aggregate. -->
+                <bean id="finalise" parent="incommon_finalise_parent">
+                    <property name="transformParameters">
+                        <map>
+                            <entry key="extraText" value="Contains InCommon and eduGAIN metadata"/>
+                            <entry key="publisher" value-ref="us_incommon_registrar"/>
+                        </map>
+                    </property>
+                </bean>
+
+                <!-- Normalise the use of namespace prefixes in the resulting XML document. -->
+                <bean id="normalise" parent="mda.XSLTransformationStage"
+                    p:XSLResource="classpath:incommon/ns_norm_import.xsl"/>
+            </list>
+        </property>
+    </bean>
+
+    <!--
+        *****************************************
+        ***                                   ***
+        ***   I D P - O N L Y   O U T P U T   ***
+        ***                                   ***
+        *****************************************
+    -->
+
+    <!--
+        idp.serialize
+
+        Writes the IdP-only aggregate out to a file.
+    -->
+    <bean id="idp.serialize" parent="mda.SerializationStage">
+        <property name="serializer" ref="serializer"/>
+        <property name="outputFile">
+            <bean parent="File">
+                <constructor-arg value="${mdq.output}/idps/all.xml"/>
+            </bean>
+        </property>
+    </bean>
+
+    <!--
+        idp.selector
+
+        Entities in the IdP-only aggregate are restricted to identity providers.
+    -->
+    <bean id="idp.selector" parent="mda.XPathItemSelectionStrategy">
+        <constructor-arg value="/md:EntityDescriptor[md:IDPSSODescriptor]"/>
+        <constructor-arg ref="commonNamespaces"/>
+    </bean>
+
+    <!--
+        idp.pipeline
+
+        Generates the IdP-only aggregate. The selector has already taken care
+        of removing non-IdP entities.
+    -->
+    <bean id="idp.pipeline" parent="mda.SimplePipeline">
+        <property name="stages">
+            <list>
+                <!-- Perform common steps in constructing an output aggregate. -->
+                <ref bean="common.output"/>
+
+                <!-- Write the resulting aggregate out to a file. -->
+                <ref bean="idp.serialize"/>
+            </list>
+        </property>
+    </bean>
+
+    <!--
+        Generate all required metadata for the InCommon MDQ service
+        using the localkey configuration:
+
+          * Per-entity metadata
+          * InCommon aggregate
+          * InCommon IdP-only aggregate
+
+        Properties:
+
+        Set sign.uk.keyPassword (note the slightly anomalous "uk" in that
+        identifier) to set the password to access the key resource. If not set,
+        this will be prompted for.
+    -->
+    <bean id="mdq-combined-cloudhsm" parent="mda.SimplePipeline">
+        <property name="stages">
+            <list>
+                <!--
+                    Start with the unsigned production aggregate.
+                    TODO: (MDQ-177) For now, we're actually starting with a signed aggregate;
+                      this pipeline isn't configured to check the signature
+                -->
+                <bean id="production_aggregate" parent="mda.DOMResourceSourceStage">
+                    <property name="parserPool" ref="parserPool"/>
+                    <property name="DOMResource">
+                        <bean parent="FileSystemResource" c:_0="${mdq.input}"/>
+                    </property>
+                </bean>
+
+                <!-- Break down into individual entities. -->
+                <ref bean="disassemble"/>
+
+                <!-- Include a default registrationAuthority for each entity. -->
+                <ref bean="us_incommon_default_regauth"/>
+
+                <!-- Populate identifiers for future actions. -->
+                <ref bean="populateItemIds"/>
+                <ref bean="populateRegistrationAuthorities"/>
+
+                <!-- Discard entities in the InCommon entity blacklist. -->
+                <ref bean="remove_blacklisted_incommon_entities"/>
+
+                <!-- Fork new pipelines to generate special aggregates. -->
+                <bean id="demux" parent="mda.PipelineDemultiplexerStage">
+                    <property name="pipelineAndSelectionStrategies">
+                        <list>
+                            <!-- IdP-only aggregate. -->
+                            <bean class="net.shibboleth.utilities.java.support.collection.Pair">
+                                <constructor-arg ref="idp.pipeline"/>
+                                <constructor-arg ref="idp.selector"/>
+                            </bean>
+                        </list>
+                    </property>
+                    <property name="waitingForPipelines" value="true"/>
+                </bean>
+
+                <!-- Set ID, cacheDuration and validUntil attributes. -->
+                <bean parent="mda.GenerateIdStage"/>
+                <bean parent="mda.SetValidUntilStage" p:validityDuration="P14D"/>
+
+                <!-- Identity transform fixes signing issues. -->
+                <bean parent="mda.XSLTransformationStage"
+                    p:XSLResource="classpath:identity.xsl"/>
+
+                <!-- Sign each item. -->
+                <bean id="perform.signature" parent="mda.XMLSignatureSigningStage">
+                    <property name="privateKey">
+                      <bean class="uk.org.iay.incommon.mda.cloudhsm.CaviumPrivateKeyFactoryBean"
+                          p:username="${sign.keyUser}"
+                          p:password="${sign.keyPassword}"
+                          p:keyHandle="${sign.keyHandle}"
+                      />
+                    </property>
+                </bean>
+
+                <!-- Write individual entity documents to files. -->
+                <bean id="write.perentity" parent="mda.MultiOutputSerializationStage">
+                    <property name="serializer" ref="serializer"/>
+                    <property name="outputStrategy">
+                        <bean parent="mda.FilesInDirectoryMultiOutputStrategy" p:nameSuffix=".xml">
+                            <property name="directory">
+                                <bean parent="File" c:_="${mdq.output}"/>
+                            </property>
+                            <property name="nameTransformer">
+                                <bean parent="mda.PathSegmentStringTransformer"/>
+                            </property>
+                        </bean>
+                    </property>
+                </bean>
+
+            </list>
+        </property>
+
+    </bean>
+
+</beans>
diff --git a/mdx/incommon/mdq-combined-localkey.xml b/mdx/incommon/mdq-combined-localkey.xml
new file mode 100644
index 00000000..5e686953
--- /dev/null
+++ b/mdx/incommon/mdq-combined-localkey.xml
@@ -0,0 +1,223 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+    Generate and sign per-entity metadata using a local private key resource.
+-->
+<beans xmlns="http://www.springframework.org/schema/beans"
+    default-lazy-init="true"
+    xmlns:c="http://www.springframework.org/schema/c"
+    xmlns:p="http://www.springframework.org/schema/p"
+    xmlns:util="http://www.springframework.org/schema/util"
+    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+    xsi:schemaLocation="
+        http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd
+        http://www.springframework.org/schema/util http://www.springframework.org/schema/util/spring-util.xsd">
+
+    <!--
+        Import commonly used beans.
+    -->
+    <import resource="classpath:common-beans.xml"/>
+
+    <!--
+        Import inc-mda beans. (from import.xml)
+    -->
+    <import resource="classpath:uk/org/iay/incommon/mda/beans.xml"/>
+
+    <!--
+        Import channel-specific beans.
+    -->
+    <import resource="classpath:incommon/beans.xml"/>
+
+    <!-- This bean MUST be called "conversionService" to work properly. -->
+    <bean id="conversionService" class="org.springframework.context.support.ConversionServiceFactoryBean">
+        <property name="converters">
+            <set>
+                <bean class="net.shibboleth.ext.spring.config.DurationToLongConverter" />
+                <bean class="net.shibboleth.ext.spring.config.StringToIPRangeConverter" />
+                <bean class="net.shibboleth.ext.spring.config.BooleanToPredicateConverter" />
+                <bean class="net.shibboleth.ext.spring.config.StringBooleanToPredicateConverter" />
+                <bean class="net.shibboleth.ext.spring.config.StringToResourceConverter" />
+            </set>
+        </property>
+    </bean>
+
+    <!--
+        ***********************************************************
+        ***                                                     ***
+        ***   C O M M O N   O U T P U T   P R O C E S S I N G   ***
+        ***                                                     ***
+        ***********************************************************
+    -->
+
+    <bean id="common.output" parent="mda.CompositeStage">
+        <property name="composedStages">
+            <list>
+                <!-- Construct a new aggregate from the collection of entities. -->
+                <bean id="inc.assemble" parent="mda.EntitiesDescriptorAssemblerStage">
+                    <property name="itemOrderingStrategy">
+                        <bean parent="inc.InCommonEntityOrderingStrategy"
+                            c:_-ref="us_incommon_registrar"/>
+                    </property>
+                </bean>
+
+                <!-- Apply final tweaks to the aggregate. -->
+                <bean id="finalise" parent="incommon_finalise_parent">
+                    <property name="transformParameters">
+                        <map>
+                            <entry key="extraText" value="Contains InCommon and eduGAIN metadata"/>
+                            <entry key="publisher" value-ref="us_incommon_registrar"/>
+                        </map>
+                    </property>
+                </bean>
+
+                <!-- Normalise the use of namespace prefixes in the resulting XML document. -->
+                <bean id="normalise" parent="mda.XSLTransformationStage"
+                    p:XSLResource="classpath:incommon/ns_norm_import.xsl"/>
+            </list>
+        </property>
+    </bean>
+
+    <!--
+        *****************************************
+        ***                                   ***
+        ***   I D P - O N L Y   O U T P U T   ***
+        ***                                   ***
+        *****************************************
+    -->
+
+    <!--
+        idp.serialize
+
+        Writes the IdP-only aggregate out to a file.
+    -->
+    <bean id="idp.serialize" parent="mda.SerializationStage">
+        <property name="serializer" ref="serializer"/>
+        <property name="outputFile">
+            <bean parent="File">
+                <constructor-arg value="${mdq.output}/idps/all.xml"/>
+            </bean>
+        </property>
+    </bean>
+
+    <!--
+        idp.selector
+
+        Entities in the IdP-only aggregate are restricted to identity providers.
+    -->
+    <bean id="idp.selector" parent="mda.XPathItemSelectionStrategy">
+        <constructor-arg value="/md:EntityDescriptor[md:IDPSSODescriptor]"/>
+        <constructor-arg ref="commonNamespaces"/>
+    </bean>
+
+    <!--
+        idp.pipeline
+
+        Generates the IdP-only aggregate. The selector has already taken care
+        of removing non-IdP entities.
+    -->
+    <bean id="idp.pipeline" parent="mda.SimplePipeline">
+        <property name="stages">
+            <list>
+                <!-- Perform common steps in constructing an output aggregate. -->
+                <ref bean="common.output"/>
+
+                <!-- Write the resulting aggregate out to a file. -->
+                <ref bean="idp.serialize"/>
+            </list>
+        </property>
+    </bean>
+
+    <!--
+        Generate all required metadata for the InCommon MDQ service
+        using the localkey configuration:
+
+          * Per-entity metadata
+          * InCommon aggregate
+          * InCommon IdP-only aggregate
+
+        Properties:
+
+        Set sign.uk.keyPassword (note the slightly anomalous "uk" in that
+        identifier) to set the password to access the key resource. If not set,
+        this will be prompted for.
+    -->
+    <bean id="mdq-combined-localkey" parent="mda.SimplePipeline">
+        <property name="stages">
+            <list>
+                <!--
+                    Start with the unsigned production aggregate.
+                    TODO: (MDQ-177) For now, we're actually starting with a signed aggregate;
+                      this pipeline isn't configured to check the signature
+                -->
+                <bean id="production_aggregate" parent="mda.DOMResourceSourceStage">
+                    <property name="parserPool" ref="parserPool"/>
+                    <property name="DOMResource">
+                        <bean parent="FileSystemResource" c:_0="${mdq.input}"/>
+                    </property>
+                </bean>
+
+                <!-- Break down into individual entities. -->
+                <ref bean="disassemble"/>
+
+                <!-- Include a default registrationAuthority for each entity. -->
+                <ref bean="us_incommon_default_regauth"/>
+
+                <!-- Populate identifiers for future actions. -->
+                <ref bean="populateItemIds"/>
+                <ref bean="populateRegistrationAuthorities"/>
+
+                <!-- Discard entities in the InCommon entity blacklist. -->
+                <ref bean="remove_blacklisted_incommon_entities"/>
+
+                <!-- Fork new pipelines to generate special aggregates. -->
+                <bean id="demux" parent="mda.PipelineDemultiplexerStage">
+                    <property name="pipelineAndSelectionStrategies">
+                        <list>
+                            <!-- IdP-only aggregate. -->
+                            <bean class="net.shibboleth.utilities.java.support.collection.Pair">
+                                <constructor-arg ref="idp.pipeline"/>
+                                <constructor-arg ref="idp.selector"/>
+                            </bean>
+                        </list>
+                    </property>
+                    <property name="waitingForPipelines" value="true"/>
+                </bean>
+
+                <!-- Set ID, cacheDuration and validUntil attributes. -->
+                <bean parent="mda.GenerateIdStage"/>
+                <bean parent="mda.SetValidUntilStage" p:validityDuration="P14D"/>
+
+                <!-- Identity transform fixes signing issues. -->
+                <bean parent="mda.XSLTransformationStage"
+                    p:XSLResource="classpath:identity.xsl"/>
+
+                <!-- Sign each item. -->
+                <bean id="perform.signature" parent="mda.XMLSignatureSigningStage">
+                    <property name="privateKey">
+                        <bean parent="PrivateKeyFactoryBean"
+                            p:resource="${sign.keyResource}"
+                            p:privateKeyPassword="${sign.keyPassword}"
+                        />
+                    </property>
+                </bean>
+
+                <!-- Write individual entity documents to files. -->
+                <bean id="write.perentity" parent="mda.MultiOutputSerializationStage">
+                    <property name="serializer" ref="serializer"/>
+                    <property name="outputStrategy">
+                        <bean parent="mda.FilesInDirectoryMultiOutputStrategy" p:nameSuffix=".xml">
+                            <property name="directory">
+                                <bean parent="File" c:_="${mdq.output}"/>
+                            </property>
+                            <property name="nameTransformer">
+                                <bean parent="mda.PathSegmentStringTransformer"/>
+                            </property>
+                        </bean>
+                    </property>
+                </bean>
+
+            </list>
+        </property>
+
+    </bean>
+
+</beans>

From 270cfcfd179611e10cf8a6af12440fba6b35051e Mon Sep 17 00:00:00 2001
From: David Shafer <dshafer@internet2.edu>
Date: Wed, 6 Feb 2019 16:24:19 -0600
Subject: [PATCH 11/19] Create new pipeline to produce all MDQ metadata

---
 build.xml                              |  68 +++----
 mdx/incommon/mdq-all-cloudhsm.xml      |  46 +++++
 mdx/incommon/mdq-all-localkey.xml      |  46 +++++
 mdx/incommon/mdq-all.xml               | 245 +++++++++++++++++++++++++
 mdx/incommon/mdq-combined-cloudhsm.xml | 224 ----------------------
 mdx/incommon/mdq-combined-localkey.xml | 223 ----------------------
 6 files changed, 367 insertions(+), 485 deletions(-)
 create mode 100644 mdx/incommon/mdq-all-cloudhsm.xml
 create mode 100644 mdx/incommon/mdq-all-localkey.xml
 create mode 100644 mdx/incommon/mdq-all.xml
 delete mode 100644 mdx/incommon/mdq-combined-cloudhsm.xml
 delete mode 100644 mdx/incommon/mdq-combined-localkey.xml

diff --git a/build.xml b/build.xml
index bdfb205e..350505bb 100644
--- a/build.xml
+++ b/build.xml
@@ -2665,13 +2665,10 @@
     </target>
 
     <!--
-        inc.mdq-combined.localkey
+        inc.mdq.generate.all.cloudhsm
 
-        Generate all required metadata for the InCommon MDQ service
-        using the localkey configuration:
-
-          * Per-entity metadata
-          * InCommon IdP-only aggregate
+        Generate all metadata required for the InCommon federation MDQ service
+        using the AWS CloudHSM configuration.
 
         Properties:
 
@@ -2679,27 +2676,27 @@
         identifier) to set the password to access the key resource. If not set,
         this will be prompted for.
     -->
-    <target name="inc.mdq.combined.localkey" depends="get.sign.uk.keyPassword">
-      <property name="mda.mdq.input" value="${mda.inc.imported.xml}"/>
-      <property name="mda.sign.keyPassword" value="${sign.uk.keyPassword}"/>
-      <echo>Generating per-entity metadata in ${mda.mdq.output}</echo>
-      <echo>    (IdP-only aggregate in ${mda.mdq.output}/idps.xml)</echo>
-      <echo>    from unsigned aggregate in ${mda.mdq.input}</echo>
-      <delete dir="${mdq.output.dir}" quiet="true"/>
-      <mkdir dir="${mdq.output.dir}"/>
-      <mkdir dir="${mdq.output.dir}/idps"/>
-      <CHANNEL.do channel="incommon" verb="mdq-combined-localkey"/>
-      <echo>Generation complete.</echo>
+    <target name="inc.mdq.generate.all.cloudhsm" depends="get.sign.uk.keyPassword">
+        <property name="mda.classpath.extra" value="inc-mda-cloudhsm"/>
+        <property name="mda.jni.path" value="/opt/cloudhsm/lib"/>
+		<property name="mda.mdq.input" value="${mda.inc.imported.xml}"/>
+        <property name="mda.sign.keyHandle" value="${sign.keyHandle}"/>
+		<property name="mda.sign.keyPassword" value="${sign.uk.keyPassword}"/>
+        <property name="mda.sign.keyUser" value="${sign.keyUser}"/>
+		<echo>Generating MDQ metadata in ${mda.mdq.output}</echo>
+		<echo>    from unsigned aggregate in ${mda.mdq.input}</echo>
+		<delete dir="${mdq.output.dir}" quiet="true"/>
+		<mkdir dir="${mdq.output.dir}"/>
+		<mkdir dir="${mdq.output.dir}/idps"/>
+		<CHANNEL.do channel="incommon" verb="mdq-all-cloudhsm"/>
+		<echo>Generation complete.</echo>
     </target>
 
     <!--
-        inc.mdq.combined.cloudhsm
-
-        Generate all required metadata for the InCommon MDQ service
-        using the cloudhsm configuration:
+        inc.mdq.generate.all.localkey
 
-          * Per-entity metadata
-          * InCommon IdP-only aggregate
+        Generate all metadata required for the InCommon federation MDQ service
+        using the local key configuration.
 
         Properties:
 
@@ -2707,21 +2704,16 @@
         identifier) to set the password to access the key resource. If not set,
         this will be prompted for.
     -->
-    <target name="inc.mdq.combined.cloudhsm" depends="get.sign.uk.keyPassword">
-      <property name="mda.mdq.input" value="${mda.inc.imported.xml}"/>
-      <property name="mda.sign.keyHandle" value="${sign.keyHandle}"/>
-      <property name="mda.sign.keyUser" value="${sign.keyUser}"/>
-      <property name="mda.sign.keyPassword" value="${sign.keyPassword}"/>
-      <property name="mda.classpath.extra" value="inc-mda-cloudhsm"/>
-      <property name="mda.jni.path" value="/opt/cloudhsm/lib"/>
-      <echo>Generating per-entity metadata in ${mda.mdq.output}</echo>
-      <echo>    (IdP-only aggregate in ${mda.mdq.output}/idps.xml)</echo>
-      <echo>    from unsigned aggregate in ${mda.mdq.input}</echo>
-      <delete dir="${mdq.output.dir}" quiet="true"/>
-      <mkdir dir="${mdq.output.dir}"/>
-      <mkdir dir="${mdq.output.dir}/idps"/>
-      <CHANNEL.do channel="incommon" verb="mdq-combined-cloudhsm"/>
-      <echo>Generation complete.</echo>
+	<target name="inc.mdq.generate.all.localkey" depends="get.sign.uk.keyPassword">
+		<property name="mda.mdq.input" value="${mda.inc.imported.xml}"/>
+		<property name="mda.sign.keyPassword" value="${sign.uk.keyPassword}"/>
+		<echo>Generating MDQ metadata in ${mda.mdq.output}</echo>
+		<echo>    from unsigned aggregate in ${mda.mdq.input}</echo>
+		<delete dir="${mdq.output.dir}" quiet="true"/>
+		<mkdir dir="${mdq.output.dir}"/>
+		<mkdir dir="${mdq.output.dir}/idps"/>
+		<CHANNEL.do channel="incommon" verb="mdq-all-localkey"/>
+		<echo>Generation complete.</echo>
     </target>
 
     <!--
diff --git a/mdx/incommon/mdq-all-cloudhsm.xml b/mdx/incommon/mdq-all-cloudhsm.xml
new file mode 100644
index 00000000..e54e711a
--- /dev/null
+++ b/mdx/incommon/mdq-all-cloudhsm.xml
@@ -0,0 +1,46 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<beans xmlns="http://www.springframework.org/schema/beans" default-lazy-init="true"
+    xmlns:c="http://www.springframework.org/schema/c" xmlns:p="http://www.springframework.org/schema/p"
+    xmlns:util="http://www.springframework.org/schema/util" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+    xsi:schemaLocation="
+        http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd
+        http://www.springframework.org/schema/util http://www.springframework.org/schema/util/spring-util.xsd">
+
+    <!-- Import commonly used beans -->
+    <import resource="classpath:common-beans.xml" />
+
+    <!-- Import inc-mda beans (from import.xml) -->
+    <import resource="classpath:uk/org/iay/incommon/mda/beans.xml" />
+
+    <!-- Import channel-specific beans -->
+    <import resource="classpath:incommon/beans.xml" />
+
+    <!-- Import mdq-all beans -->
+    <import resource="classpath:incommon/mdq-all.xml" />
+
+    <!-- This bean MUST be called "conversionService" to work properly -->
+    <bean id="conversionService" class="org.springframework.context.support.ConversionServiceFactoryBean">
+        <property name="converters">
+            <set>
+                <bean class="net.shibboleth.ext.spring.config.DurationToLongConverter" />
+                <bean class="net.shibboleth.ext.spring.config.StringToIPRangeConverter" />
+                <bean class="net.shibboleth.ext.spring.config.BooleanToPredicateConverter" />
+                <bean class="net.shibboleth.ext.spring.config.StringBooleanToPredicateConverter" />
+                <bean class="net.shibboleth.ext.spring.config.StringToResourceConverter" />
+            </set>
+        </property>
+    </bean>
+
+    <bean id="privateKeyFactory" class="uk.org.iay.incommon.mda.cloudhsm.CaviumPrivateKeyFactoryBean"
+        p:username="${sign.keyUser}" p:password="${sign.keyPassword}" p:keyHandle="${sign.keyHandle}" />
+
+    <bean id="mdq-all-cloudhsm" parent="mda.SimplePipeline">
+        <property name="stages">
+            <list>
+                <ref bean="generateAll" />
+            </list>
+        </property>
+    </bean>
+
+</beans>
diff --git a/mdx/incommon/mdq-all-localkey.xml b/mdx/incommon/mdq-all-localkey.xml
new file mode 100644
index 00000000..d329dc75
--- /dev/null
+++ b/mdx/incommon/mdq-all-localkey.xml
@@ -0,0 +1,46 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<beans xmlns="http://www.springframework.org/schema/beans" default-lazy-init="true"
+    xmlns:c="http://www.springframework.org/schema/c" xmlns:p="http://www.springframework.org/schema/p"
+    xmlns:util="http://www.springframework.org/schema/util" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+    xsi:schemaLocation="
+        http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd
+        http://www.springframework.org/schema/util http://www.springframework.org/schema/util/spring-util.xsd">
+
+    <!-- Import commonly used beans -->
+    <import resource="classpath:common-beans.xml" />
+
+    <!-- Import inc-mda beans (from import.xml) -->
+    <import resource="classpath:uk/org/iay/incommon/mda/beans.xml" />
+
+    <!-- Import channel-specific beans -->
+    <import resource="classpath:incommon/beans.xml" />
+
+    <!-- Import mdq-all beans -->
+    <import resource="classpath:incommon/mdq-all.xml" />
+
+    <!-- This bean MUST be called "conversionService" to work properly -->
+    <bean id="conversionService" class="org.springframework.context.support.ConversionServiceFactoryBean">
+        <property name="converters">
+            <set>
+                <bean class="net.shibboleth.ext.spring.config.DurationToLongConverter" />
+                <bean class="net.shibboleth.ext.spring.config.StringToIPRangeConverter" />
+                <bean class="net.shibboleth.ext.spring.config.BooleanToPredicateConverter" />
+                <bean class="net.shibboleth.ext.spring.config.StringBooleanToPredicateConverter" />
+                <bean class="net.shibboleth.ext.spring.config.StringToResourceConverter" />
+            </set>
+        </property>
+    </bean>
+
+    <bean id="privateKeyFactory" parent="PrivateKeyFactoryBean" p:resource="${sign.keyResource}"
+        p:privateKeyPassword="${sign.keyPassword}" />
+
+    <bean id="mdq-all-localkey" parent="mda.SimplePipeline">
+        <property name="stages">
+            <list>
+                <ref bean="generateAll" />
+            </list>
+        </property>
+    </bean>
+
+</beans>
diff --git a/mdx/incommon/mdq-all.xml b/mdx/incommon/mdq-all.xml
new file mode 100644
index 00000000..ca85098a
--- /dev/null
+++ b/mdx/incommon/mdq-all.xml
@@ -0,0 +1,245 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<beans xmlns="http://www.springframework.org/schema/beans" default-lazy-init="true"
+    xmlns:c="http://www.springframework.org/schema/c" xmlns:p="http://www.springframework.org/schema/p"
+    xmlns:if="ant:if" xmlns:unless="ant:unless" xmlns:util="http://www.springframework.org/schema/util"
+    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+    xsi:schemaLocation="
+        http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd
+        http://www.springframework.org/schema/util http://www.springframework.org/schema/util/spring-util.xsd">
+
+    <!-- Import commonly used beans -->
+    <import resource="classpath:common-beans.xml" />
+
+    <!-- Import inc-mda beans (from import.xml) -->
+    <import resource="classpath:uk/org/iay/incommon/mda/beans.xml" />
+
+    <!-- Import channel-specific beans -->
+    <import resource="classpath:incommon/beans.xml" />
+
+    <!-- This bean MUST be called "conversionService" to work properly -->
+    <bean id="conversionService" class="org.springframework.context.support.ConversionServiceFactoryBean">
+        <property name="converters">
+            <set>
+                <bean class="net.shibboleth.ext.spring.config.DurationToLongConverter" />
+                <bean class="net.shibboleth.ext.spring.config.StringToIPRangeConverter" />
+                <bean class="net.shibboleth.ext.spring.config.BooleanToPredicateConverter" />
+                <bean class="net.shibboleth.ext.spring.config.StringBooleanToPredicateConverter" />
+                <bean class="net.shibboleth.ext.spring.config.StringToResourceConverter" />
+            </set>
+        </property>
+    </bean>
+
+    <!-- Predicate for matching everything -->
+    <bean id="matchEverything" class="com.google.common.base.Predicates" factory-method="alwaysTrue" />
+
+    <!-- Generate all required metadata for the InCommon MDQ service -->
+    <bean id="generateAll" parent="mda.CompositeStage">
+        <property name="composedStages">
+            <list>
+
+                <!-- Begin with the published aggregate -->
+                <ref bean="commonSourceStage" />
+
+                <!-- Fork pipelines for creating signed aggregates -->
+                <bean id="demux" parent="mda.PipelineDemultiplexerStage">
+
+                    <property name="pipelineAndSelectionStrategies">
+                        <list>
+
+                            <!-- Create the signed aggregate -->
+                            <bean class="net.shibboleth.utilities.java.support.collection.Pair">
+                                <constructor-arg ref="aggregatePipeline" />
+                                <constructor-arg ref="matchEverything" />
+                            </bean>
+
+                            <!-- Create the signed IdP-only aggregate -->
+                            <bean class="net.shibboleth.utilities.java.support.collection.Pair">
+                                <constructor-arg ref="idpPipeline" />
+                                <constructor-arg ref="idpSelectionStrategy" />
+                            </bean>
+
+                        </list>
+                    </property>
+
+                    <property name="waitingForPipelines" value="true" />
+
+                </bean>
+
+                <!-- Create per-entity metadata -->
+                <ref bean="perEntityStage" />
+
+            </list>
+        </property>
+    </bean>
+
+    <!-- Pipeline that produces the signed aggregate from a collection of entities -->
+    <bean id="aggregatePipeline" parent="mda.SimplePipeline">
+        <property name="stages">
+            <list>
+                <ref bean="aggregateStage" />
+            </list>
+        </property>
+    </bean>
+
+    <!-- Stage that produces the signed aggregate from a collection of entities -->
+    <bean id="aggregateStage" parent="mda.CompositeStage">
+        <property name="composedStages">
+            <list>
+
+                <!-- Perform common steps in constructing an output aggregate -->
+                <ref bean="commonOutputStage" />
+
+                <!-- Write the resulting aggregate to a file -->
+                <ref bean="aggregateSerializationStage" />
+
+            </list>
+        </property>
+    </bean>
+
+    <!-- Stage that writes the aggregate to a file -->
+    <bean id="aggregateSerializationStage" parent="mda.SerializationStage">
+        <property name="serializer" ref="serializer" />
+        <property name="outputFile">
+            <bean parent="File">
+                <constructor-arg value="${mdq.output}/all.xml" />
+            </bean>
+        </property>
+    </bean>
+
+    <!-- Output stage used by all MDQ aggregate pipelines -->
+    <bean id="commonOutputStage" parent="mda.CompositeStage">
+        <property name="composedStages">
+            <list>
+                <!-- Construct a new aggregate from the collection of entities -->
+                <bean id="inc.assemble" parent="mda.EntitiesDescriptorAssemblerStage">
+                    <property name="itemOrderingStrategy">
+                        <bean parent="inc.InCommonEntityOrderingStrategy" c:_-ref="us_incommon_registrar" />
+                    </property>
+                </bean>
+
+                <!-- Apply final tweaks to the aggregate -->
+                <bean id="finalise" parent="incommon_finalise_parent">
+                    <property name="transformParameters">
+                        <map>
+                            <entry key="extraText" value="Contains InCommon and eduGAIN metadata" />
+                            <entry key="publisher" value-ref="us_incommon_registrar" />
+                        </map>
+                    </property>
+                </bean>
+
+                <!-- Normalise the use of namespace prefixes in the resulting XML document. -->
+                <bean id="normalise" parent="mda.XSLTransformationStage"
+                    p:XSLResource="classpath:incommon/ns_norm_import.xsl" />
+
+                <!-- Sign the aggregate -->
+                <ref bean="signItems" />
+
+            </list>
+        </property>
+    </bean>
+
+    <!-- Source stage used by all MDQ pipelines -->
+    <bean id="commonSourceStage" parent="mda.CompositeStage">
+        <property name="composedStages">
+            <list>
+
+                <!-- Start with the signed production aggregate -->
+                <bean id="production_aggregate" parent="mda.DOMResourceSourceStage">
+                    <property name="parserPool" ref="parserPool" />
+                    <property name="DOMResource">
+                        <bean parent="FileSystemResource" c:_0="${mdq.input}" />
+                    </property>
+                </bean>
+
+                <!-- TODO: Check signature on source aggregate -->
+
+                <!-- Break down into individual entities. -->
+                <ref bean="disassemble" />
+
+                <!-- Include a default registrationAuthority for each entity. -->
+                <ref bean="us_incommon_default_regauth" />
+
+                <!-- Populate identifiers for future actions. -->
+                <ref bean="populateItemIds" />
+                <ref bean="populateRegistrationAuthorities" />
+
+                <!-- Discard entities in the InCommon entity blacklist. -->
+                <ref bean="remove_blacklisted_incommon_entities" />
+
+            </list>
+        </property>
+    </bean>
+
+    <!-- Pipeline that produces the IdP-only aggregate from a collection of IdP entities -->
+    <bean id="idpPipeline" parent="mda.SimplePipeline">
+        <property name="stages">
+            <list>
+
+                <!-- Perform common steps in constructing an output aggregate. -->
+                <ref bean="commonOutputStage" />
+
+                <!-- Write the resulting aggregate out to a file. -->
+                <ref bean="idpSerializationStage" />
+            </list>
+        </property>
+    </bean>
+
+    <!-- Selection strategy that selects IdP entities for the IdP-only aggregate -->
+    <bean id="idpSelectionStrategy" parent="mda.XPathItemSelectionStrategy">
+        <constructor-arg value="/md:EntityDescriptor[md:IDPSSODescriptor]" />
+        <constructor-arg ref="commonNamespaces" />
+    </bean>
+
+    <!-- Serialization stage that writes the IdP-only aggregate to a file -->
+    <bean id="idpSerializationStage" parent="mda.SerializationStage">
+        <property name="serializer" ref="serializer" />
+        <property name="outputFile">
+            <bean parent="File">
+                <constructor-arg value="${mdq.output}/idps/all.xml" />
+            </bean>
+        </property>
+    </bean>
+
+    <!-- Stage to produce signed per-entity metadata -->
+    <bean id="perEntityStage" parent="mda.CompositeStage">
+        <property name="composedStages">
+            <list>
+
+                <!-- Set ID, cacheDuration and validUntil attributes. -->
+                <bean parent="mda.GenerateIdStage" />
+                <bean parent="mda.SetValidUntilStage" p:validityDuration="P14D" />
+
+                <!-- Identity transform fixes signing issues. -->
+                <bean parent="mda.XSLTransformationStage" p:XSLResource="classpath:identity.xsl" />
+
+                <!-- Sign each item. -->
+                <ref bean="signItems" />
+
+                <!-- Write individual entity documents to files. -->
+                <bean id="write.perentity" parent="mda.MultiOutputSerializationStage">
+                    <property name="serializer" ref="serializer" />
+                    <property name="outputStrategy">
+                        <bean parent="mda.FilesInDirectoryMultiOutputStrategy" p:nameSuffix=".xml">
+                            <property name="directory">
+                                <bean parent="File" c:_="${mdq.output}" />
+                            </property>
+                            <property name="nameTransformer">
+                                <bean parent="mda.PathSegmentStringTransformer" />
+                            </property>
+                        </bean>
+                    </property>
+                </bean>
+
+            </list>
+        </property>
+    </bean>
+
+    <!-- Signs items using a privateKeyFactory defined elsewhere -->
+    <bean id="signItems" parent="mda.XMLSignatureSigningStage">
+        <property name="privateKey">
+            <ref bean="privateKeyFactory" />
+        </property>
+    </bean>
+
+</beans>
diff --git a/mdx/incommon/mdq-combined-cloudhsm.xml b/mdx/incommon/mdq-combined-cloudhsm.xml
deleted file mode 100644
index 6a54503d..00000000
--- a/mdx/incommon/mdq-combined-cloudhsm.xml
+++ /dev/null
@@ -1,224 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-    Generate and sign per-entity metadata using a local private key resource.
--->
-<beans xmlns="http://www.springframework.org/schema/beans"
-    default-lazy-init="true"
-    xmlns:c="http://www.springframework.org/schema/c"
-    xmlns:p="http://www.springframework.org/schema/p"
-    xmlns:util="http://www.springframework.org/schema/util"
-    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-    xsi:schemaLocation="
-        http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd
-        http://www.springframework.org/schema/util http://www.springframework.org/schema/util/spring-util.xsd">
-
-    <!--
-        Import commonly used beans.
-    -->
-    <import resource="classpath:common-beans.xml"/>
-
-    <!--
-        Import inc-mda beans. (from import.xml)
-    -->
-    <import resource="classpath:uk/org/iay/incommon/mda/beans.xml"/>
-
-    <!--
-        Import channel-specific beans.
-    -->
-    <import resource="classpath:incommon/beans.xml"/>
-
-    <!-- This bean MUST be called "conversionService" to work properly. -->
-    <bean id="conversionService" class="org.springframework.context.support.ConversionServiceFactoryBean">
-        <property name="converters">
-            <set>
-                <bean class="net.shibboleth.ext.spring.config.DurationToLongConverter" />
-                <bean class="net.shibboleth.ext.spring.config.StringToIPRangeConverter" />
-                <bean class="net.shibboleth.ext.spring.config.BooleanToPredicateConverter" />
-                <bean class="net.shibboleth.ext.spring.config.StringBooleanToPredicateConverter" />
-                <bean class="net.shibboleth.ext.spring.config.StringToResourceConverter" />
-            </set>
-        </property>
-    </bean>
-
-    <!--
-        ***********************************************************
-        ***                                                     ***
-        ***   C O M M O N   O U T P U T   P R O C E S S I N G   ***
-        ***                                                     ***
-        ***********************************************************
-    -->
-
-    <bean id="common.output" parent="mda.CompositeStage">
-        <property name="composedStages">
-            <list>
-                <!-- Construct a new aggregate from the collection of entities. -->
-                <bean id="inc.assemble" parent="mda.EntitiesDescriptorAssemblerStage">
-                    <property name="itemOrderingStrategy">
-                        <bean parent="inc.InCommonEntityOrderingStrategy"
-                            c:_-ref="us_incommon_registrar"/>
-                    </property>
-                </bean>
-
-                <!-- Apply final tweaks to the aggregate. -->
-                <bean id="finalise" parent="incommon_finalise_parent">
-                    <property name="transformParameters">
-                        <map>
-                            <entry key="extraText" value="Contains InCommon and eduGAIN metadata"/>
-                            <entry key="publisher" value-ref="us_incommon_registrar"/>
-                        </map>
-                    </property>
-                </bean>
-
-                <!-- Normalise the use of namespace prefixes in the resulting XML document. -->
-                <bean id="normalise" parent="mda.XSLTransformationStage"
-                    p:XSLResource="classpath:incommon/ns_norm_import.xsl"/>
-            </list>
-        </property>
-    </bean>
-
-    <!--
-        *****************************************
-        ***                                   ***
-        ***   I D P - O N L Y   O U T P U T   ***
-        ***                                   ***
-        *****************************************
-    -->
-
-    <!--
-        idp.serialize
-
-        Writes the IdP-only aggregate out to a file.
-    -->
-    <bean id="idp.serialize" parent="mda.SerializationStage">
-        <property name="serializer" ref="serializer"/>
-        <property name="outputFile">
-            <bean parent="File">
-                <constructor-arg value="${mdq.output}/idps/all.xml"/>
-            </bean>
-        </property>
-    </bean>
-
-    <!--
-        idp.selector
-
-        Entities in the IdP-only aggregate are restricted to identity providers.
-    -->
-    <bean id="idp.selector" parent="mda.XPathItemSelectionStrategy">
-        <constructor-arg value="/md:EntityDescriptor[md:IDPSSODescriptor]"/>
-        <constructor-arg ref="commonNamespaces"/>
-    </bean>
-
-    <!--
-        idp.pipeline
-
-        Generates the IdP-only aggregate. The selector has already taken care
-        of removing non-IdP entities.
-    -->
-    <bean id="idp.pipeline" parent="mda.SimplePipeline">
-        <property name="stages">
-            <list>
-                <!-- Perform common steps in constructing an output aggregate. -->
-                <ref bean="common.output"/>
-
-                <!-- Write the resulting aggregate out to a file. -->
-                <ref bean="idp.serialize"/>
-            </list>
-        </property>
-    </bean>
-
-    <!--
-        Generate all required metadata for the InCommon MDQ service
-        using the localkey configuration:
-
-          * Per-entity metadata
-          * InCommon aggregate
-          * InCommon IdP-only aggregate
-
-        Properties:
-
-        Set sign.uk.keyPassword (note the slightly anomalous "uk" in that
-        identifier) to set the password to access the key resource. If not set,
-        this will be prompted for.
-    -->
-    <bean id="mdq-combined-cloudhsm" parent="mda.SimplePipeline">
-        <property name="stages">
-            <list>
-                <!--
-                    Start with the unsigned production aggregate.
-                    TODO: (MDQ-177) For now, we're actually starting with a signed aggregate;
-                      this pipeline isn't configured to check the signature
-                -->
-                <bean id="production_aggregate" parent="mda.DOMResourceSourceStage">
-                    <property name="parserPool" ref="parserPool"/>
-                    <property name="DOMResource">
-                        <bean parent="FileSystemResource" c:_0="${mdq.input}"/>
-                    </property>
-                </bean>
-
-                <!-- Break down into individual entities. -->
-                <ref bean="disassemble"/>
-
-                <!-- Include a default registrationAuthority for each entity. -->
-                <ref bean="us_incommon_default_regauth"/>
-
-                <!-- Populate identifiers for future actions. -->
-                <ref bean="populateItemIds"/>
-                <ref bean="populateRegistrationAuthorities"/>
-
-                <!-- Discard entities in the InCommon entity blacklist. -->
-                <ref bean="remove_blacklisted_incommon_entities"/>
-
-                <!-- Fork new pipelines to generate special aggregates. -->
-                <bean id="demux" parent="mda.PipelineDemultiplexerStage">
-                    <property name="pipelineAndSelectionStrategies">
-                        <list>
-                            <!-- IdP-only aggregate. -->
-                            <bean class="net.shibboleth.utilities.java.support.collection.Pair">
-                                <constructor-arg ref="idp.pipeline"/>
-                                <constructor-arg ref="idp.selector"/>
-                            </bean>
-                        </list>
-                    </property>
-                    <property name="waitingForPipelines" value="true"/>
-                </bean>
-
-                <!-- Set ID, cacheDuration and validUntil attributes. -->
-                <bean parent="mda.GenerateIdStage"/>
-                <bean parent="mda.SetValidUntilStage" p:validityDuration="P14D"/>
-
-                <!-- Identity transform fixes signing issues. -->
-                <bean parent="mda.XSLTransformationStage"
-                    p:XSLResource="classpath:identity.xsl"/>
-
-                <!-- Sign each item. -->
-                <bean id="perform.signature" parent="mda.XMLSignatureSigningStage">
-                    <property name="privateKey">
-                      <bean class="uk.org.iay.incommon.mda.cloudhsm.CaviumPrivateKeyFactoryBean"
-                          p:username="${sign.keyUser}"
-                          p:password="${sign.keyPassword}"
-                          p:keyHandle="${sign.keyHandle}"
-                      />
-                    </property>
-                </bean>
-
-                <!-- Write individual entity documents to files. -->
-                <bean id="write.perentity" parent="mda.MultiOutputSerializationStage">
-                    <property name="serializer" ref="serializer"/>
-                    <property name="outputStrategy">
-                        <bean parent="mda.FilesInDirectoryMultiOutputStrategy" p:nameSuffix=".xml">
-                            <property name="directory">
-                                <bean parent="File" c:_="${mdq.output}"/>
-                            </property>
-                            <property name="nameTransformer">
-                                <bean parent="mda.PathSegmentStringTransformer"/>
-                            </property>
-                        </bean>
-                    </property>
-                </bean>
-
-            </list>
-        </property>
-
-    </bean>
-
-</beans>
diff --git a/mdx/incommon/mdq-combined-localkey.xml b/mdx/incommon/mdq-combined-localkey.xml
deleted file mode 100644
index 5e686953..00000000
--- a/mdx/incommon/mdq-combined-localkey.xml
+++ /dev/null
@@ -1,223 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-    Generate and sign per-entity metadata using a local private key resource.
--->
-<beans xmlns="http://www.springframework.org/schema/beans"
-    default-lazy-init="true"
-    xmlns:c="http://www.springframework.org/schema/c"
-    xmlns:p="http://www.springframework.org/schema/p"
-    xmlns:util="http://www.springframework.org/schema/util"
-    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-    xsi:schemaLocation="
-        http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd
-        http://www.springframework.org/schema/util http://www.springframework.org/schema/util/spring-util.xsd">
-
-    <!--
-        Import commonly used beans.
-    -->
-    <import resource="classpath:common-beans.xml"/>
-
-    <!--
-        Import inc-mda beans. (from import.xml)
-    -->
-    <import resource="classpath:uk/org/iay/incommon/mda/beans.xml"/>
-
-    <!--
-        Import channel-specific beans.
-    -->
-    <import resource="classpath:incommon/beans.xml"/>
-
-    <!-- This bean MUST be called "conversionService" to work properly. -->
-    <bean id="conversionService" class="org.springframework.context.support.ConversionServiceFactoryBean">
-        <property name="converters">
-            <set>
-                <bean class="net.shibboleth.ext.spring.config.DurationToLongConverter" />
-                <bean class="net.shibboleth.ext.spring.config.StringToIPRangeConverter" />
-                <bean class="net.shibboleth.ext.spring.config.BooleanToPredicateConverter" />
-                <bean class="net.shibboleth.ext.spring.config.StringBooleanToPredicateConverter" />
-                <bean class="net.shibboleth.ext.spring.config.StringToResourceConverter" />
-            </set>
-        </property>
-    </bean>
-
-    <!--
-        ***********************************************************
-        ***                                                     ***
-        ***   C O M M O N   O U T P U T   P R O C E S S I N G   ***
-        ***                                                     ***
-        ***********************************************************
-    -->
-
-    <bean id="common.output" parent="mda.CompositeStage">
-        <property name="composedStages">
-            <list>
-                <!-- Construct a new aggregate from the collection of entities. -->
-                <bean id="inc.assemble" parent="mda.EntitiesDescriptorAssemblerStage">
-                    <property name="itemOrderingStrategy">
-                        <bean parent="inc.InCommonEntityOrderingStrategy"
-                            c:_-ref="us_incommon_registrar"/>
-                    </property>
-                </bean>
-
-                <!-- Apply final tweaks to the aggregate. -->
-                <bean id="finalise" parent="incommon_finalise_parent">
-                    <property name="transformParameters">
-                        <map>
-                            <entry key="extraText" value="Contains InCommon and eduGAIN metadata"/>
-                            <entry key="publisher" value-ref="us_incommon_registrar"/>
-                        </map>
-                    </property>
-                </bean>
-
-                <!-- Normalise the use of namespace prefixes in the resulting XML document. -->
-                <bean id="normalise" parent="mda.XSLTransformationStage"
-                    p:XSLResource="classpath:incommon/ns_norm_import.xsl"/>
-            </list>
-        </property>
-    </bean>
-
-    <!--
-        *****************************************
-        ***                                   ***
-        ***   I D P - O N L Y   O U T P U T   ***
-        ***                                   ***
-        *****************************************
-    -->
-
-    <!--
-        idp.serialize
-
-        Writes the IdP-only aggregate out to a file.
-    -->
-    <bean id="idp.serialize" parent="mda.SerializationStage">
-        <property name="serializer" ref="serializer"/>
-        <property name="outputFile">
-            <bean parent="File">
-                <constructor-arg value="${mdq.output}/idps/all.xml"/>
-            </bean>
-        </property>
-    </bean>
-
-    <!--
-        idp.selector
-
-        Entities in the IdP-only aggregate are restricted to identity providers.
-    -->
-    <bean id="idp.selector" parent="mda.XPathItemSelectionStrategy">
-        <constructor-arg value="/md:EntityDescriptor[md:IDPSSODescriptor]"/>
-        <constructor-arg ref="commonNamespaces"/>
-    </bean>
-
-    <!--
-        idp.pipeline
-
-        Generates the IdP-only aggregate. The selector has already taken care
-        of removing non-IdP entities.
-    -->
-    <bean id="idp.pipeline" parent="mda.SimplePipeline">
-        <property name="stages">
-            <list>
-                <!-- Perform common steps in constructing an output aggregate. -->
-                <ref bean="common.output"/>
-
-                <!-- Write the resulting aggregate out to a file. -->
-                <ref bean="idp.serialize"/>
-            </list>
-        </property>
-    </bean>
-
-    <!--
-        Generate all required metadata for the InCommon MDQ service
-        using the localkey configuration:
-
-          * Per-entity metadata
-          * InCommon aggregate
-          * InCommon IdP-only aggregate
-
-        Properties:
-
-        Set sign.uk.keyPassword (note the slightly anomalous "uk" in that
-        identifier) to set the password to access the key resource. If not set,
-        this will be prompted for.
-    -->
-    <bean id="mdq-combined-localkey" parent="mda.SimplePipeline">
-        <property name="stages">
-            <list>
-                <!--
-                    Start with the unsigned production aggregate.
-                    TODO: (MDQ-177) For now, we're actually starting with a signed aggregate;
-                      this pipeline isn't configured to check the signature
-                -->
-                <bean id="production_aggregate" parent="mda.DOMResourceSourceStage">
-                    <property name="parserPool" ref="parserPool"/>
-                    <property name="DOMResource">
-                        <bean parent="FileSystemResource" c:_0="${mdq.input}"/>
-                    </property>
-                </bean>
-
-                <!-- Break down into individual entities. -->
-                <ref bean="disassemble"/>
-
-                <!-- Include a default registrationAuthority for each entity. -->
-                <ref bean="us_incommon_default_regauth"/>
-
-                <!-- Populate identifiers for future actions. -->
-                <ref bean="populateItemIds"/>
-                <ref bean="populateRegistrationAuthorities"/>
-
-                <!-- Discard entities in the InCommon entity blacklist. -->
-                <ref bean="remove_blacklisted_incommon_entities"/>
-
-                <!-- Fork new pipelines to generate special aggregates. -->
-                <bean id="demux" parent="mda.PipelineDemultiplexerStage">
-                    <property name="pipelineAndSelectionStrategies">
-                        <list>
-                            <!-- IdP-only aggregate. -->
-                            <bean class="net.shibboleth.utilities.java.support.collection.Pair">
-                                <constructor-arg ref="idp.pipeline"/>
-                                <constructor-arg ref="idp.selector"/>
-                            </bean>
-                        </list>
-                    </property>
-                    <property name="waitingForPipelines" value="true"/>
-                </bean>
-
-                <!-- Set ID, cacheDuration and validUntil attributes. -->
-                <bean parent="mda.GenerateIdStage"/>
-                <bean parent="mda.SetValidUntilStage" p:validityDuration="P14D"/>
-
-                <!-- Identity transform fixes signing issues. -->
-                <bean parent="mda.XSLTransformationStage"
-                    p:XSLResource="classpath:identity.xsl"/>
-
-                <!-- Sign each item. -->
-                <bean id="perform.signature" parent="mda.XMLSignatureSigningStage">
-                    <property name="privateKey">
-                        <bean parent="PrivateKeyFactoryBean"
-                            p:resource="${sign.keyResource}"
-                            p:privateKeyPassword="${sign.keyPassword}"
-                        />
-                    </property>
-                </bean>
-
-                <!-- Write individual entity documents to files. -->
-                <bean id="write.perentity" parent="mda.MultiOutputSerializationStage">
-                    <property name="serializer" ref="serializer"/>
-                    <property name="outputStrategy">
-                        <bean parent="mda.FilesInDirectoryMultiOutputStrategy" p:nameSuffix=".xml">
-                            <property name="directory">
-                                <bean parent="File" c:_="${mdq.output}"/>
-                            </property>
-                            <property name="nameTransformer">
-                                <bean parent="mda.PathSegmentStringTransformer"/>
-                            </property>
-                        </bean>
-                    </property>
-                </bean>
-
-            </list>
-        </property>
-
-    </bean>
-
-</beans>

From 7cad1df2630487d4be4ccb7f709dd4ccc0888deb Mon Sep 17 00:00:00 2001
From: David Shafer <dshafer@internet2.edu>
Date: Fri, 8 Feb 2019 15:09:26 -0600
Subject: [PATCH 12/19] Comment conversionService

---
 mdx/incommon/mdq-all-cloudhsm.xml | 24 ++++++++++++------------
 mdx/incommon/mdq-all-localkey.xml | 24 ++++++++++++------------
 2 files changed, 24 insertions(+), 24 deletions(-)

diff --git a/mdx/incommon/mdq-all-cloudhsm.xml b/mdx/incommon/mdq-all-cloudhsm.xml
index e54e711a..2ddcb105 100644
--- a/mdx/incommon/mdq-all-cloudhsm.xml
+++ b/mdx/incommon/mdq-all-cloudhsm.xml
@@ -19,18 +19,18 @@
     <!-- Import mdq-all beans -->
     <import resource="classpath:incommon/mdq-all.xml" />
 
-    <!-- This bean MUST be called "conversionService" to work properly -->
-    <bean id="conversionService" class="org.springframework.context.support.ConversionServiceFactoryBean">
-        <property name="converters">
-            <set>
-                <bean class="net.shibboleth.ext.spring.config.DurationToLongConverter" />
-                <bean class="net.shibboleth.ext.spring.config.StringToIPRangeConverter" />
-                <bean class="net.shibboleth.ext.spring.config.BooleanToPredicateConverter" />
-                <bean class="net.shibboleth.ext.spring.config.StringBooleanToPredicateConverter" />
-                <bean class="net.shibboleth.ext.spring.config.StringToResourceConverter" />
-            </set>
-        </property>
-    </bean>
+<!--     This bean MUST be called "conversionService" to work properly -->
+<!--     <bean id="conversionService" class="org.springframework.context.support.ConversionServiceFactoryBean"> -->
+<!--         <property name="converters"> -->
+<!--             <set> -->
+<!--                 <bean class="net.shibboleth.ext.spring.config.DurationToLongConverter" /> -->
+<!--                 <bean class="net.shibboleth.ext.spring.config.StringToIPRangeConverter" /> -->
+<!--                 <bean class="net.shibboleth.ext.spring.config.BooleanToPredicateConverter" /> -->
+<!--                 <bean class="net.shibboleth.ext.spring.config.StringBooleanToPredicateConverter" /> -->
+<!--                 <bean class="net.shibboleth.ext.spring.config.StringToResourceConverter" /> -->
+<!--             </set> -->
+<!--         </property> -->
+<!--     </bean> -->
 
     <bean id="privateKeyFactory" class="uk.org.iay.incommon.mda.cloudhsm.CaviumPrivateKeyFactoryBean"
         p:username="${sign.keyUser}" p:password="${sign.keyPassword}" p:keyHandle="${sign.keyHandle}" />
diff --git a/mdx/incommon/mdq-all-localkey.xml b/mdx/incommon/mdq-all-localkey.xml
index d329dc75..f9a87501 100644
--- a/mdx/incommon/mdq-all-localkey.xml
+++ b/mdx/incommon/mdq-all-localkey.xml
@@ -19,18 +19,18 @@
     <!-- Import mdq-all beans -->
     <import resource="classpath:incommon/mdq-all.xml" />
 
-    <!-- This bean MUST be called "conversionService" to work properly -->
-    <bean id="conversionService" class="org.springframework.context.support.ConversionServiceFactoryBean">
-        <property name="converters">
-            <set>
-                <bean class="net.shibboleth.ext.spring.config.DurationToLongConverter" />
-                <bean class="net.shibboleth.ext.spring.config.StringToIPRangeConverter" />
-                <bean class="net.shibboleth.ext.spring.config.BooleanToPredicateConverter" />
-                <bean class="net.shibboleth.ext.spring.config.StringBooleanToPredicateConverter" />
-                <bean class="net.shibboleth.ext.spring.config.StringToResourceConverter" />
-            </set>
-        </property>
-    </bean>
+<!--     This bean MUST be called "conversionService" to work properly -->
+<!--     <bean id="conversionService" class="org.springframework.context.support.ConversionServiceFactoryBean"> -->
+<!--         <property name="converters"> -->
+<!--             <set> -->
+<!--                 <bean class="net.shibboleth.ext.spring.config.DurationToLongConverter" /> -->
+<!--                 <bean class="net.shibboleth.ext.spring.config.StringToIPRangeConverter" /> -->
+<!--                 <bean class="net.shibboleth.ext.spring.config.BooleanToPredicateConverter" /> -->
+<!--                 <bean class="net.shibboleth.ext.spring.config.StringBooleanToPredicateConverter" /> -->
+<!--                 <bean class="net.shibboleth.ext.spring.config.StringToResourceConverter" /> -->
+<!--             </set> -->
+<!--         </property> -->
+<!--     </bean> -->
 
     <bean id="privateKeyFactory" parent="PrivateKeyFactoryBean" p:resource="${sign.keyResource}"
         p:privateKeyPassword="${sign.keyPassword}" />

From d3507b66dcae83ed6b34ad7a9a4deda6e129b5e5 Mon Sep 17 00:00:00 2001
From: David Shafer <dshafer@internet2.edu>
Date: Fri, 8 Feb 2019 15:43:18 -0600
Subject: [PATCH 13/19] Finish rule updates

---
 build.xml                         |  2 +-
 mdx/incommon/mdq-all-cloudhsm.xml | 13 -------------
 mdx/incommon/mdq-all-localkey.xml | 13 -------------
 3 files changed, 1 insertion(+), 27 deletions(-)

diff --git a/build.xml b/build.xml
index 350505bb..efb0ef13 100644
--- a/build.xml
+++ b/build.xml
@@ -2694,7 +2694,7 @@
 
     <!--
         inc.mdq.generate.all.localkey
-
+    
         Generate all metadata required for the InCommon federation MDQ service
         using the local key configuration.
 
diff --git a/mdx/incommon/mdq-all-cloudhsm.xml b/mdx/incommon/mdq-all-cloudhsm.xml
index 2ddcb105..35a917d7 100644
--- a/mdx/incommon/mdq-all-cloudhsm.xml
+++ b/mdx/incommon/mdq-all-cloudhsm.xml
@@ -19,19 +19,6 @@
     <!-- Import mdq-all beans -->
     <import resource="classpath:incommon/mdq-all.xml" />
 
-<!--     This bean MUST be called "conversionService" to work properly -->
-<!--     <bean id="conversionService" class="org.springframework.context.support.ConversionServiceFactoryBean"> -->
-<!--         <property name="converters"> -->
-<!--             <set> -->
-<!--                 <bean class="net.shibboleth.ext.spring.config.DurationToLongConverter" /> -->
-<!--                 <bean class="net.shibboleth.ext.spring.config.StringToIPRangeConverter" /> -->
-<!--                 <bean class="net.shibboleth.ext.spring.config.BooleanToPredicateConverter" /> -->
-<!--                 <bean class="net.shibboleth.ext.spring.config.StringBooleanToPredicateConverter" /> -->
-<!--                 <bean class="net.shibboleth.ext.spring.config.StringToResourceConverter" /> -->
-<!--             </set> -->
-<!--         </property> -->
-<!--     </bean> -->
-
     <bean id="privateKeyFactory" class="uk.org.iay.incommon.mda.cloudhsm.CaviumPrivateKeyFactoryBean"
         p:username="${sign.keyUser}" p:password="${sign.keyPassword}" p:keyHandle="${sign.keyHandle}" />
 
diff --git a/mdx/incommon/mdq-all-localkey.xml b/mdx/incommon/mdq-all-localkey.xml
index f9a87501..01d1b22c 100644
--- a/mdx/incommon/mdq-all-localkey.xml
+++ b/mdx/incommon/mdq-all-localkey.xml
@@ -19,19 +19,6 @@
     <!-- Import mdq-all beans -->
     <import resource="classpath:incommon/mdq-all.xml" />
 
-<!--     This bean MUST be called "conversionService" to work properly -->
-<!--     <bean id="conversionService" class="org.springframework.context.support.ConversionServiceFactoryBean"> -->
-<!--         <property name="converters"> -->
-<!--             <set> -->
-<!--                 <bean class="net.shibboleth.ext.spring.config.DurationToLongConverter" /> -->
-<!--                 <bean class="net.shibboleth.ext.spring.config.StringToIPRangeConverter" /> -->
-<!--                 <bean class="net.shibboleth.ext.spring.config.BooleanToPredicateConverter" /> -->
-<!--                 <bean class="net.shibboleth.ext.spring.config.StringBooleanToPredicateConverter" /> -->
-<!--                 <bean class="net.shibboleth.ext.spring.config.StringToResourceConverter" /> -->
-<!--             </set> -->
-<!--         </property> -->
-<!--     </bean> -->
-
     <bean id="privateKeyFactory" parent="PrivateKeyFactoryBean" p:resource="${sign.keyResource}"
         p:privateKeyPassword="${sign.keyPassword}" />
 

From 3eb2900f6e0b6eb16fec5e3e3bc97d0629aff931 Mon Sep 17 00:00:00 2001
From: David Shafer <dshafer@internet2.edu>
Date: Fri, 8 Feb 2019 16:20:43 -0600
Subject: [PATCH 14/19] Add comments

---
 mdx/incommon/mdq-all-cloudhsm.xml | 14 ++++----------
 mdx/incommon/mdq-all-localkey.xml | 14 ++++----------
 2 files changed, 8 insertions(+), 20 deletions(-)

diff --git a/mdx/incommon/mdq-all-cloudhsm.xml b/mdx/incommon/mdq-all-cloudhsm.xml
index 35a917d7..db3e85db 100644
--- a/mdx/incommon/mdq-all-cloudhsm.xml
+++ b/mdx/incommon/mdq-all-cloudhsm.xml
@@ -1,5 +1,6 @@
 <?xml version="1.0" encoding="UTF-8"?>
 
+<!-- Generate and sign (using AWS CloudHSM) all metadata required by the InCommon MDQ service -->
 <beans xmlns="http://www.springframework.org/schema/beans" default-lazy-init="true"
     xmlns:c="http://www.springframework.org/schema/c" xmlns:p="http://www.springframework.org/schema/p"
     xmlns:util="http://www.springframework.org/schema/util" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
@@ -7,21 +8,14 @@
         http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd
         http://www.springframework.org/schema/util http://www.springframework.org/schema/util/spring-util.xsd">
 
-    <!-- Import commonly used beans -->
-    <import resource="classpath:common-beans.xml" />
-
-    <!-- Import inc-mda beans (from import.xml) -->
-    <import resource="classpath:uk/org/iay/incommon/mda/beans.xml" />
-
-    <!-- Import channel-specific beans -->
-    <import resource="classpath:incommon/beans.xml" />
-
-    <!-- Import mdq-all beans -->
+    <!-- Import generateAll and supporting beans -->
     <import resource="classpath:incommon/mdq-all.xml" />
 
+    <!-- Define a private key factory (based on AWS CloudHSM) to be used by the generateAll stage -->
     <bean id="privateKeyFactory" class="uk.org.iay.incommon.mda.cloudhsm.CaviumPrivateKeyFactoryBean"
         p:username="${sign.keyUser}" p:password="${sign.keyPassword}" p:keyHandle="${sign.keyHandle}" />
 
+    <!-- Generate all signed metadata required by the InCommon MDQ service  -->
     <bean id="mdq-all-cloudhsm" parent="mda.SimplePipeline">
         <property name="stages">
             <list>
diff --git a/mdx/incommon/mdq-all-localkey.xml b/mdx/incommon/mdq-all-localkey.xml
index 01d1b22c..c2617cb6 100644
--- a/mdx/incommon/mdq-all-localkey.xml
+++ b/mdx/incommon/mdq-all-localkey.xml
@@ -1,5 +1,6 @@
 <?xml version="1.0" encoding="UTF-8"?>
 
+<!-- Generate and sign (using a local key) all metadata required by the InCommon MDQ service -->
 <beans xmlns="http://www.springframework.org/schema/beans" default-lazy-init="true"
     xmlns:c="http://www.springframework.org/schema/c" xmlns:p="http://www.springframework.org/schema/p"
     xmlns:util="http://www.springframework.org/schema/util" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
@@ -7,21 +8,14 @@
         http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd
         http://www.springframework.org/schema/util http://www.springframework.org/schema/util/spring-util.xsd">
 
-    <!-- Import commonly used beans -->
-    <import resource="classpath:common-beans.xml" />
-
-    <!-- Import inc-mda beans (from import.xml) -->
-    <import resource="classpath:uk/org/iay/incommon/mda/beans.xml" />
-
-    <!-- Import channel-specific beans -->
-    <import resource="classpath:incommon/beans.xml" />
-
-    <!-- Import mdq-all beans -->
+    <!-- Import generateAll and supporting beans -->
     <import resource="classpath:incommon/mdq-all.xml" />
 
+    <!-- Define a private key factory (based on a local key) to be used by the generateAll stage -->
     <bean id="privateKeyFactory" parent="PrivateKeyFactoryBean" p:resource="${sign.keyResource}"
         p:privateKeyPassword="${sign.keyPassword}" />
 
+    <!-- Generate all signed metadata required by the InCommon MDQ service  -->
     <bean id="mdq-all-localkey" parent="mda.SimplePipeline">
         <property name="stages">
             <list>

From f8e63ef39eeb0b47c633b0fd4331ae4b8ec1ba44 Mon Sep 17 00:00:00 2001
From: David Shafer <dshafer@internet2.edu>
Date: Tue, 12 Feb 2019 12:01:08 -0600
Subject: [PATCH 15/19] Update properties

---
 build.xml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/build.xml b/build.xml
index efb0ef13..515ea496 100644
--- a/build.xml
+++ b/build.xml
@@ -2681,7 +2681,7 @@
         <property name="mda.jni.path" value="/opt/cloudhsm/lib"/>
 		<property name="mda.mdq.input" value="${mda.inc.imported.xml}"/>
         <property name="mda.sign.keyHandle" value="${sign.keyHandle}"/>
-		<property name="mda.sign.keyPassword" value="${sign.uk.keyPassword}"/>
+		<property name="mda.sign.keyPassword" value="${sign.keyPassword}"/>
         <property name="mda.sign.keyUser" value="${sign.keyUser}"/>
 		<echo>Generating MDQ metadata in ${mda.mdq.output}</echo>
 		<echo>    from unsigned aggregate in ${mda.mdq.input}</echo>
@@ -2706,7 +2706,7 @@
     -->
 	<target name="inc.mdq.generate.all.localkey" depends="get.sign.uk.keyPassword">
 		<property name="mda.mdq.input" value="${mda.inc.imported.xml}"/>
-		<property name="mda.sign.keyPassword" value="${sign.uk.keyPassword}"/>
+		<property name="mda.sign.keyPassword" value="${sign.keyPassword}"/>
 		<echo>Generating MDQ metadata in ${mda.mdq.output}</echo>
 		<echo>    from unsigned aggregate in ${mda.mdq.input}</echo>
 		<delete dir="${mdq.output.dir}" quiet="true"/>

From 55e7fc7d5964a3b9223719d3323d7f63a7fa04e2 Mon Sep 17 00:00:00 2001
From: David Shafer <dshafer@internet2.edu>
Date: Tue, 12 Mar 2019 18:39:16 -0500
Subject: [PATCH 16/19] Remove spaces from empty line

---
 build.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build.xml b/build.xml
index 515ea496..af581e18 100644
--- a/build.xml
+++ b/build.xml
@@ -2694,7 +2694,7 @@
 
     <!--
         inc.mdq.generate.all.localkey
-    
+
         Generate all metadata required for the InCommon federation MDQ service
         using the local key configuration.
 

From b5cef645096405175435e4e8e16d47a704bec04a Mon Sep 17 00:00:00 2001
From: David Shafer <dshafer@internet2.edu>
Date: Tue, 12 Mar 2019 18:55:50 -0500
Subject: [PATCH 17/19] Change tabs to spaces

---
 build.xml | 38 +++++++++++++++++++-------------------
 1 file changed, 19 insertions(+), 19 deletions(-)

diff --git a/build.xml b/build.xml
index af581e18..558a7171 100644
--- a/build.xml
+++ b/build.xml
@@ -2679,17 +2679,17 @@
     <target name="inc.mdq.generate.all.cloudhsm" depends="get.sign.uk.keyPassword">
         <property name="mda.classpath.extra" value="inc-mda-cloudhsm"/>
         <property name="mda.jni.path" value="/opt/cloudhsm/lib"/>
-		<property name="mda.mdq.input" value="${mda.inc.imported.xml}"/>
+        <property name="mda.mdq.input" value="${mda.inc.imported.xml}"/>
         <property name="mda.sign.keyHandle" value="${sign.keyHandle}"/>
-		<property name="mda.sign.keyPassword" value="${sign.keyPassword}"/>
+        <property name="mda.sign.keyPassword" value="${sign.keyPassword}"/>
         <property name="mda.sign.keyUser" value="${sign.keyUser}"/>
-		<echo>Generating MDQ metadata in ${mda.mdq.output}</echo>
-		<echo>    from unsigned aggregate in ${mda.mdq.input}</echo>
-		<delete dir="${mdq.output.dir}" quiet="true"/>
-		<mkdir dir="${mdq.output.dir}"/>
-		<mkdir dir="${mdq.output.dir}/idps"/>
-		<CHANNEL.do channel="incommon" verb="mdq-all-cloudhsm"/>
-		<echo>Generation complete.</echo>
+        <echo>Generating MDQ metadata in ${mda.mdq.output}</echo>
+        <echo>    from unsigned aggregate in ${mda.mdq.input}</echo>
+        <delete dir="${mdq.output.dir}" quiet="true"/>
+        <mkdir dir="${mdq.output.dir}"/>
+        <mkdir dir="${mdq.output.dir}/idps"/>
+        <CHANNEL.do channel="incommon" verb="mdq-all-cloudhsm"/>
+        <echo>Generation complete.</echo>
     </target>
 
     <!--
@@ -2704,16 +2704,16 @@
         identifier) to set the password to access the key resource. If not set,
         this will be prompted for.
     -->
-	<target name="inc.mdq.generate.all.localkey" depends="get.sign.uk.keyPassword">
-		<property name="mda.mdq.input" value="${mda.inc.imported.xml}"/>
-		<property name="mda.sign.keyPassword" value="${sign.keyPassword}"/>
-		<echo>Generating MDQ metadata in ${mda.mdq.output}</echo>
-		<echo>    from unsigned aggregate in ${mda.mdq.input}</echo>
-		<delete dir="${mdq.output.dir}" quiet="true"/>
-		<mkdir dir="${mdq.output.dir}"/>
-		<mkdir dir="${mdq.output.dir}/idps"/>
-		<CHANNEL.do channel="incommon" verb="mdq-all-localkey"/>
-		<echo>Generation complete.</echo>
+    <target name="inc.mdq.generate.all.localkey" depends="get.sign.uk.keyPassword">
+        <property name="mda.mdq.input" value="${mda.inc.imported.xml}"/>
+        <property name="mda.sign.keyPassword" value="${sign.keyPassword}"/>
+        <echo>Generating MDQ metadata in ${mda.mdq.output}</echo>
+        <echo>    from unsigned aggregate in ${mda.mdq.input}</echo>
+        <delete dir="${mdq.output.dir}" quiet="true"/>
+        <mkdir dir="${mdq.output.dir}"/>
+        <mkdir dir="${mdq.output.dir}/idps"/>
+        <CHANNEL.do channel="incommon" verb="mdq-all-localkey"/>
+        <echo>Generation complete.</echo>
     </target>
 
     <!--

From 4f92fecb91cbae2a9530464ca9321830a51607db Mon Sep 17 00:00:00 2001
From: David Shafer <dshafer@internet2.edu>
Date: Tue, 12 Mar 2019 19:02:43 -0500
Subject: [PATCH 18/19] Reorganize targets

---
 build.xml | 72 ++++++++++++++++++++++++++++---------------------------
 1 file changed, 37 insertions(+), 35 deletions(-)

diff --git a/build.xml b/build.xml
index 558a7171..51c577c9 100644
--- a/build.xml
+++ b/build.xml
@@ -2665,10 +2665,10 @@
     </target>
 
     <!--
-        inc.mdq.generate.all.cloudhsm
+        inc.mdq.generate.localkey
 
-        Generate all metadata required for the InCommon federation MDQ service
-        using the AWS CloudHSM configuration.
+        Generate per-entity metadata for the InCommon federation
+        using the localkey configuration.
 
         Properties:
 
@@ -2676,22 +2676,18 @@
         identifier) to set the password to access the key resource. If not set,
         this will be prompted for.
     -->
-    <target name="inc.mdq.generate.all.cloudhsm" depends="get.sign.uk.keyPassword">
-        <property name="mda.classpath.extra" value="inc-mda-cloudhsm"/>
-        <property name="mda.jni.path" value="/opt/cloudhsm/lib"/>
+    <target name="inc.mdq.generate.localkey" depends="get.sign.uk.keyPassword">
         <property name="mda.mdq.input" value="${mda.inc.imported.xml}"/>
-        <property name="mda.sign.keyHandle" value="${sign.keyHandle}"/>
-        <property name="mda.sign.keyPassword" value="${sign.keyPassword}"/>
-        <property name="mda.sign.keyUser" value="${sign.keyUser}"/>
-        <echo>Generating MDQ metadata in ${mda.mdq.output}</echo>
+        <property name="mda.sign.keyPassword" value="${sign.uk.keyPassword}"/>
+        <echo>Generating per-entity metadata in ${mda.mdq.output}</echo>
         <echo>    from unsigned aggregate in ${mda.mdq.input}</echo>
         <delete dir="${mdq.output.dir}" quiet="true"/>
         <mkdir dir="${mdq.output.dir}"/>
-        <mkdir dir="${mdq.output.dir}/idps"/>
-        <CHANNEL.do channel="incommon" verb="mdq-all-cloudhsm"/>
+        <CHANNEL.do channel="incommon" verb="mdq-multisign-localkey"/>
         <echo>Generation complete.</echo>
     </target>
 
+
     <!--
         inc.mdq.generate.all.localkey
 
@@ -2716,29 +2712,7 @@
         <echo>Generation complete.</echo>
     </target>
 
-    <!--
-        inc.mdq.generate.localkey
-
-        Generate per-entity metadata for the InCommon federation
-        using the localkey configuration.
-
-        Properties:
-
-        Set sign.uk.keyPassword (note the slightly anomalous "uk" in that
-        identifier) to set the password to access the key resource. If not set,
-        this will be prompted for.
-    -->
-    <target name="inc.mdq.generate.localkey" depends="get.sign.uk.keyPassword">
-        <property name="mda.mdq.input" value="${mda.inc.imported.xml}"/>
-        <property name="mda.sign.keyPassword" value="${sign.uk.keyPassword}"/>
-        <echo>Generating per-entity metadata in ${mda.mdq.output}</echo>
-        <echo>    from unsigned aggregate in ${mda.mdq.input}</echo>
-        <delete dir="${mdq.output.dir}" quiet="true"/>
-        <mkdir dir="${mdq.output.dir}"/>
-        <CHANNEL.do channel="incommon" verb="mdq-multisign-localkey"/>
-        <echo>Generation complete.</echo>
-    </target>
-
+    
     <!--
         inc.generate.mdq
 
@@ -2775,6 +2749,34 @@
         <CHANNEL.do channel="incommon" verb="mdq-multisign-cloudhsm"/>
         <echo>Generation complete.</echo>
     </target>
+    
+    <!--
+        inc.mdq.generate.all.cloudhsm
+
+        Generate all metadata required for the InCommon federation MDQ service
+        using the AWS CloudHSM configuration.
+
+        Properties:
+
+        Set sign.uk.keyPassword (note the slightly anomalous "uk" in that
+        identifier) to set the password to access the key resource. If not set,
+        this will be prompted for.
+    -->
+    <target name="inc.mdq.generate.all.cloudhsm" depends="get.sign.uk.keyPassword">
+        <property name="mda.classpath.extra" value="inc-mda-cloudhsm"/>
+        <property name="mda.jni.path" value="/opt/cloudhsm/lib"/>
+        <property name="mda.mdq.input" value="${mda.inc.imported.xml}"/>
+        <property name="mda.sign.keyHandle" value="${sign.keyHandle}"/>
+        <property name="mda.sign.keyPassword" value="${sign.keyPassword}"/>
+        <property name="mda.sign.keyUser" value="${sign.keyUser}"/>
+        <echo>Generating MDQ metadata in ${mda.mdq.output}</echo>
+        <echo>    from unsigned aggregate in ${mda.mdq.input}</echo>
+        <delete dir="${mdq.output.dir}" quiet="true"/>
+        <mkdir dir="${mdq.output.dir}"/>
+        <mkdir dir="${mdq.output.dir}/idps"/>
+        <CHANNEL.do channel="incommon" verb="mdq-all-cloudhsm"/>
+        <echo>Generation complete.</echo>
+    </target>
 
     <!--
         inc.edugain.report

From c8c0170fadf79460be854014fda9742a4b202bc6 Mon Sep 17 00:00:00 2001
From: David Shafer <dshafer@internet2.edu>
Date: Wed, 13 Mar 2019 00:09:24 -0500
Subject: [PATCH 19/19] Fix white space, comments, dependencies

---
 build.xml | 11 ++++-------
 1 file changed, 4 insertions(+), 7 deletions(-)

diff --git a/build.xml b/build.xml
index 51c577c9..c7d0170f 100644
--- a/build.xml
+++ b/build.xml
@@ -2687,7 +2687,6 @@
         <echo>Generation complete.</echo>
     </target>
 
-
     <!--
         inc.mdq.generate.all.localkey
 
@@ -2712,7 +2711,6 @@
         <echo>Generation complete.</echo>
     </target>
 
-    
     <!--
         inc.generate.mdq
 
@@ -2749,7 +2747,7 @@
         <CHANNEL.do channel="incommon" verb="mdq-multisign-cloudhsm"/>
         <echo>Generation complete.</echo>
     </target>
-    
+
     <!--
         inc.mdq.generate.all.cloudhsm
 
@@ -2758,11 +2756,10 @@
 
         Properties:
 
-        Set sign.uk.keyPassword (note the slightly anomalous "uk" in that
-        identifier) to set the password to access the key resource. If not set,
-        this will be prompted for.
+        Set sign.keyUser and sign.keyPassword with credentials for the HSM user
+        accessing the key.
     -->
-    <target name="inc.mdq.generate.all.cloudhsm" depends="get.sign.uk.keyPassword">
+    <target name="inc.mdq.generate.all.cloudhsm">
         <property name="mda.classpath.extra" value="inc-mda-cloudhsm"/>
         <property name="mda.jni.path" value="/opt/cloudhsm/lib"/>
         <property name="mda.mdq.input" value="${mda.inc.imported.xml}"/>