From 6ad0a86e5951f0fd97db5e20eaa98a738c5102b2 Mon Sep 17 00:00:00 2001
From: David Shafer <dshafer@internet2.edu>
Date: Mon, 28 Jan 2019 16:11:50 -0600
Subject: [PATCH] Create the IdP-only aggregate (#1)

* Create new pipeline for MDQ and IdP-only aggregate
* Move IdP-only aggregate to idps/all.xml
* Rename mdq-new-* to mdq-combined-*
* Reference JIRA issues in TODOs
---
 build.xml                              |  60 +++++++
 mdx/incommon/mdq-combined-cloudhsm.xml | 224 +++++++++++++++++++++++++
 mdx/incommon/mdq-combined-localkey.xml | 223 ++++++++++++++++++++++++
 3 files changed, 507 insertions(+)
 create mode 100644 mdx/incommon/mdq-combined-cloudhsm.xml
 create mode 100644 mdx/incommon/mdq-combined-localkey.xml

diff --git a/build.xml b/build.xml
index 1c8348cf..ba49a694 100644
--- a/build.xml
+++ b/build.xml
@@ -2655,6 +2655,66 @@
         <echo>Generation complete.</echo>
     </target>
 
+    <!--
+        inc.mdq-combined.localkey
+
+        Generate all required metadata for the InCommon MDQ service
+        using the localkey configuration:
+
+          * Per-entity metadata
+          * InCommon IdP-only aggregate
+
+        Properties:
+
+        Set sign.uk.keyPassword (note the slightly anomalous "uk" in that
+        identifier) to set the password to access the key resource. If not set,
+        this will be prompted for.
+    -->
+    <target name="inc.mdq.combined.localkey" depends="get.sign.uk.keyPassword">
+      <property name="mda.mdq.input" value="${mda.inc.imported.xml}"/>
+      <property name="mda.sign.keyPassword" value="${sign.uk.keyPassword}"/>
+      <echo>Generating per-entity metadata in ${mda.mdq.output}</echo>
+      <echo>    (IdP-only aggregate in ${mda.mdq.output}/idps.xml)</echo>
+      <echo>    from unsigned aggregate in ${mda.mdq.input}</echo>
+      <delete dir="${mdq.output.dir}" quiet="true"/>
+      <mkdir dir="${mdq.output.dir}"/>
+      <mkdir dir="${mdq.output.dir}/idps"/>
+      <CHANNEL.do channel="incommon" verb="mdq-combined-localkey"/>
+      <echo>Generation complete.</echo>
+    </target>
+
+    <!--
+        inc.mdq.combined.cloudhsm
+
+        Generate all required metadata for the InCommon MDQ service
+        using the cloudhsm configuration:
+
+          * Per-entity metadata
+          * InCommon IdP-only aggregate
+
+        Properties:
+
+        Set sign.uk.keyPassword (note the slightly anomalous "uk" in that
+        identifier) to set the password to access the key resource. If not set,
+        this will be prompted for.
+    -->
+    <target name="inc.mdq.combined.cloudhsm" depends="get.sign.uk.keyPassword">
+      <property name="mda.mdq.input" value="${mda.inc.imported.xml}"/>
+      <property name="mda.sign.keyHandle" value="${sign.keyHandle}"/>
+      <property name="mda.sign.keyUser" value="${sign.keyUser}"/>
+      <property name="mda.sign.keyPassword" value="${sign.keyPassword}"/>
+      <property name="mda.classpath.extra" value="inc-mda-cloudhsm"/>
+      <property name="mda.jni.path" value="/opt/cloudhsm/lib"/>
+      <echo>Generating per-entity metadata in ${mda.mdq.output}</echo>
+      <echo>    (IdP-only aggregate in ${mda.mdq.output}/idps.xml)</echo>
+      <echo>    from unsigned aggregate in ${mda.mdq.input}</echo>
+      <delete dir="${mdq.output.dir}" quiet="true"/>
+      <mkdir dir="${mdq.output.dir}"/>
+      <mkdir dir="${mdq.output.dir}/idps"/>
+      <CHANNEL.do channel="incommon" verb="mdq-combined-cloudhsm"/>
+      <echo>Generation complete.</echo>
+    </target>
+
     <!--
         inc.mdq.generate.localkey
 
diff --git a/mdx/incommon/mdq-combined-cloudhsm.xml b/mdx/incommon/mdq-combined-cloudhsm.xml
new file mode 100644
index 00000000..6a54503d
--- /dev/null
+++ b/mdx/incommon/mdq-combined-cloudhsm.xml
@@ -0,0 +1,224 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+    Generate and sign per-entity metadata using a local private key resource.
+-->
+<beans xmlns="http://www.springframework.org/schema/beans"
+    default-lazy-init="true"
+    xmlns:c="http://www.springframework.org/schema/c"
+    xmlns:p="http://www.springframework.org/schema/p"
+    xmlns:util="http://www.springframework.org/schema/util"
+    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+    xsi:schemaLocation="
+        http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd
+        http://www.springframework.org/schema/util http://www.springframework.org/schema/util/spring-util.xsd">
+
+    <!--
+        Import commonly used beans.
+    -->
+    <import resource="classpath:common-beans.xml"/>
+
+    <!--
+        Import inc-mda beans. (from import.xml)
+    -->
+    <import resource="classpath:uk/org/iay/incommon/mda/beans.xml"/>
+
+    <!--
+        Import channel-specific beans.
+    -->
+    <import resource="classpath:incommon/beans.xml"/>
+
+    <!-- This bean MUST be called "conversionService" to work properly. -->
+    <bean id="conversionService" class="org.springframework.context.support.ConversionServiceFactoryBean">
+        <property name="converters">
+            <set>
+                <bean class="net.shibboleth.ext.spring.config.DurationToLongConverter" />
+                <bean class="net.shibboleth.ext.spring.config.StringToIPRangeConverter" />
+                <bean class="net.shibboleth.ext.spring.config.BooleanToPredicateConverter" />
+                <bean class="net.shibboleth.ext.spring.config.StringBooleanToPredicateConverter" />
+                <bean class="net.shibboleth.ext.spring.config.StringToResourceConverter" />
+            </set>
+        </property>
+    </bean>
+
+    <!--
+        ***********************************************************
+        ***                                                     ***
+        ***   C O M M O N   O U T P U T   P R O C E S S I N G   ***
+        ***                                                     ***
+        ***********************************************************
+    -->
+
+    <bean id="common.output" parent="mda.CompositeStage">
+        <property name="composedStages">
+            <list>
+                <!-- Construct a new aggregate from the collection of entities. -->
+                <bean id="inc.assemble" parent="mda.EntitiesDescriptorAssemblerStage">
+                    <property name="itemOrderingStrategy">
+                        <bean parent="inc.InCommonEntityOrderingStrategy"
+                            c:_-ref="us_incommon_registrar"/>
+                    </property>
+                </bean>
+
+                <!-- Apply final tweaks to the aggregate. -->
+                <bean id="finalise" parent="incommon_finalise_parent">
+                    <property name="transformParameters">
+                        <map>
+                            <entry key="extraText" value="Contains InCommon and eduGAIN metadata"/>
+                            <entry key="publisher" value-ref="us_incommon_registrar"/>
+                        </map>
+                    </property>
+                </bean>
+
+                <!-- Normalise the use of namespace prefixes in the resulting XML document. -->
+                <bean id="normalise" parent="mda.XSLTransformationStage"
+                    p:XSLResource="classpath:incommon/ns_norm_import.xsl"/>
+            </list>
+        </property>
+    </bean>
+
+    <!--
+        *****************************************
+        ***                                   ***
+        ***   I D P - O N L Y   O U T P U T   ***
+        ***                                   ***
+        *****************************************
+    -->
+
+    <!--
+        idp.serialize
+
+        Writes the IdP-only aggregate out to a file.
+    -->
+    <bean id="idp.serialize" parent="mda.SerializationStage">
+        <property name="serializer" ref="serializer"/>
+        <property name="outputFile">
+            <bean parent="File">
+                <constructor-arg value="${mdq.output}/idps/all.xml"/>
+            </bean>
+        </property>
+    </bean>
+
+    <!--
+        idp.selector
+
+        Entities in the IdP-only aggregate are restricted to identity providers.
+    -->
+    <bean id="idp.selector" parent="mda.XPathItemSelectionStrategy">
+        <constructor-arg value="/md:EntityDescriptor[md:IDPSSODescriptor]"/>
+        <constructor-arg ref="commonNamespaces"/>
+    </bean>
+
+    <!--
+        idp.pipeline
+
+        Generates the IdP-only aggregate. The selector has already taken care
+        of removing non-IdP entities.
+    -->
+    <bean id="idp.pipeline" parent="mda.SimplePipeline">
+        <property name="stages">
+            <list>
+                <!-- Perform common steps in constructing an output aggregate. -->
+                <ref bean="common.output"/>
+
+                <!-- Write the resulting aggregate out to a file. -->
+                <ref bean="idp.serialize"/>
+            </list>
+        </property>
+    </bean>
+
+    <!--
+        Generate all required metadata for the InCommon MDQ service
+        using the localkey configuration:
+
+          * Per-entity metadata
+          * InCommon aggregate
+          * InCommon IdP-only aggregate
+
+        Properties:
+
+        Set sign.uk.keyPassword (note the slightly anomalous "uk" in that
+        identifier) to set the password to access the key resource. If not set,
+        this will be prompted for.
+    -->
+    <bean id="mdq-combined-cloudhsm" parent="mda.SimplePipeline">
+        <property name="stages">
+            <list>
+                <!--
+                    Start with the unsigned production aggregate.
+                    TODO: (MDQ-177) For now, we're actually starting with a signed aggregate;
+                      this pipeline isn't configured to check the signature
+                -->
+                <bean id="production_aggregate" parent="mda.DOMResourceSourceStage">
+                    <property name="parserPool" ref="parserPool"/>
+                    <property name="DOMResource">
+                        <bean parent="FileSystemResource" c:_0="${mdq.input}"/>
+                    </property>
+                </bean>
+
+                <!-- Break down into individual entities. -->
+                <ref bean="disassemble"/>
+
+                <!-- Include a default registrationAuthority for each entity. -->
+                <ref bean="us_incommon_default_regauth"/>
+
+                <!-- Populate identifiers for future actions. -->
+                <ref bean="populateItemIds"/>
+                <ref bean="populateRegistrationAuthorities"/>
+
+                <!-- Discard entities in the InCommon entity blacklist. -->
+                <ref bean="remove_blacklisted_incommon_entities"/>
+
+                <!-- Fork new pipelines to generate special aggregates. -->
+                <bean id="demux" parent="mda.PipelineDemultiplexerStage">
+                    <property name="pipelineAndSelectionStrategies">
+                        <list>
+                            <!-- IdP-only aggregate. -->
+                            <bean class="net.shibboleth.utilities.java.support.collection.Pair">
+                                <constructor-arg ref="idp.pipeline"/>
+                                <constructor-arg ref="idp.selector"/>
+                            </bean>
+                        </list>
+                    </property>
+                    <property name="waitingForPipelines" value="true"/>
+                </bean>
+
+                <!-- Set ID, cacheDuration and validUntil attributes. -->
+                <bean parent="mda.GenerateIdStage"/>
+                <bean parent="mda.SetValidUntilStage" p:validityDuration="P14D"/>
+
+                <!-- Identity transform fixes signing issues. -->
+                <bean parent="mda.XSLTransformationStage"
+                    p:XSLResource="classpath:identity.xsl"/>
+
+                <!-- Sign each item. -->
+                <bean id="perform.signature" parent="mda.XMLSignatureSigningStage">
+                    <property name="privateKey">
+                      <bean class="uk.org.iay.incommon.mda.cloudhsm.CaviumPrivateKeyFactoryBean"
+                          p:username="${sign.keyUser}"
+                          p:password="${sign.keyPassword}"
+                          p:keyHandle="${sign.keyHandle}"
+                      />
+                    </property>
+                </bean>
+
+                <!-- Write individual entity documents to files. -->
+                <bean id="write.perentity" parent="mda.MultiOutputSerializationStage">
+                    <property name="serializer" ref="serializer"/>
+                    <property name="outputStrategy">
+                        <bean parent="mda.FilesInDirectoryMultiOutputStrategy" p:nameSuffix=".xml">
+                            <property name="directory">
+                                <bean parent="File" c:_="${mdq.output}"/>
+                            </property>
+                            <property name="nameTransformer">
+                                <bean parent="mda.PathSegmentStringTransformer"/>
+                            </property>
+                        </bean>
+                    </property>
+                </bean>
+
+            </list>
+        </property>
+
+    </bean>
+
+</beans>
diff --git a/mdx/incommon/mdq-combined-localkey.xml b/mdx/incommon/mdq-combined-localkey.xml
new file mode 100644
index 00000000..5e686953
--- /dev/null
+++ b/mdx/incommon/mdq-combined-localkey.xml
@@ -0,0 +1,223 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+    Generate and sign per-entity metadata using a local private key resource.
+-->
+<beans xmlns="http://www.springframework.org/schema/beans"
+    default-lazy-init="true"
+    xmlns:c="http://www.springframework.org/schema/c"
+    xmlns:p="http://www.springframework.org/schema/p"
+    xmlns:util="http://www.springframework.org/schema/util"
+    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+    xsi:schemaLocation="
+        http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd
+        http://www.springframework.org/schema/util http://www.springframework.org/schema/util/spring-util.xsd">
+
+    <!--
+        Import commonly used beans.
+    -->
+    <import resource="classpath:common-beans.xml"/>
+
+    <!--
+        Import inc-mda beans. (from import.xml)
+    -->
+    <import resource="classpath:uk/org/iay/incommon/mda/beans.xml"/>
+
+    <!--
+        Import channel-specific beans.
+    -->
+    <import resource="classpath:incommon/beans.xml"/>
+
+    <!-- This bean MUST be called "conversionService" to work properly. -->
+    <bean id="conversionService" class="org.springframework.context.support.ConversionServiceFactoryBean">
+        <property name="converters">
+            <set>
+                <bean class="net.shibboleth.ext.spring.config.DurationToLongConverter" />
+                <bean class="net.shibboleth.ext.spring.config.StringToIPRangeConverter" />
+                <bean class="net.shibboleth.ext.spring.config.BooleanToPredicateConverter" />
+                <bean class="net.shibboleth.ext.spring.config.StringBooleanToPredicateConverter" />
+                <bean class="net.shibboleth.ext.spring.config.StringToResourceConverter" />
+            </set>
+        </property>
+    </bean>
+
+    <!--
+        ***********************************************************
+        ***                                                     ***
+        ***   C O M M O N   O U T P U T   P R O C E S S I N G   ***
+        ***                                                     ***
+        ***********************************************************
+    -->
+
+    <bean id="common.output" parent="mda.CompositeStage">
+        <property name="composedStages">
+            <list>
+                <!-- Construct a new aggregate from the collection of entities. -->
+                <bean id="inc.assemble" parent="mda.EntitiesDescriptorAssemblerStage">
+                    <property name="itemOrderingStrategy">
+                        <bean parent="inc.InCommonEntityOrderingStrategy"
+                            c:_-ref="us_incommon_registrar"/>
+                    </property>
+                </bean>
+
+                <!-- Apply final tweaks to the aggregate. -->
+                <bean id="finalise" parent="incommon_finalise_parent">
+                    <property name="transformParameters">
+                        <map>
+                            <entry key="extraText" value="Contains InCommon and eduGAIN metadata"/>
+                            <entry key="publisher" value-ref="us_incommon_registrar"/>
+                        </map>
+                    </property>
+                </bean>
+
+                <!-- Normalise the use of namespace prefixes in the resulting XML document. -->
+                <bean id="normalise" parent="mda.XSLTransformationStage"
+                    p:XSLResource="classpath:incommon/ns_norm_import.xsl"/>
+            </list>
+        </property>
+    </bean>
+
+    <!--
+        *****************************************
+        ***                                   ***
+        ***   I D P - O N L Y   O U T P U T   ***
+        ***                                   ***
+        *****************************************
+    -->
+
+    <!--
+        idp.serialize
+
+        Writes the IdP-only aggregate out to a file.
+    -->
+    <bean id="idp.serialize" parent="mda.SerializationStage">
+        <property name="serializer" ref="serializer"/>
+        <property name="outputFile">
+            <bean parent="File">
+                <constructor-arg value="${mdq.output}/idps/all.xml"/>
+            </bean>
+        </property>
+    </bean>
+
+    <!--
+        idp.selector
+
+        Entities in the IdP-only aggregate are restricted to identity providers.
+    -->
+    <bean id="idp.selector" parent="mda.XPathItemSelectionStrategy">
+        <constructor-arg value="/md:EntityDescriptor[md:IDPSSODescriptor]"/>
+        <constructor-arg ref="commonNamespaces"/>
+    </bean>
+
+    <!--
+        idp.pipeline
+
+        Generates the IdP-only aggregate. The selector has already taken care
+        of removing non-IdP entities.
+    -->
+    <bean id="idp.pipeline" parent="mda.SimplePipeline">
+        <property name="stages">
+            <list>
+                <!-- Perform common steps in constructing an output aggregate. -->
+                <ref bean="common.output"/>
+
+                <!-- Write the resulting aggregate out to a file. -->
+                <ref bean="idp.serialize"/>
+            </list>
+        </property>
+    </bean>
+
+    <!--
+        Generate all required metadata for the InCommon MDQ service
+        using the localkey configuration:
+
+          * Per-entity metadata
+          * InCommon aggregate
+          * InCommon IdP-only aggregate
+
+        Properties:
+
+        Set sign.uk.keyPassword (note the slightly anomalous "uk" in that
+        identifier) to set the password to access the key resource. If not set,
+        this will be prompted for.
+    -->
+    <bean id="mdq-combined-localkey" parent="mda.SimplePipeline">
+        <property name="stages">
+            <list>
+                <!--
+                    Start with the unsigned production aggregate.
+                    TODO: (MDQ-177) For now, we're actually starting with a signed aggregate;
+                      this pipeline isn't configured to check the signature
+                -->
+                <bean id="production_aggregate" parent="mda.DOMResourceSourceStage">
+                    <property name="parserPool" ref="parserPool"/>
+                    <property name="DOMResource">
+                        <bean parent="FileSystemResource" c:_0="${mdq.input}"/>
+                    </property>
+                </bean>
+
+                <!-- Break down into individual entities. -->
+                <ref bean="disassemble"/>
+
+                <!-- Include a default registrationAuthority for each entity. -->
+                <ref bean="us_incommon_default_regauth"/>
+
+                <!-- Populate identifiers for future actions. -->
+                <ref bean="populateItemIds"/>
+                <ref bean="populateRegistrationAuthorities"/>
+
+                <!-- Discard entities in the InCommon entity blacklist. -->
+                <ref bean="remove_blacklisted_incommon_entities"/>
+
+                <!-- Fork new pipelines to generate special aggregates. -->
+                <bean id="demux" parent="mda.PipelineDemultiplexerStage">
+                    <property name="pipelineAndSelectionStrategies">
+                        <list>
+                            <!-- IdP-only aggregate. -->
+                            <bean class="net.shibboleth.utilities.java.support.collection.Pair">
+                                <constructor-arg ref="idp.pipeline"/>
+                                <constructor-arg ref="idp.selector"/>
+                            </bean>
+                        </list>
+                    </property>
+                    <property name="waitingForPipelines" value="true"/>
+                </bean>
+
+                <!-- Set ID, cacheDuration and validUntil attributes. -->
+                <bean parent="mda.GenerateIdStage"/>
+                <bean parent="mda.SetValidUntilStage" p:validityDuration="P14D"/>
+
+                <!-- Identity transform fixes signing issues. -->
+                <bean parent="mda.XSLTransformationStage"
+                    p:XSLResource="classpath:identity.xsl"/>
+
+                <!-- Sign each item. -->
+                <bean id="perform.signature" parent="mda.XMLSignatureSigningStage">
+                    <property name="privateKey">
+                        <bean parent="PrivateKeyFactoryBean"
+                            p:resource="${sign.keyResource}"
+                            p:privateKeyPassword="${sign.keyPassword}"
+                        />
+                    </property>
+                </bean>
+
+                <!-- Write individual entity documents to files. -->
+                <bean id="write.perentity" parent="mda.MultiOutputSerializationStage">
+                    <property name="serializer" ref="serializer"/>
+                    <property name="outputStrategy">
+                        <bean parent="mda.FilesInDirectoryMultiOutputStrategy" p:nameSuffix=".xml">
+                            <property name="directory">
+                                <bean parent="File" c:_="${mdq.output}"/>
+                            </property>
+                            <property name="nameTransformer">
+                                <bean parent="mda.PathSegmentStringTransformer"/>
+                            </property>
+                        </bean>
+                    </property>
+                </bean>
+
+            </list>
+        </property>
+
+    </bean>
+
+</beans>