From 97b634ea8ce8be12af53e3b37d11713520afc2c5 Mon Sep 17 00:00:00 2001
From: Alex Stuart <alex.stuart@jisc.ac.uk>
Date: Tue, 29 Sep 2020 11:27:10 +0100
Subject: [PATCH] Re-apply "Generate a new aggregate for Wugen"

This reverts commit 72497572f461e4c3cfa137d85adaa92269f56393.
---
 build.xml           | 26 ++++++++++++++++++-
 mdx/uk/generate.xml | 61 ++++++++++++++++++++++++++++++++++++++++++++-
 2 files changed, 85 insertions(+), 2 deletions(-)

diff --git a/build.xml b/build.xml
index 82aebd81..a879e1a4 100644
--- a/build.xml
+++ b/build.xml
@@ -301,7 +301,7 @@
 
     <!--
         There are many separate processing "streams":  production, test, export,
-        fallback, "wayf", and "cdsall".
+        fallback, "wayf", "cdsall" and "wugen".
 
         Each stream has its own unsigned file, as follows:
     -->
@@ -313,6 +313,7 @@
     <property name="mdaggr.back.unsigned"   value="ukfederation-back-unsigned.xml"/>
     <property name="mdaggr.wayf.unsigned"   value="ukfederation-wayf-unsigned.xml"/>
     <property name="mdaggr.cdsall.unsigned" value="ukfederation-cdsall-unsigned.xml"/>
+    <property name="mdaggr.wugen.unsigned"  value="ukfederation-wugen-unsigned.xml"/>
 
     <!--
         Each stream also has its own signed file, as follows:
@@ -325,6 +326,7 @@
     <property name="mdaggr.back.signed"     value="ukfederation-back.xml"/>
     <property name="mdaggr.wayf.signed"     value="ukfederation-wayf.xml"/>
     <property name="mdaggr.cdsall.signed"   value="ukfederation-cdsall.xml"/>
+    <property name="mdaggr.wugen.signed"    value="ukfederation-wugen.xml"/>
 
     <!--
         Other files
@@ -1392,6 +1394,8 @@
             property="mdaggr.export.signed.checksum"/>
         <checksum file="${aggregates.dir}/${mdaggr.export.preview.signed}"
             property="mdaggr.export.preview.signed.checksum"/>
+        <checksum file="${aggregates.dir}/${mdaggr.wugen.signed}"
+            property="mdaggr.wugen.signed.checksum"/>
 
         <echo>Verifying metadata held at ${cdi-master.name}</echo>
         <VFY.remote.and.checksum i="http://${cdi-master.name}${cdi-master.md.path.name}${mdaggr.prod.signed}"
@@ -1408,6 +1412,8 @@
             checksum="${mdaggr.export.signed.checksum}"/>
         <VFY.remote.and.checksum i="http://${cdi-master.name}${cdi-master.md.path.name}${mdaggr.export.preview.signed}"
             checksum="${mdaggr.export.preview.signed.checksum}"/>
+        <VFY.remote.and.checksum i="http://${cdi-master.name}${cdi-master.md.path.name}${mdaggr.wugen.signed}"
+            checksum="${mdaggr.wugen.signed.checksum}"/>
 
         <echo>Verifying metadata held at ${md.dist.host-ne-01.name}</echo>
         <VFY.remote.and.checksum i="http://${md.dist.host-ne-01.name}${md.dist.path.name}${mdaggr.prod.signed}"
@@ -1424,6 +1430,8 @@
             checksum="${mdaggr.export.signed.checksum}"/>
         <VFY.remote.and.checksum i="http://${md.dist.host-ne-01.name}${md.dist.path.name}${mdaggr.export.preview.signed}"
             checksum="${mdaggr.export.preview.signed.checksum}"/>
+        <VFY.remote.and.checksum i="http://${md.dist.host-ne-01.name}${md.dist.path.name}${mdaggr.wugen.signed}"
+            checksum="${mdaggr.wugen.signed.checksum}"/>
 
         <echo>Verifying metadata held at ${md.dist.host-ne-02.name}</echo>
         <VFY.remote.and.checksum i="http://${md.dist.host-ne-02.name}${md.dist.path.name}${mdaggr.prod.signed}"
@@ -1440,6 +1448,8 @@
             checksum="${mdaggr.export.signed.checksum}"/>
         <VFY.remote.and.checksum i="http://${md.dist.host-ne-02.name}${md.dist.path.name}${mdaggr.export.preview.signed}"
             checksum="${mdaggr.export.preview.signed.checksum}"/>
+        <VFY.remote.and.checksum i="http://${md.dist.host-ne-02.name}${md.dist.path.name}${mdaggr.wugen.signed}"
+            checksum="${mdaggr.wugen.signed.checksum}"/>
 
         <echo>Verifying metadata held at ${md.dist.host-we-01.name}</echo>
         <VFY.remote.and.checksum i="http://${md.dist.host-we-01.name}${md.dist.path.name}${mdaggr.prod.signed}"
@@ -1456,6 +1466,8 @@
             checksum="${mdaggr.export.signed.checksum}"/>
         <VFY.remote.and.checksum i="http://${md.dist.host-we-01.name}${md.dist.path.name}${mdaggr.export.preview.signed}"
             checksum="${mdaggr.export.preview.signed.checksum}"/>
+        <VFY.remote.and.checksum i="http://${md.dist.host-we-01.name}${md.dist.path.name}${mdaggr.wugen.signed}"
+            checksum="${mdaggr.wugen.signed.checksum}"/>
 
         <echo>Verifying metadata held at ${md.dist.host-we-02.name}</echo>
         <VFY.remote.and.checksum i="http://${md.dist.host-we-02.name}${md.dist.path.name}${mdaggr.prod.signed}"
@@ -1472,6 +1484,8 @@
             checksum="${mdaggr.export.signed.checksum}"/>
         <VFY.remote.and.checksum i="http://${md.dist.host-we-02.name}${md.dist.path.name}${mdaggr.export.preview.signed}"
             checksum="${mdaggr.export.preview.signed.checksum}"/>
+        <VFY.remote.and.checksum i="http://${md.dist.host-we-02.name}${md.dist.path.name}${mdaggr.wugen.signed}"
+            checksum="${mdaggr.wugen.signed.checksum}"/>
 
         <echo>Verification completed.</echo>
     </target>
@@ -1559,6 +1573,7 @@
         <MDNORM i="${output.dir}/${mdaggr.prod.unsigned}"/>
         <MDNORM i="${output.dir}/${mdaggr.wayf.unsigned}"/>
         <MDNORM.noblank i="${output.dir}/${mdaggr.cdsall.unsigned}"/>
+        <MDNORM.noblank i="${output.dir}/${mdaggr.wugen.unsigned}"/>
         <MDNORM i="${output.dir}/${mdaggr.test.unsigned}"/>
         <MDNORM i="${output.dir}/${mdaggr.export.unsigned}"/>
         <MDNORM i="${output.dir}/${mdaggr.export.preview.unsigned}"/>
@@ -1964,6 +1979,9 @@
         <echo>Signing UKfed fallback metadata.</echo>
         <SIGN.uk i="${output.dir}/${mdaggr.back.unsigned}" o="${output.dir}/${mdaggr.back.signed}" digest="SHA-256"/>
 
+        <echo>Signing UKfed Wugen metadata.</echo>
+        <SIGN.uk i="${output.dir}/${mdaggr.wugen.unsigned}" o="${output.dir}/${mdaggr.wugen.signed}" digest="SHA-256"/>
+
     </target>
 
     <!--
@@ -2002,6 +2020,9 @@
         <echo>Verifying signed UK fallback metadata.</echo>
         <XMLSECTOOL.VFY.uk i="${output.dir}/${mdaggr.back.signed}"/>
 
+        <echo>Verifying signed UK Wugen metadata.</echo>
+        <XMLSECTOOL.VFY.uk i="${output.dir}/${mdaggr.wugen.signed}"/>
+
         <echo>Verification completed.</echo>
     </target>
 
@@ -2091,6 +2112,7 @@
                 <include name="${mdaggr.back.unsigned}"/>
                 <include name="${mdaggr.export.unsigned}"/>
                 <include name="${mdaggr.export.preview.unsigned}"/>
+                <include name="${mdaggr.wugen.unsigned}"/>
                 <include name="${mdaggr.stats}"/>
             </fileset>
         </scp>
@@ -2107,6 +2129,7 @@
                 <include name="${mdaggr.back.unsigned}"/>
                 <include name="${mdaggr.export.unsigned}"/>
                 <include name="${mdaggr.export.preview.unsigned}"/>
+                <include name="${mdaggr.wugen.unsigned}"/>
             </fileset>
         </scp>
     </target>
@@ -2120,6 +2143,7 @@
         <scp failonerror="true" remoteFile="${keymaster.url}/${mdaggr.back.signed}" todir="${aggregates.dir}" keyfile="~/.ssh/id_rsa" knownhosts="~/.ssh/known_hosts"/>
         <scp failonerror="true" remoteFile="${keymaster.url}/${mdaggr.export.signed}" todir="${aggregates.dir}" keyfile="~/.ssh/id_rsa" knownhosts="~/.ssh/known_hosts"/>
         <scp failonerror="true" remoteFile="${keymaster.url}/${mdaggr.export.preview.signed}" todir="${aggregates.dir}" keyfile="~/.ssh/id_rsa" knownhosts="~/.ssh/known_hosts"/>
+        <scp failonerror="true" remoteFile="${keymaster.url}/${mdaggr.wugen.signed}" todir="${aggregates.dir}" keyfile="~/.ssh/id_rsa" knownhosts="~/.ssh/known_hosts"/>
     </target>
 
     <target name="fs.scp.mdqcache.from.keymaster">
diff --git a/mdx/uk/generate.xml b/mdx/uk/generate.xml
index 253bd4d6..1c7696ba 100644
--- a/mdx/uk/generate.xml
+++ b/mdx/uk/generate.xml
@@ -419,7 +419,6 @@
                 <ref bean="stripComments"/>
 
                 <ref bean="stripAlgNamespace"/>
-                <ref bean="stripInitNamespace"/>
                 <ref bean="stripMdattrNamespace"/>
                 <ref bean="stripMdrpiNamespace"/>
                 <ref bean="stripUkfedlabelNamespace"/>
@@ -504,6 +503,7 @@
                     p:cacheDuration="${cacheDuration.aggregate.duration}"/>
 
                 <!-- remove many things that the CDS doesn't look at -->
+                <ref bean="stripInitNamespace"/>
                 <ref bean="CDSStripUnwanted"/>
 
                 <!--
@@ -525,6 +525,61 @@
         </property>
     </bean>
 
+    <!--
+        *****************************************
+        ***                                   ***
+        ***   W U G E N   A G G R E G A T E   ***
+        ***                                   ***
+        *****************************************
+    -->
+
+    <bean id="serializeWugenAggregate" parent="mda.SerializationStage">
+        <property name="serializer" ref="serializer"/>
+        <property name="outputFile">
+            <bean parent="File">
+                <constructor-arg value="${output.dir}/ukfederation-wugen-unsigned.xml"/>
+            </bean>
+        </property>
+    </bean>
+
+    <bean id="WugenPipeline" parent="mda.SimplePipeline">
+        <property name="stages">
+            <list>
+                <!-- Apply flow constraints for this flow. -->
+                <bean id="flowConstraints" parent="ukf.FlowConstraintApplyingStage"
+                    p:flowName="wugen"/>
+
+                <!--
+                    Enforce IdP display name uniqueness before assembling aggregate
+                -->
+                <ref bean="check_dup_display"/>
+                <ref bean="errorTerminatingFilter"/>
+
+                <!-- make an aggregate first so that we're only traversing one item -->
+                <ref bean="assembleAggregate"/>
+
+                <!-- remove many things that the CDS doesn't look at -->
+                <ref bean="CDSStripUnwanted"/>
+
+                <!--
+                    Remove embedded images used in mdui:Logo elements.
+                -->
+                <ref bean="stripMDUILogoData"/>
+                <ref bean="stripEmptyMDUIUIInfo"/>
+                <ref bean="stripEmptyExtensions"/>
+
+                <ref bean="CDSFinalise"/>
+                <ref bean="CDSNormaliseNamespaces"/>
+
+                <!-- schema validity check MUST pass -->
+                <ref bean="checkSchemas"/>
+                <ref bean="errorTerminatingFilter"/>
+
+                <ref bean="serializeWugenAggregate"/>
+            </list>
+        </property>
+    </bean>
+
     <!--
         ***********************************************
         ***                                         ***
@@ -1110,6 +1165,10 @@
                                 <constructor-arg ref="CDSAllPipeline"/>
                                 <constructor-arg ref="CDSAllSelector"/>
                             </bean>
+                            <bean class="net.shibboleth.utilities.java.support.collection.Pair">
+                                <constructor-arg ref="WugenPipeline"/>
+                                <constructor-arg ref="CDSAllSelector"/>
+                            </bean>
                         </list>
                     </property>
                     <property name="waitingForPipelines" value="true"/>