From ac0de808457ff17b77294dc85eea283dab618cd2 Mon Sep 17 00:00:00 2001
From: Ian Young <ian@iay.org.uk>
Date: Wed, 13 Apr 2016 11:11:24 +0100
Subject: [PATCH] Implement two-phase fetch/process import for eduGAIN.

New target inc.edugain.download acquires new eduGAIN metadata and
(if successful) writes it to a file which is then used instead
of a direct download by the import pipeline. For consistency, the
intermediate file is also used by inc.edugain.report.
---
 build.xml                         | 22 ++++++++++-
 mdx/incommon/.gitignore           |  1 +
 mdx/incommon/download-edugain.xml | 61 +++++++++++++++++++++++++++++++
 mdx/incommon/import.xml           | 28 +++++++++++++-
 mdx/incommon/report.xml           | 28 +++++++++++++-
 5 files changed, 136 insertions(+), 4 deletions(-)
 create mode 100644 mdx/incommon/download-edugain.xml

diff --git a/build.xml b/build.xml
index 22d3b7bc..86c81138 100644
--- a/build.xml
+++ b/build.xml
@@ -1299,6 +1299,13 @@
     -->
     <property name="mda.inc.imported.xml" value="${mda.incdir}/imported.xml"/>
 
+    <!--
+        Default local file location to which to write the eduGAIN aggregate.
+
+        This can be overridden in build.properties.
+    -->
+    <property name="mda.inc.edugain.xml" value="${mda.incdir}/edugain.xml"/>
+
     <!--
         inc.fetch.production
 
@@ -1326,6 +1333,17 @@
         <echo>Generation complete.</echo>
     </target>
 
+    <!--
+        inc.edugain.download
+
+        Downloads the eduGAIN aggregate and writes it into a file if successful.
+    -->
+    <target name="inc.edugain.download">
+        <echo>Downloading eduGAIN aggregate to ${mda.inc.edugain.xml}...</echo>
+        <CHANNEL.do channel="incommon" verb="download-edugain"/>
+        <echo>Download to ${mda.inc.edugain.xml} completed.</echo>
+    </target>
+
     <!--
         inc.generate.import
 
@@ -1334,7 +1352,7 @@
     <target name="inc.generate.import">
         <echo>Generating InCommon import aggregate in ${mda.inc.imported.xml}</echo>
         <echo>    from production aggregate in ${mda.inc.production.xml}</echo>
-        <echo>    and selected eduGAIN entities...</echo>
+        <echo>    and selected eduGAIN entities from ${mda.inc.edugain.xml}...</echo>
         <CHANNEL.do channel="incommon" verb="import"/>
         <echo>Generation complete.</echo>
     </target>
@@ -1345,7 +1363,7 @@
         Report on the eduGAIN entities filtered out because of errors detected.
     -->
     <target name="inc.edugain.report">
-        <echo>Looking for errors in eduGAIN entities...</echo>
+        <echo>Looking for errors in eduGAIN entities from ${mda.inc.edugain.xml}...</echo>
         <CHANNEL.do channel="incommon" verb="report"/>
         <echo>Report complete.</echo>
     </target>
diff --git a/mdx/incommon/.gitignore b/mdx/incommon/.gitignore
index 43811e09..0ebbc03b 100644
--- a/mdx/incommon/.gitignore
+++ b/mdx/incommon/.gitignore
@@ -1,3 +1,4 @@
+edugain.xml
 exported.xml
 imported.xml
 production.xml
diff --git a/mdx/incommon/download-edugain.xml b/mdx/incommon/download-edugain.xml
new file mode 100644
index 00000000..bf0c11ef
--- /dev/null
+++ b/mdx/incommon/download-edugain.xml
@@ -0,0 +1,61 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+    Download eduGAIN aggregate.
+-->
+<beans xmlns="http://www.springframework.org/schema/beans"
+    default-lazy-init="true"
+    xmlns:c="http://www.springframework.org/schema/c"
+    xmlns:p="http://www.springframework.org/schema/p"
+    xmlns:util="http://www.springframework.org/schema/util"
+    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+    xsi:schemaLocation="
+        http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd
+        http://www.springframework.org/schema/util http://www.springframework.org/schema/util/spring-util.xsd">
+
+    <!--
+        Import commonly used beans.
+    -->
+    <import resource="classpath:common-beans.xml"/>
+
+    <!--
+        Import channel-specific beans.
+    -->
+    <import resource="classpath:incommon/beans.xml"/>
+
+    <!--
+        Import eduGAIN channel beans.
+    -->
+    <import resource="classpath:int_edugain/beans.xml"/>
+
+    <bean id="serializeImported" parent="SerializationStage">
+        <property name="outputFile">
+            <bean class="java.io.File">
+                <constructor-arg value="${inc.edugain.xml}"/>
+            </bean>
+        </property>
+    </bean>
+
+    <!--
+        Download eduGAIN aggregate.
+    -->
+    <bean id="download-edugain" parent="SimplePipeline">
+        <property name="stages">
+            <list>
+                <ref bean="int_edugain_productionAggregate"/>
+
+                <!--
+                    Check for fatal errors at the aggregate level:
+                        missing or expired validUntil attribute
+                        invalid signature
+                -->
+                <ref bean="check_validUntil"/>
+                <ref bean="int_edugain_checkSignature"/>
+                <ref bean="errorTerminatingFilter"/>
+
+                <!-- Write the downloaded aggregate out to a file. -->
+                <ref bean="serializeImported"/>
+            </list>
+        </property>
+    </bean>
+
+</beans>
diff --git a/mdx/incommon/import.xml b/mdx/incommon/import.xml
index 5579320d..d9a86965 100644
--- a/mdx/incommon/import.xml
+++ b/mdx/incommon/import.xml
@@ -39,7 +39,33 @@
     <bean id="incommon_edugain_importPipeline" parent="SimplePipeline">
         <property name="stages">
             <list>
-                <ref bean="int_edugain_productionEntities"/>
+                <!--
+                    Load the saved eduGAIN aggregate from a file.
+                -->
+                <bean id="edugain_aggregate" parent="DOMResourceSourceStage">
+                    <property name="DOMResource">
+                        <bean parent="FileSystemResource" c:_0="${inc.edugain.xml}"/>
+                    </property>
+                </bean>
+
+                <!--
+                    Check for fatal errors at the aggregate level:
+                        missing or expired validUntil attribute
+                        invalid signature
+                -->
+                <ref bean="check_validUntil"/>
+                <ref bean="int_edugain_checkSignature"/>
+                <ref bean="errorTerminatingFilter"/>
+
+                <ref bean="disassemble"/>
+
+                <ref bean="int_edugain_removeBlacklistedEntities"/>
+
+                <!--
+                    All eduGAIN entities should have mdrpi:RegistrationInfo elements, but
+                    we can't check the actual values.
+                -->
+                <ref bean="check_hasreginfo"/>
 
                 <!-- Populate identifiers for future actions. -->
                 <ref bean="populateItemIds"/>
diff --git a/mdx/incommon/report.xml b/mdx/incommon/report.xml
index fafdc182..774071f6 100644
--- a/mdx/incommon/report.xml
+++ b/mdx/incommon/report.xml
@@ -41,7 +41,33 @@
     <bean id="report" parent="SimplePipeline">
         <property name="stages">
             <list>
-                <ref bean="int_edugain_productionEntities"/>
+                <!--
+                    Load the saved eduGAIN aggregate from a file.
+                -->
+                <bean id="edugain_aggregate" parent="DOMResourceSourceStage">
+                    <property name="DOMResource">
+                        <bean parent="FileSystemResource" c:_0="${inc.edugain.xml}"/>
+                    </property>
+                </bean>
+
+                <!--
+                    Check for fatal errors at the aggregate level:
+                        missing or expired validUntil attribute
+                        invalid signature
+                -->
+                <ref bean="check_validUntil"/>
+                <ref bean="int_edugain_checkSignature"/>
+                <ref bean="errorTerminatingFilter"/>
+
+                <ref bean="disassemble"/>
+
+                <ref bean="int_edugain_removeBlacklistedEntities"/>
+
+                <!--
+                    All eduGAIN entities should have mdrpi:RegistrationInfo elements, but
+                    we can't check the actual values.
+                -->
+                <ref bean="check_hasreginfo"/>
 
                 <!-- Populate identifiers for future actions. -->
                 <ref bean="populateItemIds"/>