Externalise inputs and outputs into a shared workspace.

Inputs now come from entities.dir and members.dir, unsigned product goes to output.dir and temporary files are put in temp.dir. The /xml/ directory which used to contain most of the above has been entirely removed. The shared workspace is defined by default in the dev environment, and a non-dev environment will result in a failure if shared.ws.dir is not defined via Jenkins. A check is also made for the existence of the build subdirectory of the shared workspace directory. This may be a step too far in the long run, as many operations don't need it to be present.
InCommon · Jul 31, 2016 · 783ec24 · 783ec24
1 parent 5012efd
commit 783ec24
Show file tree

Hide file tree

Showing 6 changed files with 91 additions and 56 deletions.
diff --git a/build.xml b/build.xml
@@ -89,14 +89,49 @@
     <property name="md.dist.host2.name" value="md2.infr.ukfederation.org.uk"/>
     <property name="md.dist.host3.name" value="md3.infr.ukfederation.org.uk"/>
 
+    <!--
+        *******************************************************
+        ***                                                 ***
+        ***   F I L E   S Y S T E M   P R O P E R T I E S   ***
+        ***                                                 ***
+        *******************************************************
+    -->
+
+    <!--
+        Shared workspace location.
+
+        The property shared.ws.dir must be defined in order to locate the shared
+        workspace used for inputs to and outputs from the tooling. In production,
+        this location is passed through from the Jenkins job. In the dev environment,
+        it is assumed (by dev.properties) to be in this repository's parent directory.
+    -->
+    <fail unless="shared.ws.dir" message="shared.ws.dir was not defined"/>
+
+    <!--
+        There must be a build directory in the shared workspace.
+    -->
+    <fail message="shared workspace ${shared.ws.dir} doesn't contain a build directory">
+        <condition>
+            <not>
+                <available file="${shared.ws.dir}/build" type="dir"/>
+            </not>
+        </condition>
+    </fail>
+
+    <!--
+        External specialised directories.
+    -->
+    <property name="entities.dir" value="${shared.ws.dir}/ukf-data/entities"/>
+    <property name="members.dir" value="${shared.ws.dir}/ukf-data/members"/>
+    <property name="output.dir" value="${shared.ws.dir}/build"/>
+    <property name="temp.dir" value="${shared.ws.dir}/build"/>
+
     <!--
         Local specialised directories.
     -->
     <property name="build.dir" value="${basedir}/build"/>
-    <property name="entities.dir" value="entities"/>
     <property name="mdx.dir" value="${basedir}/mdx"/>
     <property name="rules.dir" value="${mdx.dir}/_rules"/>
-    <property name="xml.dir" value="${basedir}/xml"/>
 
     <!--
         Location of externally supplied tool bundles.
@@ -176,7 +211,7 @@
     <!--
         Full paths to a couple of temporary file locations.
     -->
-    <property name="temp.xml" value="${xml.dir}/temp.xml"/>
+    <property name="temp.xml" value="${temp.dir}/temp.xml"/>
 
 
     <!--
@@ -207,18 +242,18 @@
         <attribute name="i"/>
         <sequential>
             <echo>Verifying @{i}...</echo>
-            <delete file="${xml.dir}/temp.xml" quiet="true" verbose="false"/>
-            <get src="@{i}" dest="${xml.dir}/temp.xml"/>
+            <delete file="${temp.dir}/temp.xml" quiet="true" verbose="false"/>
+            <get src="@{i}" dest="${temp.dir}/temp.xml"/>
 
             <!--
                 Verify using xmlsectool.
             -->
-            <XMLSECTOOL.VFY.uk i="${xml.dir}/temp.xml"/>
+            <XMLSECTOOL.VFY.uk i="${temp.dir}/temp.xml"/>
 
             <!--
                 Delete the temporary file.
             -->
-            <delete file="${xml.dir}/temp.xml" quiet="true" verbose="false"/>
+            <delete file="${temp.dir}/temp.xml" quiet="true" verbose="false"/>
         </sequential>
     </macrodef>
 
@@ -233,23 +268,23 @@
         <attribute name="i"/>
         <sequential>
             <echo>Verifying @{i}...</echo>
-            <delete file="${xml.dir}/temp.xml" quiet="true" verbose="false"/>
-            <get src="@{i}" dest="${xml.dir}/temp.xml"/>
+            <delete file="${temp.dir}/temp.xml" quiet="true" verbose="false"/>
+            <get src="@{i}" dest="${temp.dir}/temp.xml"/>
 
             <!--
                 Verify using metadatatool.
             -->
-            <MDT.VFY.uk i="${xml.dir}/temp.xml"/>
+            <MDT.VFY.uk i="${temp.dir}/temp.xml"/>
 
             <!--
                 Verify using xmlsectool.
             -->
-            <XMLSECTOOL.VFY.uk i="${xml.dir}/temp.xml"/>
+            <XMLSECTOOL.VFY.uk i="${temp.dir}/temp.xml"/>
 
             <!--
                 Delete the temporary file.
             -->
-            <delete file="${xml.dir}/temp.xml" quiet="true" verbose="false"/>
+            <delete file="${temp.dir}/temp.xml" quiet="true" verbose="false"/>
         </sequential>
     </macrodef>
 
@@ -304,8 +339,8 @@
     <target name="generate.members.cms">
         <echo>Generating CMS version of members.xml</echo>
         <XALAN
-            i="${xml.dir}/members.xml"
-            o="${xml.dir}/members-cms.xml"
+            i="${members.dir}/members.xml"
+            o="${output.dir}/members-cms.xml"
             x="${mdx.dir}/uk/members-cms.xsl"/>
         <echo>Generated CMS version of members.xml</echo>
     </target>
@@ -377,15 +412,15 @@
         <!--
             Post-process mda-generated output files.
         -->
-        <MDNORM i="${xml.dir}/${mdaggr.prod.unsigned}"/>
-        <MDNORM i="${xml.dir}/${mdaggr.wayf.unsigned}"/>
-        <MDNORM.noblank i="${xml.dir}/${mdaggr.cdsall.unsigned}"/>
-        <MDNORM.noblank i="${xml.dir}/${mdaggr.cms.unsigned}"/>
-        <MDNORM i="${xml.dir}/${mdaggr.test.unsigned}"/>
-        <MDNORM i="${xml.dir}/${mdaggr.export.unsigned}"/>
-        <MDNORM i="${xml.dir}/${mdaggr.export.preview.unsigned}"/>
-        <MDNORM i="${xml.dir}/${mdaggr.back.unsigned}"/>
-        <fixcrlf file="${xml.dir}/${mdaggr.stats}" eol="lf" encoding="UTF-8"/>
+        <MDNORM i="${output.dir}/${mdaggr.prod.unsigned}"/>
+        <MDNORM i="${output.dir}/${mdaggr.wayf.unsigned}"/>
+        <MDNORM.noblank i="${output.dir}/${mdaggr.cdsall.unsigned}"/>
+        <MDNORM.noblank i="${output.dir}/${mdaggr.cms.unsigned}"/>
+        <MDNORM i="${output.dir}/${mdaggr.test.unsigned}"/>
+        <MDNORM i="${output.dir}/${mdaggr.export.unsigned}"/>
+        <MDNORM i="${output.dir}/${mdaggr.export.preview.unsigned}"/>
+        <MDNORM i="${output.dir}/${mdaggr.back.unsigned}"/>
+        <fixcrlf file="${output.dir}/${mdaggr.stats}" eol="lf" encoding="UTF-8"/>
 
         <echo>Generated UK unsigned metadata.</echo>
     </target>
@@ -411,8 +446,10 @@
         by properties defined in the external properties files, e.g., build.properties.
     -->
     <property name="mda.java.endorsed.dirs" value="${tools.dir}/endorsed"/>
-    <property name="mda.basedir" value="${basedir}"/>
+    <property name="mda.entities.dir" value="${entities.dir}"/>
     <property name="mda.mdx.dir" value="${mdx.dir}"/>
+    <property name="mda.members.dir" value="${members.dir}"/>
+    <property name="mda.output.dir" value="${output.dir}"/>
 
     <!--
         Build a property set of all the properties to be passed through, with
@@ -628,7 +665,7 @@
         <attribute name="digest"/><!-- digest function to use -->
         <sequential>
             <!-- delete the temporary file to be sure we don't use old data -->
-            <delete file="${xml.dir}/temp.xml" quiet="true" verbose="false"/>
+            <delete file="${temp.dir}/temp.xml" quiet="true" verbose="false"/>
 
             <echo>Signing @{i} using digest @{digest}.</echo>
 
@@ -830,25 +867,25 @@
     -->
     <target name="verify">
         <echo>Verifying signed UK metadata.</echo>
-        <VFY.uk.both i="${xml.dir}/${mdaggr.prod.signed}"/>
+        <VFY.uk.both i="${output.dir}/${mdaggr.prod.signed}"/>
 
         <echo>Verifying signed UK WAYF metadata.</echo>
-        <VFY.uk.both i="${xml.dir}/${mdaggr.wayf.signed}"/>
+        <VFY.uk.both i="${output.dir}/${mdaggr.wayf.signed}"/>
 
         <echo>Verifying signed UK CDS full metadata.</echo>
-        <VFY.uk.both i="${xml.dir}/${mdaggr.cdsall.signed}"/>
+        <VFY.uk.both i="${output.dir}/${mdaggr.cdsall.signed}"/>
 
         <echo>Verifying signed UK test metadata.</echo>
-        <VFY.uk.both i="${xml.dir}/${mdaggr.test.signed}"/>
+        <VFY.uk.both i="${output.dir}/${mdaggr.test.signed}"/>
 
         <echo>Verifying signed UK export metadata.</echo>
-        <VFY.uk.both i="${xml.dir}/${mdaggr.export.signed}"/>
+        <VFY.uk.both i="${output.dir}/${mdaggr.export.signed}"/>
 
         <echo>Verifying signed UK export preview metadata.</echo>
-        <VFY.uk.both i="${xml.dir}/${mdaggr.export.preview.signed}"/>
+        <VFY.uk.both i="${output.dir}/${mdaggr.export.preview.signed}"/>
 
         <echo>Verifying signed UK fallback metadata.</echo>
-        <VFY.uk.both i="${xml.dir}/${mdaggr.back.signed}"/>
+        <VFY.uk.both i="${output.dir}/${mdaggr.back.signed}"/>
 
         <echo>Verification completed.</echo>
     </target>
@@ -1059,7 +1096,7 @@
     -->
     <target name="stats">
         <CHANNEL.do channel="uk" verb="statistics"/>
-        <fixcrlf file="${xml.dir}/${mdaggr.stats}" eol="lf" encoding="UTF-8"/>
+        <fixcrlf file="${output.dir}/${mdaggr.stats}" eol="lf" encoding="UTF-8"/>
     </target>
 
     <!--
@@ -1185,7 +1222,7 @@
         <echo>Extracting embedded certificates</echo>
         <XALAN
             i="${uk.collected}"
-            o="${xml.dir}/embedded.pem"
+            o="${temp.dir}/embedded.pem"
             x="${build.dir}/extract_embedded.xsl"/>
     </target>
 
@@ -1194,12 +1231,12 @@
     -->
     <target name="check.embedded" depends="extract.embedded">
         <echo>Checking embedded certificates</echo>
-        <exec executable="perl" dir="${xml.dir}"
-            input="${xml.dir}/embedded.pem">
+        <exec executable="perl" dir="${temp.dir}"
+            input="${temp.dir}/embedded.pem">
             <arg value="${build.dir}/check_embedded.pl"/>
             <arg value="-q"/>
         </exec>
-        <delete file="${xml.dir}/embedded.pem" quiet="true" verbose="false"/>
+        <delete file="${temp.dir}/embedded.pem" quiet="true" verbose="false"/>
     </target>
 
     <!--

diff --git a/dev.properties b/dev.properties
@@ -3,3 +3,6 @@
 #
 # Properties defined for the env=dev deployment environment.
 #
+
+# Shared workspace location is assumed to be the parent directory.
+shared.ws.dir=${basedir}/..
diff --git a/mdx/uk/beans.xml b/mdx/uk/beans.xml
@@ -113,7 +113,7 @@
         <property name="parserPool" ref="parserPool"/>
         <property name="source">
             <bean class="java.io.File">
-                <constructor-arg value="${basedir}/entities"/>
+                <constructor-arg value="${entities.dir}"/>
             </bean>
         </property>
         <property name="sourceFileFilter">
@@ -130,7 +130,7 @@
         A Resource referencing the members.xml document.
     -->
     <bean id="uk_membersResource" parent="FileSystemResource">
-        <constructor-arg value="${basedir}/xml/members.xml"/>
+        <constructor-arg value="${members.dir}/members.xml"/>
     </bean>
 
 
@@ -153,7 +153,7 @@
     <bean id="uk_members" class="uk.org.ukfederation.members.Members">
         <constructor-arg>
             <bean class="java.io.File">
-                <constructor-arg value="${basedir}/xml/members.xml"/>
+                <constructor-arg value="${members.dir}/members.xml"/>
             </bean>
         </constructor-arg>
     </bean>
@@ -453,7 +453,7 @@
     <bean id="uk_serializeStatistics" parent="SerializationStage">
         <property name="outputFile">
             <bean class="java.io.File">
-                <constructor-arg value="${basedir}/xml/ukfederation-stats.html"/>
+                <constructor-arg value="${output.dir}/ukfederation-stats.html"/>
             </bean>
         </property>
     </bean>

diff --git a/mdx/uk/generate.xml b/mdx/uk/generate.xml
@@ -234,7 +234,7 @@
     <bean id="serializeUnsignedProductionAggregate" parent="SerializationStage">
         <property name="outputFile">
             <bean class="java.io.File">
-                <constructor-arg value="${basedir}/xml/ukfederation-metadata-unsigned.xml"/>
+                <constructor-arg value="${output.dir}/ukfederation-metadata-unsigned.xml"/>
             </bean>
         </property>
     </bean>
@@ -291,7 +291,7 @@
     <bean id="serializeUnsignedWayfAggregate" parent="SerializationStage">
         <property name="outputFile">
             <bean class="java.io.File">
-                <constructor-arg value="${basedir}/xml/ukfederation-wayf-unsigned.xml"/>
+                <constructor-arg value="${output.dir}/ukfederation-wayf-unsigned.xml"/>
             </bean>
         </property>
     </bean>
@@ -407,7 +407,7 @@
     <bean id="serializeCDSAllAggregate" parent="SerializationStage">
         <property name="outputFile">
             <bean class="java.io.File">
-                <constructor-arg value="${basedir}/xml/ukfederation-cdsall-unsigned.xml"/>
+                <constructor-arg value="${output.dir}/ukfederation-cdsall-unsigned.xml"/>
             </bean>
         </property>
     </bean>
@@ -469,7 +469,7 @@
     <bean id="serializeUnsignedFallbackAggregate" parent="SerializationStage">
         <property name="outputFile">
             <bean class="java.io.File">
-                <constructor-arg value="${basedir}/xml/ukfederation-back-unsigned.xml"/>
+                <constructor-arg value="${output.dir}/ukfederation-back-unsigned.xml"/>
             </bean>
         </property>
     </bean>
@@ -523,7 +523,7 @@
     <bean id="serializeUnsignedTestAggregate" parent="SerializationStage">
         <property name="outputFile">
             <bean class="java.io.File">
-                <constructor-arg value="${basedir}/xml/ukfederation-test-unsigned.xml"/>
+                <constructor-arg value="${output.dir}/ukfederation-test-unsigned.xml"/>
             </bean>
         </property>
     </bean>
@@ -579,7 +579,7 @@
     <bean id="serializeUnsignedExportAggregate" parent="SerializationStage">
         <property name="outputFile">
             <bean class="java.io.File">
-                <constructor-arg value="${basedir}/xml/ukfederation-export-unsigned.xml"/>
+                <constructor-arg value="${output.dir}/ukfederation-export-unsigned.xml"/>
             </bean>
         </property>
     </bean>
@@ -790,7 +790,7 @@
                 <bean id="serializeUnsignedExportPreviewAggregate" parent="SerializationStage">
                     <property name="outputFile">
                         <bean class="java.io.File">
-                            <constructor-arg value="${basedir}/xml/ukfederation-export-preview-unsigned.xml"/>
+                            <constructor-arg value="${output.dir}/ukfederation-export-preview-unsigned.xml"/>
                         </bean>
                     </property>
                 </bean>
@@ -864,7 +864,7 @@
     <bean id="serializeCMSAggregate" parent="SerializationStage">
         <property name="outputFile">
             <bean class="java.io.File">
-                <constructor-arg value="${basedir}/xml/cms-metadata.xml"/>
+                <constructor-arg value="${output.dir}/cms-metadata.xml"/>
             </bean>
         </property>
     </bean>

diff --git a/mdx/uk/verbs.xml b/mdx/uk/verbs.xml
@@ -194,7 +194,7 @@
     <bean id="fetchImportMetadata" parent="DOMFilesystemSourceStage">
         <property name="source">
             <bean class="java.io.File">
-                <constructor-arg value="${basedir}/entities/import.xml"/>
+                <constructor-arg value="${entities.dir}/import.xml"/>
             </bean>
         </property>
     </bean>
@@ -207,7 +207,7 @@
     <bean id="serializeImportedMetadata" parent="SerializationStage">
         <property name="outputFile">
             <bean class="java.io.File">
-                <constructor-arg value="${basedir}/entities/imported.xml"/>
+                <constructor-arg value="${entities.dir}/imported.xml"/>
             </bean>
         </property>
     </bean>

diff --git a/xml/.cvsignore b/xml/.cvsignore