Add process_main_aggregate.sh script

bin/process_main_aggregate.sh

@@ -0,0 +1,264 @@
+#!/bin/bash
+
+#######################################################################
+# Copyright 2017 Internet2
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#######################################################################
+
+#######################################################################
+# Help message
+#######################################################################
+
+display_help () {
+/bin/cat <<- HELP_MSG
+	This script produces the following files from the InCommon 
+	main aggregate:
+	
+	  $csv_filename1
+	  $csv_filename2
+	
+	The script is intended to be run as a cron job. It can (and
+	should) be run often, to pick up the latest changes to metadata.
+	There is little penalty in doing so since the script uses HTTP
+	Conditional GET to retrieve metadata.
+	
+	Usage: ${0##*/} [-hv] -d OUT_DIR
+	
+	Options:
+	   -h      Display this help message
+	   -v      Enable DEBUG mode
+	   -d      Specify the output directory
+
+	Option -h is mutually exclusive of all other options.
+	
+	Option -d specifies the ultimate output directory, which is
+	usually a web directory. This option is REQUIRED.
+	
+	ENVIRONMENT
+	
+	This script leverages numerous environment variables:
+	
+	  BIN_DIR    A directory of executable scripts
+	  LIB_DIR    A source library directory
+	  CACHE_DIR  An HTTP cache
+	  TMPDIR     A temporary directory
+	  LOG_FILE   A persistent log file
+	  LOG_LEVEL  The global log level [0..5]
+	
+	All of the above environment variables are REQUIRED
+	except LOG_LEVEL, which defaults to LOG_LEVEL=3.
+	
+	EXAMPLES
+	
+	  \$ out_dir=/home/htdocs/www.incommonfederation.org/federation/metadata/
+	  \$ ${0##*/} -d \$out_dir
+HELP_MSG
+}
+
+#######################################################################
+# Bootstrap
+#######################################################################
+
+script_name=${0##*/}  # equivalent to basename $0
+
+# required directories
+env_vars="BIN_DIR
+LIB_DIR
+CACHE_DIR
+TMPDIR"
+
+# check required directories
+for env_var in $env_vars; do
+	eval "env_var_val=\${$env_var}"
+	if [ -z "$env_var_val" ]; then
+		echo "ERROR: $script_name requires env var $env_var" >&2
+		exit 2
+	fi
+	if [ ! -d "$env_var_val" ]; then
+		echo "ERROR: $script_name: directory does not exist: $env_var_val" >&2
+		exit 2
+	fi
+done
+
+# check the log file
+if [ -z "$LOG_FILE" ]; then
+	echo "ERROR: $script_name requires env var LOG_FILE" >&2
+	exit 2
+fi
+# devices such as /dev/tty and /dev/null are allowed
+if [ ! -f "$LOG_FILE" ] && [[ $LOG_FILE != /dev/* ]]; then
+	echo "ERROR: $script_name: file does not exist: $LOG_FILE" >&2
+	exit 2
+fi
+
+# default to INFO logging
+if [ -z "$LOG_LEVEL" ]; then
+	LOG_LEVEL=3
+fi
+
+# output filenames
+csv_filename1="all-idps-rands.csv"
+csv_filename2="all-sps-rands.csv"
+
+#######################################################################
+# Process command-line options and arguments
+#######################################################################
+
+help_mode=false; local_opts=
+while getopts ":hvd:" opt; do
+	case $opt in
+		h)
+			help_mode=true
+			;;
+		v)
+			LOG_LEVEL=4
+			local_opts="$local_opts -$opt"
+			;;
+		d)
+			out_dir="$OPTARG"
+			;;
+		\?)
+			echo "ERROR: $script_name: Unrecognized option: -$OPTARG" >&2
+			exit 2
+			;;
+		:)
+			echo "ERROR: $script_name: Option -$OPTARG requires an argument" >&2
+			exit 2
+			;;
+	esac
+done
+
+if $help_mode; then
+	display_help
+	exit 0
+fi
+
+# check the output directory
+if [ ! -d "$out_dir" ]; then
+	echo "ERROR: $script_name: directory does not exist: $out_dir" >&2
+	exit 2
+fi
+
+# check command-line arguments
+shift $(( OPTIND - 1 ))
+if [ $# -ne 0 ]; then
+	echo "ERROR: $script_name: wrong number of arguments: $# (0 required)" >&2
+	exit 2
+fi
+
+#######################################################################
+# Initialization
+#######################################################################
+
+# executable script must exist
+cget="$BIN_DIR/cget.sh"
+if [ ! -f "$cget" ]; then
+	echo "ERROR: $script_name: executable file does not exist: $cget" >&2
+	exit 2
+fi
+
+# source core library
+lib_file="$LIB_DIR/core_lib.sh"
+if [ ! -f "$lib_file" ]; then
+	echo "ERROR: $script_name: lib file does not exist: $lib_file" >&2
+	exit 2
+fi
+source "$lib_file"
+status_code=$?
+if [ $status_code -ne 0 ]; then
+	echo "ERROR: $script_name failed ($status_code) to source lib file $lib_file" >&2
+	exit 2
+fi
+
+# XSL stylesheets must exist
+xsl_file1=$LIB_DIR/list_all_RandS_IdPs_csv.xsl
+if [ ! -f "$xsl_file1" ]; then
+	echo "ERROR: $script_name: lib file does not exist: $xsl_file1" >&2
+	exit 2
+fi
+xsl_file2=$LIB_DIR/list_all_RandS_SPs_csv.xsl
+if [ ! -f "$xsl_file2" ]; then
+	echo "ERROR: $script_name: lib file does not exist: $xsl_file2" >&2
+	exit 2
+fi
+
+# create a temporary subdirectory
+tmp_dir="${TMPDIR%%/}/${script_name%%.*}_$$"
+/bin/mkdir "$tmp_dir"
+exit_code=$?
+if [ $exit_code -ne 0 ]; then
+	echo "ERROR: $script_name failed ($exit_code) to create tmp dir $tmp_dir" >&2
+	exit 2
+fi
+
+# input file
+xml_file="${tmp_dir}/saml-metadata.xml"
+
+# output files
+csv_file1="${tmp_dir}/$csv_filename1"
+csv_file2="${tmp_dir}/$csv_filename2"
+
+#######################################################################
+# Main processing
+#######################################################################
+
+print_log_message -I "$script_name BEGIN"
+
+# XML metadata
+md_location=http://md.incommon.org/InCommon/InCommon-metadata.xml
+print_log_message -D "$script_name using metadata file: $md_location"
+
+# get a fresh metadata file
+print_log_message -D "$script_name writing XML file: $xml_file"
+$cget $local_opts -F $md_location > $xml_file
+exit_code=$?
+if [ $exit_code -eq 1 ]; then
+	# short-circuit if 304 response
+	print_log_message -I "$script_name END"
+	clean_up_and_exit -d "$tmp_dir" 0
+fi
+if [ $exit_code -gt 1 ]; then
+	print_log_message -E "$script_name: cget.sh failed ($exit_code) on location: $md_location"
+    clean_up_and_exit -d "$tmp_dir" $exit_code
+fi
+
+# create the first output file
+print_log_message -D "$script_name writing output file: $csv_file1"
+/usr/bin/xsltproc $xsl_file1 $xml_file > $csv_file1
+exit_code=$?
+if [ $exit_code -ne 0 ]; then
+	print_log_message -E "$script_name: xsltproc failed ($exit_code) on stylesheet: $xsl_file1"
+    clean_up_and_exit -d "$tmp_dir" $exit_code
+fi
+
+# create the second output file
+print_log_message -D "$script_name writing output file: $csv_file2"
+/usr/bin/xsltproc $xsl_file2 $xml_file > $csv_file2
+exit_code=$?
+if [ $exit_code -ne 0 ]; then
+	print_log_message -E "$script_name: xsltproc failed ($exit_code) on stylesheet: $xsl_file2"
+    clean_up_and_exit -d "$tmp_dir" $exit_code
+fi
+
+# move the output files to the web directory
+print_log_message -D "$script_name moving output files to dir: $out_dir"
+/bin/mv $csv_file1 $csv_file2 $out_dir
+exit_code=$?
+if [ $exit_code -ne 0 ]; then
+	print_log_message -E "$script_name: mv failed ($exit_code) to dir: $out_dir"
+    clean_up_and_exit -d "$tmp_dir" $exit_code
+fi
+
+print_log_message -I "$script_name END"
+clean_up_and_exit -d "$tmp_dir" 0

install.sh

@@ -93,7 +93,8 @@ while read script_file; do
 	fi
 done <<SCRIPTS
 $script_bin/bin/http_xsltproc.sh
-$script_bin/bin/list_all_entities_exported.sh
+$script_bin/bin/process_export_aggregate.sh
+$script_bin/bin/process_main_aggregate.sh
 SCRIPTS
 
 # initialize lib dir