From bc8888720ef56a81d8aff64cbc3030a796c09e4d Mon Sep 17 00:00:00 2001 From: Tom Scavo Date: Sat, 6 May 2017 16:48:19 -0400 Subject: [PATCH] Add process_main_aggregate.sh script --- bin/process_main_aggregate.sh | 264 ++++++++++++++++++++++++++++++++++ install.sh | 3 +- 2 files changed, 266 insertions(+), 1 deletion(-) create mode 100755 bin/process_main_aggregate.sh diff --git a/bin/process_main_aggregate.sh b/bin/process_main_aggregate.sh new file mode 100755 index 0000000..aef16d0 --- /dev/null +++ b/bin/process_main_aggregate.sh @@ -0,0 +1,264 @@ +#!/bin/bash + +####################################################################### +# Copyright 2017 Internet2 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +####################################################################### + +####################################################################### +# Help message +####################################################################### + +display_help () { +/bin/cat <<- HELP_MSG + This script produces the following files from the InCommon + main aggregate: + + $csv_filename1 + $csv_filename2 + + The script is intended to be run as a cron job. It can (and + should) be run often, to pick up the latest changes to metadata. + There is little penalty in doing so since the script uses HTTP + Conditional GET to retrieve metadata. + + Usage: ${0##*/} [-hv] -d OUT_DIR + + Options: + -h Display this help message + -v Enable DEBUG mode + -d Specify the output directory + + Option -h is mutually exclusive of all other options. + + Option -d specifies the ultimate output directory, which is + usually a web directory. This option is REQUIRED. + + ENVIRONMENT + + This script leverages numerous environment variables: + + BIN_DIR A directory of executable scripts + LIB_DIR A source library directory + CACHE_DIR An HTTP cache + TMPDIR A temporary directory + LOG_FILE A persistent log file + LOG_LEVEL The global log level [0..5] + + All of the above environment variables are REQUIRED + except LOG_LEVEL, which defaults to LOG_LEVEL=3. + + EXAMPLES + + \$ out_dir=/home/htdocs/www.incommonfederation.org/federation/metadata/ + \$ ${0##*/} -d \$out_dir +HELP_MSG +} + +####################################################################### +# Bootstrap +####################################################################### + +script_name=${0##*/} # equivalent to basename $0 + +# required directories +env_vars="BIN_DIR +LIB_DIR +CACHE_DIR +TMPDIR" + +# check required directories +for env_var in $env_vars; do + eval "env_var_val=\${$env_var}" + if [ -z "$env_var_val" ]; then + echo "ERROR: $script_name requires env var $env_var" >&2 + exit 2 + fi + if [ ! -d "$env_var_val" ]; then + echo "ERROR: $script_name: directory does not exist: $env_var_val" >&2 + exit 2 + fi +done + +# check the log file +if [ -z "$LOG_FILE" ]; then + echo "ERROR: $script_name requires env var LOG_FILE" >&2 + exit 2 +fi +# devices such as /dev/tty and /dev/null are allowed +if [ ! -f "$LOG_FILE" ] && [[ $LOG_FILE != /dev/* ]]; then + echo "ERROR: $script_name: file does not exist: $LOG_FILE" >&2 + exit 2 +fi + +# default to INFO logging +if [ -z "$LOG_LEVEL" ]; then + LOG_LEVEL=3 +fi + +# output filenames +csv_filename1="all-idps-rands.csv" +csv_filename2="all-sps-rands.csv" + +####################################################################### +# Process command-line options and arguments +####################################################################### + +help_mode=false; local_opts= +while getopts ":hvd:" opt; do + case $opt in + h) + help_mode=true + ;; + v) + LOG_LEVEL=4 + local_opts="$local_opts -$opt" + ;; + d) + out_dir="$OPTARG" + ;; + \?) + echo "ERROR: $script_name: Unrecognized option: -$OPTARG" >&2 + exit 2 + ;; + :) + echo "ERROR: $script_name: Option -$OPTARG requires an argument" >&2 + exit 2 + ;; + esac +done + +if $help_mode; then + display_help + exit 0 +fi + +# check the output directory +if [ ! -d "$out_dir" ]; then + echo "ERROR: $script_name: directory does not exist: $out_dir" >&2 + exit 2 +fi + +# check command-line arguments +shift $(( OPTIND - 1 )) +if [ $# -ne 0 ]; then + echo "ERROR: $script_name: wrong number of arguments: $# (0 required)" >&2 + exit 2 +fi + +####################################################################### +# Initialization +####################################################################### + +# executable script must exist +cget="$BIN_DIR/cget.sh" +if [ ! -f "$cget" ]; then + echo "ERROR: $script_name: executable file does not exist: $cget" >&2 + exit 2 +fi + +# source core library +lib_file="$LIB_DIR/core_lib.sh" +if [ ! -f "$lib_file" ]; then + echo "ERROR: $script_name: lib file does not exist: $lib_file" >&2 + exit 2 +fi +source "$lib_file" +status_code=$? +if [ $status_code -ne 0 ]; then + echo "ERROR: $script_name failed ($status_code) to source lib file $lib_file" >&2 + exit 2 +fi + +# XSL stylesheets must exist +xsl_file1=$LIB_DIR/list_all_RandS_IdPs_csv.xsl +if [ ! -f "$xsl_file1" ]; then + echo "ERROR: $script_name: lib file does not exist: $xsl_file1" >&2 + exit 2 +fi +xsl_file2=$LIB_DIR/list_all_RandS_SPs_csv.xsl +if [ ! -f "$xsl_file2" ]; then + echo "ERROR: $script_name: lib file does not exist: $xsl_file2" >&2 + exit 2 +fi + +# create a temporary subdirectory +tmp_dir="${TMPDIR%%/}/${script_name%%.*}_$$" +/bin/mkdir "$tmp_dir" +exit_code=$? +if [ $exit_code -ne 0 ]; then + echo "ERROR: $script_name failed ($exit_code) to create tmp dir $tmp_dir" >&2 + exit 2 +fi + +# input file +xml_file="${tmp_dir}/saml-metadata.xml" + +# output files +csv_file1="${tmp_dir}/$csv_filename1" +csv_file2="${tmp_dir}/$csv_filename2" + +####################################################################### +# Main processing +####################################################################### + +print_log_message -I "$script_name BEGIN" + +# XML metadata +md_location=http://md.incommon.org/InCommon/InCommon-metadata.xml +print_log_message -D "$script_name using metadata file: $md_location" + +# get a fresh metadata file +print_log_message -D "$script_name writing XML file: $xml_file" +$cget $local_opts -F $md_location > $xml_file +exit_code=$? +if [ $exit_code -eq 1 ]; then + # short-circuit if 304 response + print_log_message -I "$script_name END" + clean_up_and_exit -d "$tmp_dir" 0 +fi +if [ $exit_code -gt 1 ]; then + print_log_message -E "$script_name: cget.sh failed ($exit_code) on location: $md_location" + clean_up_and_exit -d "$tmp_dir" $exit_code +fi + +# create the first output file +print_log_message -D "$script_name writing output file: $csv_file1" +/usr/bin/xsltproc $xsl_file1 $xml_file > $csv_file1 +exit_code=$? +if [ $exit_code -ne 0 ]; then + print_log_message -E "$script_name: xsltproc failed ($exit_code) on stylesheet: $xsl_file1" + clean_up_and_exit -d "$tmp_dir" $exit_code +fi + +# create the second output file +print_log_message -D "$script_name writing output file: $csv_file2" +/usr/bin/xsltproc $xsl_file2 $xml_file > $csv_file2 +exit_code=$? +if [ $exit_code -ne 0 ]; then + print_log_message -E "$script_name: xsltproc failed ($exit_code) on stylesheet: $xsl_file2" + clean_up_and_exit -d "$tmp_dir" $exit_code +fi + +# move the output files to the web directory +print_log_message -D "$script_name moving output files to dir: $out_dir" +/bin/mv $csv_file1 $csv_file2 $out_dir +exit_code=$? +if [ $exit_code -ne 0 ]; then + print_log_message -E "$script_name: mv failed ($exit_code) to dir: $out_dir" + clean_up_and_exit -d "$tmp_dir" $exit_code +fi + +print_log_message -I "$script_name END" +clean_up_and_exit -d "$tmp_dir" 0 diff --git a/install.sh b/install.sh index 7a84183..102cb02 100755 --- a/install.sh +++ b/install.sh @@ -93,7 +93,8 @@ while read script_file; do fi done <