From 07da7bb350911635f1d3a2d2ad52d05db85c4152 Mon Sep 17 00:00:00 2001
From: Tom Scavo <trscavo@internet2.edu>
Date: Wed, 7 Jun 2017 15:03:59 -0400
Subject: [PATCH] Commit new shell script
---
bin/compute_vital_statistics.sh | 413 ++++++++++++++++++++++++++++++++
install.sh | 1 +
2 files changed, 414 insertions(+)
create mode 100755 bin/compute_vital_statistics.sh
diff --git a/bin/compute_vital_statistics.sh b/bin/compute_vital_statistics.sh
new file mode 100755
index 0000000..2e25b8a
--- /dev/null
+++ b/bin/compute_vital_statistics.sh
@@ -0,0 +1,413 @@
+#!/bin/bash
+
+#######################################################################
+# Copyright 2017 Internet2
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#######################################################################
+
+#######################################################################
+# Help message
+#######################################################################
+
+display_help () {
+/bin/cat <<- HELP_MSG
+ This script parses one or more SAML metadata aggregates and
+ produces a JSON file of vital statistics (such as validUntil,
+ creationInstant, etc.)
+
+ The script depends on cached metadata. It will not fetch
+ a metadata file from the server.
+
+ Usage: ${0##*/} [-hv] -d OUT_DIR MD_LOCATION ...
+
+ The script takes one or more metadata locations on the
+ command line. For each location, the corresponding metadata
+ is read from cache and parsed. The script produces a JSON
+ array, with one array element for each metadata location.
+ If successful, the resulting JSON file is finally moved
+ to the output directory specified on the command line.
+
+ Options:
+ -h Display this help message
+ -v Enable DEBUG mode
+ -d Specify the output directory
+
+ Option -h is mutually exclusive of all other options.
+
+ Option -d specifies the ultimate output directory, which is
+ usually a web directory. This option is REQUIRED.
+
+ ENVIRONMENT
+
+ This script leverages a handful of environment variables:
+
+ LIB_DIR A source library directory
+ CACHE_DIR A persistent HTTP cache
+ TMPDIR A temporary directory
+ LOG_FILE A persistent log file
+ LOG_LEVEL The global log level [0..5]
+
+ All of the above environment variables are REQUIRED
+ except LOG_LEVEL, which defaults to LOG_LEVEL=3.
+
+ The following environment variables are REQUIRED:
+
+ $( printf " %s\n" ${env_vars[*]} )
+
+ The following directories will be used:
+
+ $( printf " %s\n" ${dir_paths[*]} )
+
+ The following log file will be used:
+
+ $( printf " %s\n" $LOG_FILE )
+
+ INSTALLATION
+
+ At least the following source library files MUST be installed
+ in LIB_DIR:
+
+ $( printf " %s\n" ${lib_filenames[*]} )
+
+ Also, the following XSLT file MUST be installed in LIB_DIR:
+
+ $xsl_filename
+
+ OUTPUT
+
+ The script outputs a JSON file containing a single array.
+ Each array element is a JavaScript object with the following
+ fields:
+
+ metadataLocation: location
+ currentTime: dateTime
+ validUntil: dateTime
+ creationInstant: dateTime
+ validityInterval: duration
+ untilInvalid: duration
+ sinceCreation: duration
+
+ For example:
+
+ {
+ "metadataLocation": "http://md.incommon.org/InCommon/InCommon-metadata.xml",
+ "currentTime": "2017-06-06T23:57:54Z",
+ "validUntil": "2017-06-16T18:41:12Z",
+ "creationInstant": "2017-06-02T18:41:12Z",
+ "validityInterval": "P14DT0H0M0S",
+ "untilInvalid": "P9DT18H43M18S",
+ "sinceCreation": "P4DT5H16M42S"
+ }
+
+ EXAMPLES
+
+ \$ ${0##*/} -h
+ \$ md_locations="http://md.incommon.org/InCommon/InCommon-metadata.xml
+ > http://md.incommon.org/InCommon/InCommon-metadata-export.xml"
+ \$ out_dir=/home/htdocs/www.incommonfederation.org/federation/metadata/
+ \$ ${0##*/} -d \$out_dir \$md_locations
+HELP_MSG
+}
+
+#######################################################################
+# Bootstrap
+#######################################################################
+
+script_name=${0##*/} # equivalent to basename $0
+
+# required environment variables
+env_vars[1]="LIB_DIR"
+env_vars[2]="CACHE_DIR"
+env_vars[3]="TMPDIR"
+env_vars[4]="LOG_FILE"
+
+# check environment variables
+for env_var in ${env_vars[*]}; do
+ eval "env_var_val=\${$env_var}"
+ if [ -z "$env_var_val" ]; then
+ echo "ERROR: $script_name requires env var $env_var" >&2
+ exit 2
+ fi
+done
+
+# required directories
+dir_paths[1]="$LIB_DIR"
+dir_paths[2]="$CACHE_DIR"
+dir_paths[3]="$TMPDIR"
+
+# check required directories
+for dir_path in ${dir_paths[*]}; do
+ if [ ! -d "$dir_path" ]; then
+ echo "ERROR: $script_name: directory does not exist: $dir_path" >&2
+ exit 2
+ fi
+done
+
+# check the log file
+# devices such as /dev/tty and /dev/null are allowed
+if [ ! -f "$LOG_FILE" ] && [[ $LOG_FILE != /dev/* ]]; then
+ echo "ERROR: $script_name: file does not exist: $LOG_FILE" >&2
+ exit 2
+fi
+
+# default to INFO logging
+if [ -z "$LOG_LEVEL" ]; then
+ LOG_LEVEL=3
+fi
+
+# library filenames
+lib_filenames[1]="core_lib.sh"
+lib_filenames[2]="http_tools.sh"
+lib_filenames[3]="compatible_date.sh"
+
+# check lib files
+for lib_filename in ${lib_filenames[*]}; do
+ lib_file="$LIB_DIR/$lib_filename"
+ if [ ! -f "$lib_file" ]; then
+ echo "ERROR: $script_name: file does not exist: $lib_file" >&2
+ exit 2
+ fi
+done
+
+# XSLT script
+xsl_filename="entities_timestamps_txt.xsl"
+
+# check XSLT script
+xsl_path="$LIB_DIR/$xsl_filename"
+if [ ! -f "$xsl_path" ]; then
+ echo "ERROR: $script_name: file does not exist: $xsl_path" >&2
+ exit 2
+fi
+
+#######################################################################
+# Process command-line options and arguments
+#######################################################################
+
+help_mode=false; local_opts=
+while getopts ":hvd:" opt; do
+ case $opt in
+ h)
+ help_mode=true
+ ;;
+ v)
+ LOG_LEVEL=4
+ local_opts="$local_opts -$opt"
+ ;;
+ d)
+ out_dir="$OPTARG"
+ ;;
+ \?)
+ echo "ERROR: $script_name: Unrecognized option: -$OPTARG" >&2
+ exit 2
+ ;;
+ :)
+ echo "ERROR: $script_name: Option -$OPTARG requires an argument" >&2
+ exit 2
+ ;;
+ esac
+done
+
+if $help_mode; then
+ display_help
+ exit 0
+fi
+
+# check the output directory
+if [ -z "$out_dir" ]; then
+ echo "ERROR: $script_name: no output directory specified (option -d)" >&2
+ exit 2
+fi
+if [ ! -d "$out_dir" ]; then
+ echo "ERROR: $script_name: directory does not exist: $out_dir" >&2
+ exit 2
+fi
+
+# at least one metadata location is required
+shift $(( OPTIND - 1 ))
+if [ $# -lt 1 ]; then
+ echo "ERROR: $script_name: wrong number of arguments: $# (at least 1 required)" >&2
+ exit 2
+fi
+
+#######################################################################
+# Initialization
+#######################################################################
+
+# source lib files
+for lib_filename in ${lib_filenames[*]}; do
+ lib_file="$LIB_DIR/$lib_filename"
+ source "$lib_file"
+ status_code=$?
+ if [ $status_code -ne 0 ]; then
+ echo "ERROR: $script_name failed ($status_code) to source lib file $lib_file" >&2
+ exit 2
+ fi
+done
+
+# create a temporary subdirectory
+tmp_dir="${TMPDIR%%/}/${script_name%%.*}_$$"
+/bin/mkdir "$tmp_dir"
+status_code=$?
+if [ $status_code -ne 0 ]; then
+ echo "ERROR: $script_name failed ($status_code) to create tmp dir $tmp_dir" >&2
+ exit 2
+fi
+
+# specify temporary files
+xml_path="${tmp_dir}/saml-metadata.xml"
+json_out="${tmp_dir}/md_vital_statistics.json"
+
+#######################################################################
+# Functions
+#######################################################################
+
+escape_special_json_chars () {
+ local str="$1"
+
+ # backslash (\) and double quote (") are special
+ echo "$str" | $_SED -e 's/\\/\\\\/g' -e 's/"/\\"/g'
+}
+
+append_json_object () {
+ local metadataLocation=$( escape_special_json_chars "$md_location" )
+ local currentTime=$( escape_special_json_chars "$now" )
+ local validUntil=$( escape_special_json_chars "$validUntil" )
+ local creationInstant=$( escape_special_json_chars "$creationInstant" )
+ local validityInterval=$( escape_special_json_chars "$validityInterval" )
+ local untilInvalid=$( escape_special_json_chars "$untilInvalid" )
+ local sinceCreation=$( escape_special_json_chars "$sinceCreation" )
+
+ print_log_message -I "$script_name creating JSON object for metadata file: $metadataLocation"
+
+ /bin/cat <<- JSON_OBJECT
+ {
+ "metadataLocation": "$metadataLocation",
+ "currentTime": "$currentTime",
+ "validUntil": "$validUntil",
+ "creationInstant": "$creationInstant",
+ "validityInterval": "$validityInterval",
+ "untilInvalid": "$untilInvalid",
+ "sinceCreation": "$sinceCreation"
+ }
+JSON_OBJECT
+}
+
+parse_metadata () {
+
+ md_location="$1"
+
+ # get a cached metadata file
+ conditional_get $local_opts -C -d "$CACHE_DIR" -T "$tmp_dir" "$md_location" > "$xml_path"
+ status_code=$?
+ if [ $status_code -eq 1 ]; then
+ # metadata must be cached
+ print_log_message -E "$script_name: metadata file not cached: $md_location"
+ clean_up_and_exit -d "$tmp_dir" 1
+ fi
+ if [ $status_code -gt 1 ]; then
+ print_log_message -E "$script_name: conditional_get failed ($status_code) on location: $md_location"
+ clean_up_and_exit -d "$tmp_dir" $status_code
+ fi
+ print_log_message -I "$script_name parsing cached metadata file: $md_location"
+
+ # extract @ID, @creationInstant, @validUntil (in that order)
+ tstamps=$( /usr/bin/xsltproc $xsl_path $xml_path )
+
+ # If @validUntil missing, then FAIL
+ validUntil=$( echo "$tstamps" | $_CUT -f3 )
+ if [ -z "$validUntil" ]; then
+ print_log_message -E "$script_name: @validUntil not found"
+ clean_up_and_exit -d "$tmp_dir" 4
+ fi
+ print_log_message -D "$script_name found @validUntil: $validUntil"
+
+ # If @creationInstant missing, then FAIL
+ creationInstant=$( echo "$tstamps" | $_CUT -f2 )
+ if [ -z "$creationInstant" ]; then
+ print_log_message -E "$script_name: @creationInstant not found"
+ clean_up_and_exit -d "$tmp_dir" 5
+ fi
+ print_log_message -D "$script_name found @creationInstant: $creationInstant"
+
+ # If validityInterval > 14 days, then FAIL
+ validityIntervalSecs=$( secsUntil -b $creationInstant $validUntil )
+ if (( validityIntervalSecs > 14*24*60*60 )); then
+ print_log_message -E "$script_name: validity interval exceeds maximum: $validityIntervalSecs"
+ clean_up_and_exit -d "$tmp_dir" 6
+ fi
+ validityInterval=$( secs2duration $validityIntervalSecs )
+ print_log_message -D "$script_name computed validity interval: $validityInterval"
+
+ # compute NOW
+ now=$( dateTime_now_canonical )
+
+ # If secsUntilInvalid <= 0, then FAIL
+ secsUntilInvalid=$( echo $validUntil | secsUntil -b $now )
+ if (( secsUntilInvalid <= 0 )); then
+ print_log_message -E "$script_name: seconds until invalid not positive: $secsUntilInvalid"
+ clean_up_and_exit -d "$tmp_dir" 7
+ fi
+ untilInvalid=$( secs2duration "$secsUntilInvalid" )
+ print_log_message -D "$script_name computed time until invalid: $untilInvalid"
+
+ # If secsSinceCreation <= 0, then FAIL
+ secsSinceCreation=$( echo $creationInstant | secsSince -e $now )
+ if (( secsSinceCreation <= 0 )); then
+ print_log_message -E "$script_name: seconds since creation not positive: $secsSinceCreation"
+ clean_up_and_exit -d "$tmp_dir" 8
+ fi
+ sinceCreation=$( secs2duration "$secsSinceCreation" )
+ print_log_message -D "$script_name computed time since creation: $sinceCreation"
+}
+
+print_vital_statistics () {
+
+ # begin output list
+ printf "[\n"
+
+ while true; do
+
+ parse_metadata "$1"
+ append_json_object
+
+ shift; (( "$#" )) || break
+
+ # print comma separator
+ printf " ,\n"
+ done
+
+ # end output list
+ printf "]\n"
+}
+
+#######################################################################
+# Main processing
+#######################################################################
+
+print_log_message -I "$script_name BEGIN"
+
+# create the JSON output
+print_vital_statistics "$@" > "$json_out"
+
+# move the output file to the web directory
+print_log_message -I "$script_name moving output file to dir: $out_dir"
+/bin/mv "$json_out" $out_dir
+status_code=$?
+if [ $status_code -ne 0 ]; then
+ print_log_message -E "$script_name: mv failed ($status_code) to dir: $out_dir"
+ clean_up_and_exit -d "$tmp_dir" $status_code
+fi
+
+print_log_message -I "$script_name END"
+clean_up_and_exit -d "$tmp_dir" 0
diff --git a/install.sh b/install.sh
index 73cb0f1..386e1c6 100755
--- a/install.sh
+++ b/install.sh
@@ -93,6 +93,7 @@ while read script_file; do
fi
done <<SCRIPTS
$script_bin/bin/cget.sh
+$script_bin/bin/compute_vital_statistics.sh
SCRIPTS
# initialize lib dir