Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Browse files
Browse the repository at this point in the history
Commit new shell script
Tom Scavo
committed
Jun 7, 2017
1 parent
c3e789c
commit 07da7bb
Showing
2 changed files
with
414 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,413 @@ | ||
| #!/bin/bash | ||
|
|
||
| ####################################################################### | ||
| # Copyright 2017 Internet2 | ||
| # | ||
| # Licensed under the Apache License, Version 2.0 (the "License"); | ||
| # you may not use this file except in compliance with the License. | ||
| # You may obtain a copy of the License at | ||
| # | ||
| # http://www.apache.org/licenses/LICENSE-2.0 | ||
| # | ||
| # Unless required by applicable law or agreed to in writing, software | ||
| # distributed under the License is distributed on an "AS IS" BASIS, | ||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| # See the License for the specific language governing permissions and | ||
| # limitations under the License. | ||
| ####################################################################### | ||
|
|
||
| ####################################################################### | ||
| # Help message | ||
| ####################################################################### | ||
|
|
||
| display_help () { | ||
| /bin/cat <<- HELP_MSG | ||
| This script parses one or more SAML metadata aggregates and | ||
| produces a JSON file of vital statistics (such as validUntil, | ||
| creationInstant, etc.) | ||
| The script depends on cached metadata. It will not fetch | ||
| a metadata file from the server. | ||
| Usage: ${0##*/} [-hv] -d OUT_DIR MD_LOCATION ... | ||
| The script takes one or more metadata locations on the | ||
| command line. For each location, the corresponding metadata | ||
| is read from cache and parsed. The script produces a JSON | ||
| array, with one array element for each metadata location. | ||
| If successful, the resulting JSON file is finally moved | ||
| to the output directory specified on the command line. | ||
| Options: | ||
| -h Display this help message | ||
| -v Enable DEBUG mode | ||
| -d Specify the output directory | ||
| Option -h is mutually exclusive of all other options. | ||
| Option -d specifies the ultimate output directory, which is | ||
| usually a web directory. This option is REQUIRED. | ||
| ENVIRONMENT | ||
| This script leverages a handful of environment variables: | ||
| LIB_DIR A source library directory | ||
| CACHE_DIR A persistent HTTP cache | ||
| TMPDIR A temporary directory | ||
| LOG_FILE A persistent log file | ||
| LOG_LEVEL The global log level [0..5] | ||
| All of the above environment variables are REQUIRED | ||
| except LOG_LEVEL, which defaults to LOG_LEVEL=3. | ||
| The following environment variables are REQUIRED: | ||
| $( printf " %s\n" ${env_vars[*]} ) | ||
| The following directories will be used: | ||
| $( printf " %s\n" ${dir_paths[*]} ) | ||
| The following log file will be used: | ||
| $( printf " %s\n" $LOG_FILE ) | ||
| INSTALLATION | ||
| At least the following source library files MUST be installed | ||
| in LIB_DIR: | ||
| $( printf " %s\n" ${lib_filenames[*]} ) | ||
| Also, the following XSLT file MUST be installed in LIB_DIR: | ||
| $xsl_filename | ||
| OUTPUT | ||
| The script outputs a JSON file containing a single array. | ||
| Each array element is a JavaScript object with the following | ||
| fields: | ||
| metadataLocation: location | ||
| currentTime: dateTime | ||
| validUntil: dateTime | ||
| creationInstant: dateTime | ||
| validityInterval: duration | ||
| untilInvalid: duration | ||
| sinceCreation: duration | ||
| For example: | ||
| { | ||
| "metadataLocation": "http://md.incommon.org/InCommon/InCommon-metadata.xml", | ||
| "currentTime": "2017-06-06T23:57:54Z", | ||
| "validUntil": "2017-06-16T18:41:12Z", | ||
| "creationInstant": "2017-06-02T18:41:12Z", | ||
| "validityInterval": "P14DT0H0M0S", | ||
| "untilInvalid": "P9DT18H43M18S", | ||
| "sinceCreation": "P4DT5H16M42S" | ||
| } | ||
| EXAMPLES | ||
| \$ ${0##*/} -h | ||
| \$ md_locations="http://md.incommon.org/InCommon/InCommon-metadata.xml | ||
| > http://md.incommon.org/InCommon/InCommon-metadata-export.xml" | ||
| \$ out_dir=/home/htdocs/www.incommonfederation.org/federation/metadata/ | ||
| \$ ${0##*/} -d \$out_dir \$md_locations | ||
| HELP_MSG | ||
| } | ||
|
|
||
| ####################################################################### | ||
| # Bootstrap | ||
| ####################################################################### | ||
|
|
||
| script_name=${0##*/} # equivalent to basename $0 | ||
|
|
||
| # required environment variables | ||
| env_vars[1]="LIB_DIR" | ||
| env_vars[2]="CACHE_DIR" | ||
| env_vars[3]="TMPDIR" | ||
| env_vars[4]="LOG_FILE" | ||
|
|
||
| # check environment variables | ||
| for env_var in ${env_vars[*]}; do | ||
| eval "env_var_val=\${$env_var}" | ||
| if [ -z "$env_var_val" ]; then | ||
| echo "ERROR: $script_name requires env var $env_var" >&2 | ||
| exit 2 | ||
| fi | ||
| done | ||
|
|
||
| # required directories | ||
| dir_paths[1]="$LIB_DIR" | ||
| dir_paths[2]="$CACHE_DIR" | ||
| dir_paths[3]="$TMPDIR" | ||
|
|
||
| # check required directories | ||
| for dir_path in ${dir_paths[*]}; do | ||
| if [ ! -d "$dir_path" ]; then | ||
| echo "ERROR: $script_name: directory does not exist: $dir_path" >&2 | ||
| exit 2 | ||
| fi | ||
| done | ||
|
|
||
| # check the log file | ||
| # devices such as /dev/tty and /dev/null are allowed | ||
| if [ ! -f "$LOG_FILE" ] && [[ $LOG_FILE != /dev/* ]]; then | ||
| echo "ERROR: $script_name: file does not exist: $LOG_FILE" >&2 | ||
| exit 2 | ||
| fi | ||
|
|
||
| # default to INFO logging | ||
| if [ -z "$LOG_LEVEL" ]; then | ||
| LOG_LEVEL=3 | ||
| fi | ||
|
|
||
| # library filenames | ||
| lib_filenames[1]="core_lib.sh" | ||
| lib_filenames[2]="http_tools.sh" | ||
| lib_filenames[3]="compatible_date.sh" | ||
|
|
||
| # check lib files | ||
| for lib_filename in ${lib_filenames[*]}; do | ||
| lib_file="$LIB_DIR/$lib_filename" | ||
| if [ ! -f "$lib_file" ]; then | ||
| echo "ERROR: $script_name: file does not exist: $lib_file" >&2 | ||
| exit 2 | ||
| fi | ||
| done | ||
|
|
||
| # XSLT script | ||
| xsl_filename="entities_timestamps_txt.xsl" | ||
|
|
||
| # check XSLT script | ||
| xsl_path="$LIB_DIR/$xsl_filename" | ||
| if [ ! -f "$xsl_path" ]; then | ||
| echo "ERROR: $script_name: file does not exist: $xsl_path" >&2 | ||
| exit 2 | ||
| fi | ||
|
|
||
| ####################################################################### | ||
| # Process command-line options and arguments | ||
| ####################################################################### | ||
|
|
||
| help_mode=false; local_opts= | ||
| while getopts ":hvd:" opt; do | ||
| case $opt in | ||
| h) | ||
| help_mode=true | ||
| ;; | ||
| v) | ||
| LOG_LEVEL=4 | ||
| local_opts="$local_opts -$opt" | ||
| ;; | ||
| d) | ||
| out_dir="$OPTARG" | ||
| ;; | ||
| \?) | ||
| echo "ERROR: $script_name: Unrecognized option: -$OPTARG" >&2 | ||
| exit 2 | ||
| ;; | ||
| :) | ||
| echo "ERROR: $script_name: Option -$OPTARG requires an argument" >&2 | ||
| exit 2 | ||
| ;; | ||
| esac | ||
| done | ||
|
|
||
| if $help_mode; then | ||
| display_help | ||
| exit 0 | ||
| fi | ||
|
|
||
| # check the output directory | ||
| if [ -z "$out_dir" ]; then | ||
| echo "ERROR: $script_name: no output directory specified (option -d)" >&2 | ||
| exit 2 | ||
| fi | ||
| if [ ! -d "$out_dir" ]; then | ||
| echo "ERROR: $script_name: directory does not exist: $out_dir" >&2 | ||
| exit 2 | ||
| fi | ||
|
|
||
| # at least one metadata location is required | ||
| shift $(( OPTIND - 1 )) | ||
| if [ $# -lt 1 ]; then | ||
| echo "ERROR: $script_name: wrong number of arguments: $# (at least 1 required)" >&2 | ||
| exit 2 | ||
| fi | ||
|
|
||
| ####################################################################### | ||
| # Initialization | ||
| ####################################################################### | ||
|
|
||
| # source lib files | ||
| for lib_filename in ${lib_filenames[*]}; do | ||
| lib_file="$LIB_DIR/$lib_filename" | ||
| source "$lib_file" | ||
| status_code=$? | ||
| if [ $status_code -ne 0 ]; then | ||
| echo "ERROR: $script_name failed ($status_code) to source lib file $lib_file" >&2 | ||
| exit 2 | ||
| fi | ||
| done | ||
|
|
||
| # create a temporary subdirectory | ||
| tmp_dir="${TMPDIR%%/}/${script_name%%.*}_$$" | ||
| /bin/mkdir "$tmp_dir" | ||
| status_code=$? | ||
| if [ $status_code -ne 0 ]; then | ||
| echo "ERROR: $script_name failed ($status_code) to create tmp dir $tmp_dir" >&2 | ||
| exit 2 | ||
| fi | ||
|
|
||
| # specify temporary files | ||
| xml_path="${tmp_dir}/saml-metadata.xml" | ||
| json_out="${tmp_dir}/md_vital_statistics.json" | ||
|
|
||
| ####################################################################### | ||
| # Functions | ||
| ####################################################################### | ||
|
|
||
| escape_special_json_chars () { | ||
| local str="$1" | ||
|
|
||
| # backslash (\) and double quote (") are special | ||
| echo "$str" | $_SED -e 's/\\/\\\\/g' -e 's/"/\\"/g' | ||
| } | ||
|
|
||
| append_json_object () { | ||
| local metadataLocation=$( escape_special_json_chars "$md_location" ) | ||
| local currentTime=$( escape_special_json_chars "$now" ) | ||
| local validUntil=$( escape_special_json_chars "$validUntil" ) | ||
| local creationInstant=$( escape_special_json_chars "$creationInstant" ) | ||
| local validityInterval=$( escape_special_json_chars "$validityInterval" ) | ||
| local untilInvalid=$( escape_special_json_chars "$untilInvalid" ) | ||
| local sinceCreation=$( escape_special_json_chars "$sinceCreation" ) | ||
|
|
||
| print_log_message -I "$script_name creating JSON object for metadata file: $metadataLocation" | ||
|
|
||
| /bin/cat <<- JSON_OBJECT | ||
| { | ||
| "metadataLocation": "$metadataLocation", | ||
| "currentTime": "$currentTime", | ||
| "validUntil": "$validUntil", | ||
| "creationInstant": "$creationInstant", | ||
| "validityInterval": "$validityInterval", | ||
| "untilInvalid": "$untilInvalid", | ||
| "sinceCreation": "$sinceCreation" | ||
| } | ||
| JSON_OBJECT | ||
| } | ||
|
|
||
| parse_metadata () { | ||
|
|
||
| md_location="$1" | ||
|
|
||
| # get a cached metadata file | ||
| conditional_get $local_opts -C -d "$CACHE_DIR" -T "$tmp_dir" "$md_location" > "$xml_path" | ||
| status_code=$? | ||
| if [ $status_code -eq 1 ]; then | ||
| # metadata must be cached | ||
| print_log_message -E "$script_name: metadata file not cached: $md_location" | ||
| clean_up_and_exit -d "$tmp_dir" 1 | ||
| fi | ||
| if [ $status_code -gt 1 ]; then | ||
| print_log_message -E "$script_name: conditional_get failed ($status_code) on location: $md_location" | ||
| clean_up_and_exit -d "$tmp_dir" $status_code | ||
| fi | ||
| print_log_message -I "$script_name parsing cached metadata file: $md_location" | ||
|
|
||
| # extract @ID, @creationInstant, @validUntil (in that order) | ||
| tstamps=$( /usr/bin/xsltproc $xsl_path $xml_path ) | ||
|
|
||
| # If @validUntil missing, then FAIL | ||
| validUntil=$( echo "$tstamps" | $_CUT -f3 ) | ||
| if [ -z "$validUntil" ]; then | ||
| print_log_message -E "$script_name: @validUntil not found" | ||
| clean_up_and_exit -d "$tmp_dir" 4 | ||
| fi | ||
| print_log_message -D "$script_name found @validUntil: $validUntil" | ||
|
|
||
| # If @creationInstant missing, then FAIL | ||
| creationInstant=$( echo "$tstamps" | $_CUT -f2 ) | ||
| if [ -z "$creationInstant" ]; then | ||
| print_log_message -E "$script_name: @creationInstant not found" | ||
| clean_up_and_exit -d "$tmp_dir" 5 | ||
| fi | ||
| print_log_message -D "$script_name found @creationInstant: $creationInstant" | ||
|
|
||
| # If validityInterval > 14 days, then FAIL | ||
| validityIntervalSecs=$( secsUntil -b $creationInstant $validUntil ) | ||
| if (( validityIntervalSecs > 14*24*60*60 )); then | ||
| print_log_message -E "$script_name: validity interval exceeds maximum: $validityIntervalSecs" | ||
| clean_up_and_exit -d "$tmp_dir" 6 | ||
| fi | ||
| validityInterval=$( secs2duration $validityIntervalSecs ) | ||
| print_log_message -D "$script_name computed validity interval: $validityInterval" | ||
|
|
||
| # compute NOW | ||
| now=$( dateTime_now_canonical ) | ||
|
|
||
| # If secsUntilInvalid <= 0, then FAIL | ||
| secsUntilInvalid=$( echo $validUntil | secsUntil -b $now ) | ||
| if (( secsUntilInvalid <= 0 )); then | ||
| print_log_message -E "$script_name: seconds until invalid not positive: $secsUntilInvalid" | ||
| clean_up_and_exit -d "$tmp_dir" 7 | ||
| fi | ||
| untilInvalid=$( secs2duration "$secsUntilInvalid" ) | ||
| print_log_message -D "$script_name computed time until invalid: $untilInvalid" | ||
|
|
||
| # If secsSinceCreation <= 0, then FAIL | ||
| secsSinceCreation=$( echo $creationInstant | secsSince -e $now ) | ||
| if (( secsSinceCreation <= 0 )); then | ||
| print_log_message -E "$script_name: seconds since creation not positive: $secsSinceCreation" | ||
| clean_up_and_exit -d "$tmp_dir" 8 | ||
| fi | ||
| sinceCreation=$( secs2duration "$secsSinceCreation" ) | ||
| print_log_message -D "$script_name computed time since creation: $sinceCreation" | ||
| } | ||
|
|
||
| print_vital_statistics () { | ||
|
|
||
| # begin output list | ||
| printf "[\n" | ||
|
|
||
| while true; do | ||
|
|
||
| parse_metadata "$1" | ||
| append_json_object | ||
|
|
||
| shift; (( "$#" )) || break | ||
|
|
||
| # print comma separator | ||
| printf " ,\n" | ||
| done | ||
|
|
||
| # end output list | ||
| printf "]\n" | ||
| } | ||
|
|
||
| ####################################################################### | ||
| # Main processing | ||
| ####################################################################### | ||
|
|
||
| print_log_message -I "$script_name BEGIN" | ||
|
|
||
| # create the JSON output | ||
| print_vital_statistics "$@" > "$json_out" | ||
|
|
||
| # move the output file to the web directory | ||
| print_log_message -I "$script_name moving output file to dir: $out_dir" | ||
| /bin/mv "$json_out" $out_dir | ||
| status_code=$? | ||
| if [ $status_code -ne 0 ]; then | ||
| print_log_message -E "$script_name: mv failed ($status_code) to dir: $out_dir" | ||
| clean_up_and_exit -d "$tmp_dir" $status_code | ||
| fi | ||
|
|
||
| print_log_message -I "$script_name END" | ||
| clean_up_and_exit -d "$tmp_dir" 0 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters