From 6fb46b3c7b9188d0ba9358878c514e0f51283a59 Mon Sep 17 00:00:00 2001 From: Tom Scavo Date: Mon, 7 Nov 2016 14:27:30 -0500 Subject: [PATCH] Initial commit --- bin/check_idp_error_urls.sh | 597 +++++++++++++++++++++++++ bin/list_local_idp_error_urls.sh | 355 +++++++++++++++ install.sh | 115 +++++ lib/extract_IdP_entityIDs.xsl | 41 ++ lib/extract_IdP_names.xsl | 51 +++ lib/extract_InCommon_IdP_entityIDs.xsl | 43 ++ 6 files changed, 1202 insertions(+) create mode 100755 bin/check_idp_error_urls.sh create mode 100755 bin/list_local_idp_error_urls.sh create mode 100755 install.sh create mode 100644 lib/extract_IdP_entityIDs.xsl create mode 100644 lib/extract_IdP_names.xsl create mode 100644 lib/extract_InCommon_IdP_entityIDs.xsl diff --git a/bin/check_idp_error_urls.sh b/bin/check_idp_error_urls.sh new file mode 100755 index 0000000..3b4e6cc --- /dev/null +++ b/bin/check_idp_error_urls.sh @@ -0,0 +1,597 @@ +#!/bin/bash + +####################################################################### +# Copyright 2015--2016 Internet2 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +####################################################################### + +script_version="0.3" +user_agent_string="Check IdP Error URLs ${script_version}" + +####################################################################### +# help message +####################################################################### + +display_help () { +/bin/cat <<- HELP_MSG + ${user_agent_string} + + Given a list of entityIDs and a metadata source, for each IdP + probe its errorURL in metadata (if any). + + Usage: ${0##*/} [-hvq] [-t CONNECT_TIME -m MAX_TIME] (-u MDQ_BASE_URL | -f MD_PATH) [-d OUT_DIR] [ID ...] + + The script optionally takes a sequence of identifiers on the command + line. If none are given, the script takes its input from stdin. + + Options: + -h Display this message + -v Write verbose messages to stdout + -q Run quietly (i.e., write no messages to stdout/stderr) + -t Time (in secs) allotted to connect to the host + -m Maximum time (in secs) allotted to a complete probe + -u Base URL of a Metadata Query Server + -f Path to a local metadata file + -d Path to an output directory + + Option -h is mutually exclusive of all other options. Options + -q and -v are mutually exclusive of each other. Options -u and -f + are mutually exclusive of each other as well. Option -d is required + if file output is desired. + + The argument of the -t option is the TCP connect time, that is, + the maximum time (in secs) allotted to the TCP connection. Note + that the TCP connect time includes the time it takes to do a + DNS name lookup. Since the latter is unconstrained, it may + consume all available TCP connect time. Thus the TCP connect + time should be kept small (say, less than 10 seconds) since + larger values will slow this script considerably. + + The argument of the -m option is the maximum total time (in secs) + allotted to each probe. A reasonable value is a few seconds + beyond the TCP connect time. Any value less than the TCP connect + time causes the script to immediately fail. + + Entity metadata is required to process each identifier. Metadata is + obtained in one of two ways, by consulting a Metadata Query Server + just-in-time or by using a suitably comprehensive metadata aggregate + (perhaps the same aggregate from which the entityIDs were taken). + These correspond to options -u and -f, respectively. Exactly one of + these options is required. + + Option -f takes an optional file argument (MD_PATH), the absolute + path to a local SAML metadata file. The script searches this file for + a corresponding entity descriptor as it processes each identifier. + + Option -u takes an optional URI argument (MDQ_BASE_URL), the base + URL of a Metadata Query Server (i.e., a server that conforms to the + Metadata Query Protocol). The base URL is used to construct an MDQ + request URL, which the script uses to request entity metadata + just-in-time. + + LIBRARY + + Environment variable LIB_DIR specifies a directory containing at + least the following library files, which act as helper scripts for + ${0##*/}: + + $LIB_FILENAMES + + STANDARD OUTPUT + + By default, the script outputs an abbreviated log to stdout (but + this may be suppressed by use of the -q option). A line of + standard output has the following space-delimited fields: + + 1) code: a curl exit code + 2) output: a curl output string + 3) errorURL: the actual errorURL in metadata + + See the curl man page (http://linux.die.net/man/1/curl) for a + brief description of possible exit codes. + + The output string has the following format: + + redirects:9;response:999;dns:9.999;tcp:9.999;ssl:9.999;total:9.999 + + The redirects in the output string are the number of HTTP redirects + followed by this script. The response is the ultimate HTTP response + code. If the probe does not complete, the HTTP response will be 000 + by convention. The remaining four values in the output string are + times (in secs) computed by curl: + + dns is the elapsed time up to and including the DNS lookup + (curl time_namelookup variable) + tcp is the elapsed time up to and including the TCP connection + (curl time_connect variable) + ssl is the elapsed time up to and including the SSL exchange + (curl time_appconnect variable) (only curl 7.19.0 and later) + total is the total elapsed time of the probe + (curl time_total variable) + + See the curl man page (curl --write-out option) for detailed + explanations of these timings. + + By definition, a probe succeeds if its exit code is 0. For our + purposes, a probe completely fails if its exit code is either 6 + or 7. (Exit code 6 indicates a DNS lookup failure while code 7 + means the host is unreachable on the network.) A probe that times + out (exit code 28) is labeled as nonresponsive. All other exit codes + are regarded as indeterminate. + + The errorURL is the actual URL probed by this script. + + OUTPUT FILES + + The script writes a number of output files if (and only if) the + -d option is specified on the command line. The output files are + written to the given OUT_DIR. + + ${NO_ERROR_URL_FILENAME} + + A list of those IdPs in the input that do not expose an + errorURL in metadata. + + A line in the output file has the following space-delimited fields: + + 1) entityID: the entityID + 2) registrarID: the registrar ID + + The entityID is the name of the entity. An entityID is an arbitrary + URI, as given by the entityID XML attribute on the + element in SAML metadata. + + The registrarID is the name of the registrar that registered the IdP + metadata in the first place. By convention, a registrar ID is an + arbitrary URI, as given by the registrationAuthority XML attribute + on the extension element in SAML metadata. + Since the latter element is optional in metadata, this field may be + blank in the log file (which is why it is always the last field on + any given output line). + + ${ERROR_URL_LOG_FILENAME} + + A log of each probe. Each line records the result of the probe of + a single entity. A line in the log file has the following + space-delimited fields: + + 1) code: a curl exit code + 2) output: a curl output string + 3) errorURL: the errorURL in metadata + 4) entityID: the entityID + 5) registrarID: the registrar ID + + The code, output, and errorURL fields are the same as those + printed to stdout. + + The entityID and registrarID fields are the same as in the + previous output file. An entity that ends up in this output file + was determined by the script to have an errorURL in metadata. + + ${ENTITY_NAMES_FILENAME} + + The names associated with every entity referenced on the command + line. A line in the log file has the following tab-delimited fields: + + 1) entityID: the entityID + 2) displayName: the display name of the entity + 3) organizationName: the name of the organization that owns the entity + 4) registrarID: the registrar ID + + The displayName and the organizationName are the mdui:DisplayName + and md:OrganizationName in metadata. The english language version + (xml:lang="en") is used in each case. If there is no such element + in metadata, the field is left blank. + + Examples: ${0##*/} -h + ${0##*/} -t ${connect_timeout_default} -m ${max_time_default} \$id + cat \$id_file | ${0##*/} -v -t 4 -m 6 + ${0##*/} -q -f /path/to/md_file.xml \$id1 \$id2 \$id3 + + Note that the second example above is the same as no options at all. +HELP_MSG +} + +####################################################################### +# Bootstrap +####################################################################### + +script_bin=${0%/*} # equivalent to dirname $0 +script_name=${0##*/} # equivalent to basename $0 + +connect_timeout_default=2 +max_time_default=4 + +# library filenames (always list command_paths first) +LIB_FILENAMES="command_paths.sh +compatible_mktemp.sh +md_tools.sh" + +# output filenames +NO_IDP_ROLE_FILENAME="entities-no-idp-role.txt" +NO_ERROR_URL_FILENAME="idps-no-error-url.txt" +ERROR_URL_LOG_FILENAME="idp-error-url-log.txt" +ENTITY_NAMES_FILENAME="idp-names.txt" +ERROR_LOG_FILENAME="error-log.txt" +COMPATIBILITY_SCRIPT_FILENAME="compatibility.sh" + +####################################################################### +# Process command-line options and arguments +####################################################################### + +help_mode=false; quiet_mode=false; verbose_mode=false +md_query_mode=false; md_file_mode=false +local_opts=; connect_timeout=; max_time= +while getopts ":hqvt:m:u:f:d:" opt; do + case $opt in + h) + help_mode=true + ;; + q) + quiet_mode=true + verbose_mode=false + #local_opts="$local_opts -$opt" + ;; + v) + quiet_mode=false + verbose_mode=true + local_opts="$local_opts -$opt" + ;; + t) + connect_timeout="$OPTARG" + ;; + m) + max_time="$OPTARG" + ;; + u) + md_query_mode=true + md_file_mode=false + mdq_base_url="$OPTARG" + ;; + f) + md_query_mode=false + md_file_mode=true + md_path="$OPTARG" + ;; + d) + OUT_DIR="$OPTARG" + ;; + \?) + echo "ERROR: $script_name: Unrecognized option: -$OPTARG" >&2 + exit 2 + ;; + :) + echo "ERROR: $script_name: Option -$OPTARG requires an argument" >&2 + exit 2 + ;; + esac +done + +if $help_mode; then + display_help + exit 0 +fi + +# determine the metadata source +if $md_query_mode; then + if [ -z "$mdq_base_url" ]; then + echo "ERROR: $script_name: option -u requires an argument" >&2 + exit 2 + fi + $verbose_mode && printf "$script_name using base URL: %s\n" "$mdq_base_url" +elif $md_file_mode; then + if [ -z "$md_path" ]; then + echo "ERROR: $script_name: option -f requires an argument" >&2 + exit 2 + fi + if [ ! -f "$md_path" ]; then + echo "ERROR: $script_name: file does not exist: $md_path" >&2 + exit 2 + fi + $verbose_mode && printf "$script_name using metadata file: %s\n" "$md_path" +else + echo "ERROR: $script_name: one of options -u or -f required" >&2 + exit 2 +fi + +# redirect stdout to the bit bucket (stderr below) +$quiet_mode && exec 1>/dev/null + +# determine the output directory +if [ -z "$OUT_DIR" ]; then + DO_NOT_PRINT_FILES=true + $verbose_mode && printf "$script_name not printing output files\n" + # redirect stderr to the bit bucket + $quiet_mode && exec 2>/dev/null +else + DO_NOT_PRINT_FILES=false + # create the dir if necessary + if [ -d "$OUT_DIR" ]; then + $verbose_mode && printf "$script_name using output dir: %s\n" "$OUT_DIR" + else + $verbose_mode && printf "$script_name creating output dir: %s\n" "$OUT_DIR" + /bin/mkdir "$OUT_DIR" + exit_status=$? + if [ $exit_status -ne 0 ]; then + echo "ERROR: $script_name failed to create dir: $OUT_DIR" >&2 + exit $exit_status + fi + fi + # redirect stderr to a file + $quiet_mode && exec 2>"$ERROR_LOG_FILE" +fi + +# check consistency of timeout options (both or neither are required) +if [ -z "$connect_timeout" -a -z "$max_time" ]; then + connect_timeout=$connect_timeout_default + max_time=$max_time_default +elif [ -n "$connect_timeout" -a -n "$max_time" ]; then + if [ ! "${connect_timeout}" -gt 0 ] ; then + echo "ERROR: $script_name: connect timeout must be a positive integer: ${connect_timeout}" >&2 + exit 2 + fi + if [ ! "${max_time}" -gt "${connect_timeout}" ]; then + echo "ERROR: $script_name: max time must be greater than the connect timeout: ${max_time}" >&2 + exit 2 + fi +else + echo "ERROR: $script_name: both (or neither) options -t and -m are required" >&2 + exit 2 +fi +if $verbose_mode; then + printf "$script_name using connect timeout: %d secs\n" $connect_timeout + printf "$script_name using max time: %d secs\n" $max_time +fi + +##################################################################### +# Initialization +##################################################################### + +# determine the source lib directory +if [ -z "$LIB_DIR" ]; then + echo "ERROR: $script_name requires env var LIB_DIR" >&2 + exit 2 +fi +if [ ! -d "$LIB_DIR" ]; then + echo "ERROR: $script_name: directory does not exist: $LIB_DIR" >&2 + exit 2 +fi +$verbose_mode && printf "$script_name using source lib directory: %s\n" "$LIB_DIR" + +# source lib files +for lib_filename in $LIB_FILENAMES; do + lib_file="$LIB_DIR/$lib_filename" + if [ ! -f "$lib_file" ]; then + echo "ERROR: $script_name: file does not exist: $lib_file" >&2 + exit 2 + fi + $verbose_mode && printf "$script_name sourcing lib file: %s\n" "$lib_file" + source "$lib_file" >&2 + exit_code=$? + if [ $exit_code -ne 0 ]; then + echo "ERROR: $script_name failed to source script $lib_file" >&2 + exit $exit_code + fi +done + +# determine temporary directory +if [ -n "$TMPDIR" ] && [ -d "$TMPDIR" ]; then + # use system temporary directory (remove trailing slash) + TMP_DIR="${TMPDIR%%/}/shib-idp-probe" + $verbose_mode && printf "$script_name using temp dir: %s\n" "$TMP_DIR" +else + # create temporary directory + tmp_dir="$( make_temp_file -d )" + if [ ! -d "$tmp_dir" ] ; then + printf "ERROR: $script_name unable to create temporary dir\n" >&2 + exit 2 + fi + # use temporary directory (remove trailing slash) + TMP_DIR="${tmp_dir%%/}/shib-idp-probe" + $verbose_mode && printf "$script_name creating temp dir: %s\n" "$TMP_DIR" +fi +if [ ! -d "$TMP_DIR" ]; then + /bin/mkdir "$TMP_DIR" + exit_status=$? + if [ $exit_status -ne 0 ]; then + echo "ERROR: $script_name failed to create dir: $TMP_DIR" >&2 + exit $exit_status + fi +fi + +# temp file TODO: store each response separately? +HTTP_RESPONSE_FILE="${TMP_DIR}/http_response.txt" + +# read the input into a temporary file +IN_FILE="${TMP_DIR}/tmp_entityids_in_$$.txt" +shift $(( OPTIND - 1 )) +if [ "$#" -gt 0 ]; then + # read input from the command line + while (( "$#" )); do + # copy command-line arg into the temp file + echo "$1" >> "$IN_FILE" + shift + done +else + # read input from stdin + /bin/cat - > "$IN_FILE" +fi +$verbose_mode && printf "$script_name processing temp input file: %s\n" "$IN_FILE" + +##################################################################### +# Helper functions +##################################################################### + +init_out_files () { + $DO_NOT_PRINT_FILES && return + + # output files + NO_IDP_ROLE_FILE="$OUT_DIR/$NO_IDP_ROLE_FILENAME" + NO_ERROR_URL_FILE="$OUT_DIR/$NO_ERROR_URL_FILENAME" + ERROR_URL_LOG_FILE="$OUT_DIR/$ERROR_URL_LOG_FILENAME" + ENTITY_NAMES_FILE="$OUT_DIR/$ENTITY_NAMES_FILENAME" + ERROR_LOG_FILE="$OUT_DIR/$ERROR_LOG_FILENAME" + COMPATIBILITY_SCRIPT_FILE="$OUT_DIR/$COMPATIBILITY_SCRIPT_FILENAME" + + # clean up from last time if necessary + /bin/rm -f "$NO_IDP_ROLE_FILE" + /bin/rm -f "$NO_ERROR_URL_FILE" + /bin/rm -f "$ERROR_URL_LOG_FILE" + /bin/rm -f "$ENTITY_NAMES_FILE" + /bin/rm -f "$ERROR_LOG_FILE" + + # output cross-script compatibility info + /bin/cat <<- COMPATIBILITY_SCRIPT > $COMPATIBILITY_SCRIPT_FILE + # exactly one of the following two global vars will be nonempty + MD_PATH=$md_path + MDQ_BASE_URL=$mdq_base_url + # output files + NO_IDP_ROLE_FILE=$NO_IDP_ROLE_FILE + NO_ERROR_URL_FILE=$NO_ERROR_URL_FILE + ERROR_URL_LOG_FILE=$ERROR_URL_LOG_FILE + ENTITY_NAMES_FILE=$ENTITY_NAMES_FILE + # temporary output directory + TMP_DIR="$TMP_DIR" +COMPATIBILITY_SCRIPT +} + +print_entity_names_logfile () { + $DO_NOT_PRINT_FILES && return + + local statusCode=$1 + local responseCode=$2 + local names=$3 + + printf "%s\t%s\t%s\n" "$statusCode" "$responseCode" "$names" >> "$ENTITY_NAMES_FILE" +} + +print_no_idp_role_logfile () { + $DO_NOT_PRINT_FILES && return + + local entityID=$1 + local registrarID=$2 + + printf "%s %s\n" "$entityID" "$registrarID" >> "$NO_IDP_ROLE_FILE" +} + +print_no_error_url_logfile () { + $DO_NOT_PRINT_FILES && return + + local entityID=$1 + local registrarID=$2 + + printf "%s %s\n" "$entityID" "$registrarID" >> "$NO_ERROR_URL_FILE" +} + +print_error_url_logfile () { + $DO_NOT_PRINT_FILES && return + + local logfile=$1 + + printf "%s %s %s " "$status_code" "$output" "$errorURL" >> "$ERROR_URL_LOG_FILE" + printf "%s %s\n" "$entityID" "$registrarID" >> "$ERROR_URL_LOG_FILE" +} + +##################################################################### +# Main processing +##################################################################### + +init_out_files + +if $verbose_mode; then + num_entityIDs=$( /bin/cat $IN_FILE | /usr/bin/wc -l ) + printf "$script_name processing %d entityIDs\n" $num_entityIDs +fi + +# compute curl command-line options +curl_opts="--connect-timeout ${connect_timeout} --max-time ${max_time}" +curl_opts="${curl_opts} --insecure --tlsv1" +curl_opts="${curl_opts} --location --max-redirs 7" +$verbose_mode && printf "$script_name using curl opts: %s\n" "$curl_opts" + +# iterate over all entityIDs in the file +/bin/cat $IN_FILE | while read entityID; do + + # get the entity descriptor for this entityID + if $md_file_mode; then + entityDescriptor=$( getEntityFromFile -f "$md_path" $entityID ) + else + entityDescriptor=$( getEntityFromServer -d "$TMP_DIR" -u "$mdq_base_url" $entityID ) + fi + return_code=$? + if [ "$return_code" -ne 0 ]; then + echo "ERROR: $script_name: unable to obtain metadata for entityID: $entityID" >&2 + [ "$return_code" -gt 1 ] && exit 1 + continue + fi + + # extract the registrar ID from the entity descriptor + registrarID=$( echo "$entityDescriptor" \ + | $_GREP -F -m 1 ' registrationAuthority=' \ + | $_SED -e 's/^.* registrationAuthority="\([^"]*\)".*$/\1/' + ) + + # if there is no registrar ID, work around it and continue processing + if [ -z "$registrarID" ]; then + registrarID=NULL + fi + + # short-circuit the while-loop if this is not an IdP + if ! echo "$entityDescriptor" | $_GREP -Fq 'IDPSSODescriptor '; then + print_no_idp_role_logfile "$entityID" "$registrarID" + echo "WARNING: $script_name: entity is not an IdP: $entityID" >&2 + continue + fi + + # extract the IdP names and print them to a file + names=$( echo "$entityDescriptor" \ + | /usr/bin/xsltproc $LIB_DIR/extract_IdP_names.xsl - + ) + exit_code=$? + if [ "$exit_code" -ne 0 ]; then + echo "ERROR: $script_name: unable to extract IdP names for entityID: $entityID" >&2 + continue + fi + echo "$names" >> "$ENTITY_NAMES_FILE" + + # extract the errorURL from the entity descriptor + errorURL=$( echo "$entityDescriptor" \ + | $_GREP -E -m 1 '<(md:)?IDPSSODescriptor.* errorURL=' \ + | $_SED -e 's/^.* errorURL="\([^"]*\).*$/\1/' + ) + + # if there is no errorURL, short-circuit the while-loop + if [ -z "$errorURL" ]; then + print_no_error_url_logfile "$entityID" "$registrarID" + echo "INFO: $script_name: entity has no errorURL: $entityID" + continue + fi + + # request the errorURL + output=$( /usr/bin/curl --silent $curl_opts \ + --user-agent "${user_agent_string}" \ + --output "$HTTP_RESPONSE_FILE" \ + --write-out 'redirects:%{num_redirects};response:%{http_code};dns:%{time_namelookup};tcp:%{time_connect};ssl:%{time_appconnect};total:%{time_total}' \ + "$errorURL" + ) + status_code=$? + + #response_code=$( echo "$output" | $_SED -e 's/^.*;response:\([^;]*\).*$/\1/' ) + #print_entity_names_logfile "$status_code" "$response_code" "$names" + + printf "%s %s %s\n" "$status_code" "$output" "$errorURL" + print_error_url_logfile "$ERROR_URL_LOG_FILE" +done + +exit 0 diff --git a/bin/list_local_idp_error_urls.sh b/bin/list_local_idp_error_urls.sh new file mode 100755 index 0000000..27b4234 --- /dev/null +++ b/bin/list_local_idp_error_urls.sh @@ -0,0 +1,355 @@ +#!/bin/bash + +####################################################################### +# Copyright 2015--2016 Internet2 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +####################################################################### + +####################################################################### +# help message +####################################################################### + +display_help () { +/bin/cat <<- HELP_MSG + NOTE: Run check_idp_error_urls.sh before running this script! + + Given the results of a previously executed probe, this script + converts the flat text files produced by that probe into a set + of JSON files suitable for display on a web page. + + Usage: ${0##*/} [-hvq] -d IO_DIR [-o OUT_DIR] REGISTRAR_ID + + The -d option is required. The IO_DIR argument is an input + directory that contains the output of the probe_shib_idps.sh + script (which happens to have an identical command-line option, + btw). The OUT_DIR argument to the -o option is the output + directory for the current script. All of the JSON files will + be written to this directory. If the -o option is omitted, the + JSON files will be written to IO_DIR instead. +HELP_MSG +} + +####################################################################### +# Bootstrap +####################################################################### + +script_name=${0##*/} # equivalent to basename $0 +script_bin=${0%/*} # equivalent to dirname $0 + +# library filenames +COMMAND_PATHS_FILENAME=command_paths.sh +COMPATIBLE_MKTEMP_FILENAME=compatible_mktemp.sh +MD_TOOLS_FILENAME=md_tools.sh + +####################################################################### +# Process command-line options and arguments +####################################################################### + +help_mode=false; quiet_mode=false; verbose_mode=false +local_opts= +while getopts ":hqvd:o:" opt; do + case $opt in + h) + help_mode=true + ;; + q) + quiet_mode=true + verbose_mode=false + #local_opts="$local_opts -$opt" + ;; + v) + quiet_mode=false + verbose_mode=true + local_opts="$local_opts -$opt" + ;; + d) + io_dir="$OPTARG" + ;; + o) + out_dir="$OPTARG" + ;; + \?) + echo "ERROR: $script_name: Unrecognized option: -$OPTARG" >&2 + exit 2 + ;; + :) + echo "ERROR: $script_name: Option -$OPTARG requires an argument" >&2 + exit 2 + ;; + esac +done + +if $help_mode; then + display_help + exit 0 +fi + +# check the IO directory +if [ -z "$io_dir" ]; then + echo "ERROR: $script_name: IO_DIR (option -d) is missing" >&2 + exit 2 +fi +if [ ! -d "$io_dir" ]; then + echo "ERROR: $script_name: IO_DIR does not exist: $io_dir" >&2 + exit 2 +fi +$verbose_mode && printf "$script_name using IO_DIR: %s\n" "$io_dir" + +# check the output directory +if [ -z "$out_dir" ]; then + out_dir="$io_dir" +else + if [ ! -d "$out_dir" ]; then + echo "ERROR: $script_name: OUT_DIR does not exist: $out_dir" >&2 + exit 2 + fi +fi +$verbose_mode && printf "$script_name using OUT_DIR: %s\n" "$out_dir" + +shift $(( OPTIND - 1 )) +if [ "$#" -ne 1 ]; then + echo "ERROR: $script_name found $# command-line arguments (1 required)" >&2 + exit 2 +fi +registrarID=$1 + +####################################################################### +# Initialization +####################################################################### + +# Load the compatibility script +COMPATIBILITY_SCRIPT="$io_dir/compatibility.sh" +$verbose_mode && printf "$script_name sourcing compatibility script: %s\n" "$COMPATIBILITY_SCRIPT" +source "$COMPATIBILITY_SCRIPT" >&2 +exit_status=$? +if [ $exit_status -ne 0 ]; then + echo "ERROR: $script_name failed to source $COMPATIBILITY_SCRIPT (exit status $exit_status)" >&2 + exit $exit_status +fi + +if $quiet_mode; then + # redirect stdout to the bit bucket (stderr below) + exec 1>/dev/null + + # redirect stderr to a file + exec 2>"$ERROR_LOG_FILE" +fi + +# determine the source lib directory +if [ -z "$LIB_DIR" ]; then + echo "ERROR: $script_name requires env var LIB_DIR" >&2 + exit 2 +fi +if [ ! -d "$LIB_DIR" ]; then + echo "ERROR: $script_name: directory does not exist: $LIB_DIR" >&2 + exit 2 +fi +$verbose_mode && printf "$script_name using source lib directory: %s\n" "$LIB_DIR" + +# source lib files (always source command_paths first) +for source_filename in $COMMAND_PATHS_FILENAME $COMPATIBLE_MKTEMP_FILENAME $MD_TOOLS_FILENAME; do + source_file="$LIB_DIR/$source_filename" + if [ ! -f "$source_file" ]; then + echo "ERROR: $script_name: file does not exist: $source_file" >&2 + exit 2 + fi + $verbose_mode && printf "$script_name sourcing lib file: %s\n" "$source_file" + source "$source_file" >&2 + exit_status=$? + if [ $exit_status -ne 0 ]; then + echo "ERROR: ${script_name} failed to source script $source_file" >&2 + exit $exit_status + fi +done + +# create a temporary directory +#tmp_dir=$( make_temp_file -d ) +#if [ ! -d "$tmp_dir" ] ; then +# printf "ERROR: $script_name unable to create temporary dir\n" >&2 +# exit 2 +#fi +#$verbose_mode && printf "$script_name creating temp dir: %s\n" "$tmp_dir" + +####################################################################### +# Functions +####################################################################### + +escape_special_json_chars () { + local str="$1" + echo "$str" | $_SED -e 's/"/\\"/g' +} + +append_json_total_object () { + /bin/cat <<- JSON_TOTAL_OBJECT + { + "heading": "$heading", + "value": $( printf "%d" $value ) + } +JSON_TOTAL_OBJECT +} + +print_idp_summary () { + # begin array + printf "[\n" + + heading="Number of IdPs with a good errorURL" + value=0 + if [ -f "$ERROR_URL_LOG_FILE" ]; then + value=$( /bin/cat $ERROR_URL_LOG_FILE \ + | $_GREP " $registrarID\$" \ + | $_GREP "^0 redirects:.;response:200" \ + | /usr/bin/wc -l + ) + fi + append_json_total_object + + printf " ,\n" + + heading="Number of IdPs with a bad errorURL" + value=0 + if [ -f "$ERROR_URL_LOG_FILE" ]; then + value=$( /bin/cat $ERROR_URL_LOG_FILE \ + | $_GREP " $registrarID\$" \ + | $_GREP -v "^0 redirects:.;response:200" \ + | /usr/bin/wc -l + ) + fi + append_json_total_object + + printf " ,\n" + + heading="Number of IdPs with NO errorURL" + value=0 + if [ -f "$NO_ERROR_URL_FILE" ]; then + value=$( /bin/cat $NO_ERROR_URL_FILE \ + | $_GREP " $registrarID\$" \ + | /usr/bin/wc -l + ) + fi + append_json_total_object + + # end array + printf "]\n" +} + +append_json_object () { + local orgName=$( escape_special_json_chars "$orgName" ) + local displayName=$( escape_special_json_chars "$displayName" ) + local entityID=$( escape_special_json_chars "$entityID" ) + local errorURL=$( escape_special_json_chars "$errorURL" ) + local curlExitCode=$( escape_special_json_chars "$curlExitCode" ) + local httpResponseCode=$( escape_special_json_chars "$httpResponseCode" ) + + /bin/cat <<- JSON_OBJECT + { + "OrganizationName": "$orgName", + "DisplayName": "$displayName", + "entityID": "$entityID", + "errorURL": "$errorURL", + "curlExitCode": "$curlExitCode", + "HTTPResponseCode": "$httpResponseCode" + } +JSON_OBJECT +} + +print_idp_list () { + isFirstObject=true + + # begin output list + printf "[\n" + + # print list of IdPs with an errorURL in metadata + entityIDs=$( /bin/cat "$ERROR_URL_LOG_FILE" \ + | $_GREP " $registrarID\$" \ + | $_CUT -f4 -d" " + ) + if [ -n "$entityIDs" ]; then + for entityID in $entityIDs; do + # print leading comma unless this is the first object in the array + if $isFirstObject; then + isFirstObject=false + else + printf " ,\n" + fi + + # get the names for this IdP + names=$( /bin/cat "$ENTITY_NAMES_FILE" | $_GREP -m 1 "^$entityID\t" ) + displayName=$( echo "$names" | $_CUT -f2 ) + orgName=$( echo "$names" | $_CUT -f3 ) + + # get the errorURL for this IdP + log=$( /bin/cat "$ERROR_URL_LOG_FILE" | $_GREP -F -m 1 " $entityID " ) + errorURL=$( echo "$log" | $_CUT -f3 -d" " ) + + # get the codes for this IdP + curlExitCode=$( echo "$log" | $_CUT -f1 -d" " ) + httpResponseCode=$( echo "$log" | $_SED -e 's/^.*response:\([^;]*\).*$/\1/' ) + + append_json_object + done + fi + + # print list of IdPs with no errorURL in metadata + entityIDs=$( /bin/cat "$NO_ERROR_URL_FILE" \ + | $_GREP " $registrarID\$" \ + | $_CUT -f1 -d" " + ) + if [ -n "$entityIDs" ]; then + for entityID in $entityIDs; do + # print leading comma unless this is the first object in the array + if $isFirstObject; then + isFirstObject=false + else + printf " ,\n" + fi + + # get the names for this IdP + names=$( /bin/cat "$ENTITY_NAMES_FILE" | $_GREP -m 1 "^$entityID\t" ) + displayName=$( echo "$names" | $_CUT -f2 ) + orgName=$( echo "$names" | $_CUT -f3 ) + + # the rest are no-ops + errorURL=NONE + curlExitCode=NA + httpResponseCode=NA + + append_json_object + done + fi + + # end output list + printf "]\n" +} + +####################################################################### +# Main processing +####################################################################### + +# output files +idp_summary_file="$out_dir/local-idp-error_url-summary.json" +idp_list_file="$out_dir/local-idp-error-url-list.json" + +# clean up +/bin/rm -f "$idp_summary_file" +/bin/rm -f "$idp_list_file" + +# print summary of IdPs +$verbose_mode && printf "$script_name printing file: %s\n" "$idp_summary_file" +print_idp_summary > "$idp_summary_file" + +# print list of IdP errorURLs +$verbose_mode && printf "$script_name printing file: %s\n" "$idp_list_file" +print_idp_list > "$idp_list_file" + +exit 0 diff --git a/install.sh b/install.sh new file mode 100755 index 0000000..54079d2 --- /dev/null +++ b/install.sh @@ -0,0 +1,115 @@ +#!/bin/bash + +####################################################################### +# Copyright 2016 Internet2 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +####################################################################### + +################################################################ +# +# Usage: install.sh BIN_DIR LIB_DIR +# +# Example: Install in /tmp +# +# $ export BIN_DIR=/tmp/bin +# $ export LIB_DIR=/tmp/lib +# $ install.sh $BIN_DIR $LIB_DIR +# +# Example: Install in $HOME +# +# $ export BIN_DIR=$HOME/bin +# $ export LIB_DIR=$HOME/lib +# $ install.sh $BIN_DIR $LIB_DIR +# +################################################################ + +script_bin=${0%/*} # equivalent to dirname $0 +script_name=${0##*/} # equivalent to basename $0 + +# generalize +verbose_mode=true + +# get command-line args +if [ $# -ne 2 ]; then + echo "ERROR: $script_name: wrong number of arguments: $# (2 required)" >&2 + exit 2 +fi +bin_dir=$1 +lib_dir=$2 + +# check bin dir +if [ -z "$bin_dir" ]; then + echo "ERROR: $script_name requires bin directory (BIN_DIR)" >&2 + exit 2 +fi +if [ -d "$bin_dir" ]; then + $verbose_mode && echo "$script_name using bin dir: $bin_dir" +else + $verbose_mode && echo "$script_name creating bin dir: $bin_dir" + /bin/mkdir "$bin_dir" + exit_status=$? + if [ $exit_status -ne 0 ]; then + echo "ERROR: $script_name failed to create bin dir: $bin_dir" >&2 + exit $exit_status + fi +fi + +# check lib dir +if [ -z "$lib_dir" ]; then + echo "ERROR: $script_name requires lib directory (LIB_DIR)" >&2 + exit 2 +fi +if [ -d "$lib_dir" ]; then + $verbose_mode && echo "$script_name using lib dir: $lib_dir" +else + $verbose_mode && echo "$script_name creating lib dir: $lib_dir" + /bin/mkdir "$lib_dir" + exit_status=$? + if [ $exit_status -ne 0 ]; then + echo "ERROR: $script_name failed to create lib dir: $lib_dir" >&2 + exit $exit_status + fi +fi + +# initialize bin dir +while read script_file; do + $verbose_mode && echo "$script_name copying executable file: $script_file" + /bin/cp $script_file $bin_dir + exit_status=$? + if [ $exit_status -ne 0 ]; then + echo "ERROR: $script_name failed to copy script: $script_file" >&2 + exit $exit_status + fi +done <&2 + exit $exit_status + fi +done < + + + + + + + + + + + + + + + + diff --git a/lib/extract_IdP_names.xsl b/lib/extract_IdP_names.xsl new file mode 100644 index 0000000..b144019 --- /dev/null +++ b/lib/extract_IdP_names.xsl @@ -0,0 +1,51 @@ + + + + + + + + + + + + + + + + + + + + + + + diff --git a/lib/extract_InCommon_IdP_entityIDs.xsl b/lib/extract_InCommon_IdP_entityIDs.xsl new file mode 100644 index 0000000..8465975 --- /dev/null +++ b/lib/extract_InCommon_IdP_entityIDs.xsl @@ -0,0 +1,43 @@ + + + + + + + + + + + + + + + + +