diff --git a/bin/process_export_aggregate.sh b/bin/process_export_aggregate.sh deleted file mode 100755 index 60179f4..0000000 --- a/bin/process_export_aggregate.sh +++ /dev/null @@ -1,293 +0,0 @@ -#!/bin/bash - -####################################################################### -# Copyright 2017 Internet2 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -####################################################################### - -####################################################################### -# Help message -####################################################################### - -display_help () { -/bin/cat <<- HELP_MSG - This script produces the following files from the InCommon - export aggregate: - - $( printf " %s\n" ${out_filenames[*]} ) - - The files will be written to the output directory specified on - the command line. - - The script is intended to be run as a cron job. It can (and - should) be run often, to pick up the latest changes to metadata. - There is little penalty in doing so since the script uses HTTP - Conditional GET to retrieve metadata. - - Usage: ${0##*/} [-hv] -d OUT_DIR - - Options: - -h Display this help message - -v Enable DEBUG mode - -d Specify the output directory - - Option -h is mutually exclusive of all other options. - - Option -d specifies the ultimate output directory, which is - usually a web directory. This option is REQUIRED. - - ENVIRONMENT - - This script leverages a handful of environment variables: - - LIB_DIR A source library directory - CACHE_DIR A persistent HTTP cache - TMPDIR A temporary directory - LOG_FILE A persistent log file - LOG_LEVEL The global log level [0..5] - - All of the above environment variables are REQUIRED - except LOG_LEVEL, which defaults to LOG_LEVEL=3. - - The following environment variables are REQUIRED: - - $( printf " %s\n" ${env_vars[*]} ) - - The following directories MUST exist: - - $( printf " %s\n" ${dir_paths[*]} ) - - The following files MUST exist: - - $( printf " %s\n" $LOG_FILE ) - - CONFIGURATION - - The following source library files MUST be installed in LIB_DIR: - - $( printf " %s\n" ${lib_filenames[*]} ) - - The following XSL scripts MUST be installed in LIB_DIR: - - $( printf " %s\n" ${xsl_filenames[*]} ) - - EXAMPLES - - \$ out_dir=/home/htdocs/www.incommonfederation.org/federation/metadata/ - \$ ${0##*/} -d \$out_dir -HELP_MSG -} - -####################################################################### -# Bootstrap -####################################################################### - -script_name=${0##*/} # equivalent to basename $0 - -# required environment variables -env_vars[1]="LIB_DIR" -env_vars[2]="CACHE_DIR" -env_vars[3]="TMPDIR" -env_vars[4]="LOG_FILE" - -# check environment variables -for env_var in ${env_vars[*]}; do - eval "env_var_val=\${$env_var}" - if [ -z "$env_var_val" ]; then - echo "ERROR: $script_name requires env var $env_var" >&2 - exit 2 - fi -done - -# required directories -dir_paths[1]="$LIB_DIR" -dir_paths[2]="$CACHE_DIR" -dir_paths[3]="$TMPDIR" - -# check required directories -for dir_path in ${dir_paths[*]}; do - if [ ! -d "$dir_path" ]; then - echo "ERROR: $script_name: directory does not exist: $dir_path" >&2 - exit 2 - fi -done - -# check the log file -# devices such as /dev/tty and /dev/null are allowed -if [ ! -f "$LOG_FILE" ] && [[ $LOG_FILE != /dev/* ]]; then - echo "ERROR: $script_name: file does not exist: $LOG_FILE" >&2 - exit 2 -fi - -# default to INFO logging -if [ -z "$LOG_LEVEL" ]; then - LOG_LEVEL=3 -fi - -# source lib filenames -lib_filenames[1]="core_lib.sh" -lib_filenames[2]="http_tools.sh" - -# check lib files -for lib_filename in ${lib_filenames[*]}; do - lib_file="$LIB_DIR/$lib_filename" - if [ ! -f "$lib_file" ]; then - echo "ERROR: $script_name: lib file does not exist: $lib_file" >&2 - exit 2 - fi -done - -# XSL filenames -xsl_filenames[1]="list_all_IdPs_csv.xsl" -xsl_filenames[2]="list_all_SPs_csv.xsl" - -# check XSL files -for i in ${!xsl_filenames[*]}; do - xsl_files[$i]=$LIB_DIR/${xsl_filenames[$i]} - if [ ! -f "${xsl_files[$i]}" ]; then - echo "ERROR: $script_name: lib file does not exist: ${xsl_files[$i]}" >&2 - exit 2 - fi -done - -# output filenames -out_filenames[1]="all-idps-exported.csv" -out_filenames[2]="all-sps-exported.csv" - -####################################################################### -# Process command-line options and arguments -####################################################################### - -help_mode=false; local_opts= -while getopts ":hvd:" opt; do - case $opt in - h) - help_mode=true - ;; - v) - LOG_LEVEL=4 - local_opts="$local_opts -$opt" - ;; - d) - out_dir="$OPTARG" - ;; - \?) - echo "ERROR: $script_name: Unrecognized option: -$OPTARG" >&2 - exit 2 - ;; - :) - echo "ERROR: $script_name: Option -$OPTARG requires an argument" >&2 - exit 2 - ;; - esac -done - -if $help_mode; then - display_help - exit 0 -fi - -# check the output directory -if [ -z "$out_dir" ]; then - echo "ERROR: $script_name: no output directory specified" >&2 - exit 2 -fi -if [ ! -d "$out_dir" ]; then - echo "ERROR: $script_name: directory does not exist: $out_dir" >&2 - exit 2 -fi - -# check command-line arguments -shift $(( OPTIND - 1 )) -if [ $# -ne 0 ]; then - echo "ERROR: $script_name: wrong number of arguments: $# (0 required)" >&2 - exit 2 -fi - -####################################################################### -# Initialization -####################################################################### - -# source lib files -for lib_filename in ${lib_filenames[*]}; do - lib_file="$LIB_DIR/$lib_filename" - source "$lib_file" - status_code=$? - if [ $status_code -ne 0 ]; then - echo "ERROR: $script_name failed ($status_code) to source lib file $lib_file" >&2 - exit 2 - fi -done - -# create a temporary subdirectory -tmp_dir="${TMPDIR%%/}/${script_name%%.*}_$$" -/bin/mkdir "$tmp_dir" -status_code=$? -if [ $status_code -ne 0 ]; then - echo "ERROR: $script_name failed ($status_code) to create tmp dir $tmp_dir" >&2 - exit 2 -fi - -# input file -xml_file="${tmp_dir}/saml-metadata.xml" - -# output files -for i in ${!out_filenames[*]}; do - out_files[$i]="${tmp_dir}/${out_filenames[$i]}" -done - -####################################################################### -# Main processing -####################################################################### - -print_log_message -I "$script_name BEGIN" - -# get a fresh metadata file -md_location=http://md.incommon.org/InCommon/InCommon-metadata-export.xml -print_log_message -I "$script_name requesting metadata file: $md_location" -conditional_get $local_opts -F -d "$CACHE_DIR" -T "$tmp_dir" "$md_location" > "$xml_file" -status_code=$? -if [ $status_code -eq 1 ]; then - # short-circuit if 304 response - print_log_message -I "$script_name END" - clean_up_and_exit -d "$tmp_dir" 0 -fi -if [ $status_code -gt 1 ]; then - print_log_message -E "$script_name: conditional_get failed ($status_code) on location: $md_location" - clean_up_and_exit -d "$tmp_dir" $status_code -fi -print_log_message -D "$script_name using XML file: $xml_file" - -# create the output files -for i in ${!out_files[*]}; do - print_log_message -I "$script_name writing output file: ${out_filenames[$i]}" - /usr/bin/xsltproc ${xsl_files[$i]} $xml_file > ${out_files[$i]} - status_code=$? - if [ $status_code -ne 0 ]; then - print_log_message -E "$script_name: xsltproc failed ($status_code) on stylesheet: ${xsl_files[$i]}" - clean_up_and_exit -d "$tmp_dir" $status_code - fi -done - -# move the output files to the web directory -print_log_message -I "$script_name moving output files to dir: $out_dir" -/bin/mv $( echo -n ${out_files[*]} ) $out_dir -status_code=$? -if [ $status_code -ne 0 ]; then - print_log_message -E "$script_name: mv failed ($status_code) to dir: $out_dir" - clean_up_and_exit -d "$tmp_dir" $status_code -fi - -print_log_message -I "$script_name END" -clean_up_and_exit -d "$tmp_dir" 0 diff --git a/bin/process_main_aggregate.sh b/bin/process_main_aggregate.sh deleted file mode 100755 index ead8546..0000000 --- a/bin/process_main_aggregate.sh +++ /dev/null @@ -1,301 +0,0 @@ -#!/bin/bash - -####################################################################### -# Copyright 2017 Internet2 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -####################################################################### - -####################################################################### -# Help message -####################################################################### - -display_help () { -/bin/cat <<- HELP_MSG - This script produces the following files from the InCommon - main aggregate: - - $( printf " %s\n" ${out_filenames[*]} ) - - The files will be written to the output directory specified on - the command line. - - The script is intended to be run as a cron job. It can (and - should) be run often, to pick up the latest changes to metadata. - There is little penalty in doing so since the script uses HTTP - Conditional GET to retrieve metadata. - - Usage: ${0##*/} [-hv] -d OUT_DIR - - Options: - -h Display this help message - -v Enable DEBUG mode - -d Specify the output directory - - Option -h is mutually exclusive of all other options. - - Option -d specifies the ultimate output directory, which is - usually a web directory. This option is REQUIRED. - - ENVIRONMENT - - This script leverages a handful of environment variables: - - LIB_DIR A source library directory - CACHE_DIR A persistent HTTP cache - TMPDIR A temporary directory - LOG_FILE A persistent log file - LOG_LEVEL The global log level [0..5] - - All of the above environment variables are REQUIRED - except LOG_LEVEL, which defaults to LOG_LEVEL=3. - - The following environment variables are REQUIRED: - - $( printf " %s\n" ${env_vars[*]} ) - - The following directories MUST exist: - - $( printf " %s\n" ${dir_paths[*]} ) - - The following files MUST exist: - - $( printf " %s\n" $LOG_FILE ) - - CONFIGURATION - - The following source library files MUST be installed in LIB_DIR: - - $( printf " %s\n" ${lib_filenames[*]} ) - - The following XSL scripts MUST be installed in LIB_DIR: - - $( printf " %s\n" ${xsl_filenames[*]} ) - - EXAMPLES - - \$ out_dir=/home/htdocs/www.incommonfederation.org/federation/metadata/ - \$ ${0##*/} -d \$out_dir -HELP_MSG -} - -####################################################################### -# Bootstrap -####################################################################### - -script_name=${0##*/} # equivalent to basename $0 - -# required environment variables -env_vars[1]="LIB_DIR" -env_vars[2]="CACHE_DIR" -env_vars[3]="TMPDIR" -env_vars[4]="LOG_FILE" - -# check environment variables -for env_var in ${env_vars[*]}; do - eval "env_var_val=\${$env_var}" - if [ -z "$env_var_val" ]; then - echo "ERROR: $script_name requires env var $env_var" >&2 - exit 2 - fi -done - -# required directories -dir_paths[1]="$LIB_DIR" -dir_paths[2]="$CACHE_DIR" -dir_paths[3]="$TMPDIR" - -# check required directories -for dir_path in ${dir_paths[*]}; do - if [ ! -d "$dir_path" ]; then - echo "ERROR: $script_name: directory does not exist: $dir_path" >&2 - exit 2 - fi -done - -# check the log file -# devices such as /dev/tty and /dev/null are allowed -if [ ! -f "$LOG_FILE" ] && [[ $LOG_FILE != /dev/* ]]; then - echo "ERROR: $script_name: file does not exist: $LOG_FILE" >&2 - exit 2 -fi - -# default to INFO logging -if [ -z "$LOG_LEVEL" ]; then - LOG_LEVEL=3 -fi - -# source lib filenames -lib_filenames[1]="core_lib.sh" -lib_filenames[2]="http_tools.sh" - -# check lib files -for lib_filename in ${lib_filenames[*]}; do - lib_file="$LIB_DIR/$lib_filename" - if [ ! -f "$lib_file" ]; then - echo "ERROR: $script_name: lib file does not exist: $lib_file" >&2 - exit 2 - fi -done - -# XSL filenames -xsl_filenames[1]="list_all_IdP_DisplayNames_csv.xsl" -xsl_filenames[2]="list_all_RandS_IdPs_csv.xsl" -xsl_filenames[3]="list_all_RandS_SPs_csv.xsl" -xsl_filenames[4]="security_contacts_legacy_list_csv.xsl" -xsl_filenames[5]="security_contacts_summary_json.xsl" -xsl_filenames[6]="security_contacts_summary_local_json.xsl" - -# check XSL files -for i in ${!xsl_filenames[*]}; do - xsl_files[$i]=$LIB_DIR/${xsl_filenames[$i]} - if [ ! -f "${xsl_files[$i]}" ]; then - echo "ERROR: $script_name: lib file does not exist: ${xsl_files[$i]}" >&2 - exit 2 - fi -done - -# output filenames -out_filenames[1]="all-idp-displaynames.csv" -out_filenames[2]="all-idps-rands.csv" -out_filenames[3]="all-sps-rands.csv" -out_filenames[4]="security-contacts-legacy-list.csv" -out_filenames[5]="security-contacts-summary.json" -out_filenames[6]="security-contacts-summary-local.json" - -####################################################################### -# Process command-line options and arguments -####################################################################### - -help_mode=false; local_opts= -while getopts ":hvd:" opt; do - case $opt in - h) - help_mode=true - ;; - v) - LOG_LEVEL=4 - local_opts="$local_opts -$opt" - ;; - d) - out_dir="$OPTARG" - ;; - \?) - echo "ERROR: $script_name: Unrecognized option: -$OPTARG" >&2 - exit 2 - ;; - :) - echo "ERROR: $script_name: Option -$OPTARG requires an argument" >&2 - exit 2 - ;; - esac -done - -if $help_mode; then - display_help - exit 0 -fi - -# check the output directory -if [ -z "$out_dir" ]; then - echo "ERROR: $script_name: no output directory specified" >&2 - exit 2 -fi -if [ ! -d "$out_dir" ]; then - echo "ERROR: $script_name: directory does not exist: $out_dir" >&2 - exit 2 -fi - -# check command-line arguments -shift $(( OPTIND - 1 )) -if [ $# -ne 0 ]; then - echo "ERROR: $script_name: wrong number of arguments: $# (0 required)" >&2 - exit 2 -fi - -####################################################################### -# Initialization -####################################################################### - -# source lib files -for lib_filename in ${lib_filenames[*]}; do - lib_file="$LIB_DIR/$lib_filename" - source "$lib_file" - status_code=$? - if [ $status_code -ne 0 ]; then - echo "ERROR: $script_name failed ($status_code) to source lib file $lib_file" >&2 - exit 2 - fi -done - -# create a temporary subdirectory -tmp_dir="${TMPDIR%%/}/${script_name%%.*}_$$" -/bin/mkdir "$tmp_dir" -status_code=$? -if [ $status_code -ne 0 ]; then - echo "ERROR: $script_name failed ($status_code) to create tmp dir $tmp_dir" >&2 - exit 2 -fi - -# input file -xml_file="${tmp_dir}/saml-metadata.xml" - -# output files -for i in ${!out_filenames[*]}; do - out_files[$i]="${tmp_dir}/${out_filenames[$i]}" -done - -####################################################################### -# Main processing -####################################################################### - -print_log_message -I "$script_name BEGIN" - -# get a fresh metadata file -md_location=http://md.incommon.org/InCommon/InCommon-metadata.xml -print_log_message -I "$script_name requesting metadata file: $md_location" -conditional_get $local_opts -F -d "$CACHE_DIR" -T "$tmp_dir" "$md_location" > "$xml_file" -status_code=$? -if [ $status_code -eq 1 ]; then - # short-circuit if 304 response - print_log_message -I "$script_name END" - clean_up_and_exit -d "$tmp_dir" 0 -fi -if [ $status_code -gt 1 ]; then - print_log_message -E "$script_name: conditional_get failed ($status_code) on location: $md_location" - clean_up_and_exit -d "$tmp_dir" $status_code -fi -print_log_message -D "$script_name using XML file: $xml_file" - -# create the output files -for i in ${!out_files[*]}; do - print_log_message -I "$script_name writing output file: ${out_filenames[$i]}" - /usr/bin/xsltproc ${xsl_files[$i]} $xml_file > ${out_files[$i]} - status_code=$? - if [ $status_code -ne 0 ]; then - print_log_message -E "$script_name: xsltproc failed ($status_code) on stylesheet: ${xsl_files[$i]}" - clean_up_and_exit -d "$tmp_dir" $status_code - fi -done - -# move the output files to the web directory -print_log_message -I "$script_name moving output files to dir: $out_dir" -/bin/mv $( echo -n ${out_files[*]} ) $out_dir -status_code=$? -if [ $status_code -ne 0 ]; then - print_log_message -E "$script_name: mv failed ($status_code) to dir: $out_dir" - clean_up_and_exit -d "$tmp_dir" $status_code -fi - -print_log_message -I "$script_name END" -clean_up_and_exit -d "$tmp_dir" 0