From 7876348988eb87a8c89dbc134e62ca187e58041e Mon Sep 17 00:00:00 2001
From: Tom Scavo <trscavo@internet2.edu>
Date: Sun, 7 May 2017 14:53:09 -0400
Subject: [PATCH] Remove dependency on cget.sh

---
 bin/process_export_aggregate.sh | 101 +++++++++++++++-----------------
 bin/process_main_aggregate.sh   |  87 +++++++++++++--------------
 2 files changed, 89 insertions(+), 99 deletions(-)

diff --git a/bin/process_export_aggregate.sh b/bin/process_export_aggregate.sh
index c822030..f4fc216 100755
--- a/bin/process_export_aggregate.sh
+++ b/bin/process_export_aggregate.sh
@@ -47,11 +47,10 @@ display_help () {
 	
 	ENVIRONMENT
 	
-	This script leverages numerous environment variables:
+	This script leverages a handful of environment variables:
 	
-	  BIN_DIR    A directory of executable scripts
 	  LIB_DIR    A source library directory
-	  CACHE_DIR  An HTTP cache
+	  CACHE_DIR  A persistent HTTP cache
 	  TMPDIR     A temporary directory
 	  LOG_FILE   A persistent log file
 	  LOG_LEVEL  The global log level [0..5]
@@ -73,8 +72,7 @@ HELP_MSG
 script_name=${0##*/}  # equivalent to basename $0
 
 # required directories
-env_vars="BIN_DIR
-LIB_DIR
+env_vars="LIB_DIR
 CACHE_DIR
 TMPDIR"
 
@@ -107,6 +105,10 @@ if [ -z "$LOG_LEVEL" ]; then
 	LOG_LEVEL=3
 fi
 
+# library filenames (always list core_lib first)
+LIB_FILENAMES="core_lib.sh
+http_tools.sh"
+
 # output filenames
 csv_filename1="all-idps-exported.csv"
 csv_filename2="all-sps-exported.csv"
@@ -165,25 +167,20 @@ fi
 # Initialization
 #######################################################################
 
-# executable script must exist
-cget="$BIN_DIR/cget.sh"
-if [ ! -f "$cget" ]; then
-	echo "ERROR: $script_name: executable file does not exist: $cget" >&2
-	exit 2
-fi
-
-# source core library
-lib_file="$LIB_DIR/core_lib.sh"
-if [ ! -f "$lib_file" ]; then
-	echo "ERROR: $script_name: lib file does not exist: $lib_file" >&2
-	exit 2
-fi
-source "$lib_file"
-status_code=$?
-if [ $status_code -ne 0 ]; then
-	echo "ERROR: $script_name failed ($status_code) to source lib file $lib_file" >&2
-	exit 2
-fi
+# source lib files
+for lib_filename in $LIB_FILENAMES; do
+	lib_file="$LIB_DIR/$lib_filename"
+	if [ ! -f "$lib_file" ]; then
+		echo "ERROR: $script_name: lib file does not exist: $lib_file" >&2
+		exit 2
+	fi
+	source "$lib_file"
+	status_code=$?
+	if [ $status_code -ne 0 ]; then
+		echo "ERROR: $script_name failed ($status_code) to source lib file $lib_file" >&2
+		exit 2
+	fi
+done
 
 # XSL stylesheets must exist
 xsl_file1=$LIB_DIR/list_all_IdPs_csv.xsl
@@ -200,9 +197,9 @@ fi
 # create a temporary subdirectory
 tmp_dir="${TMPDIR%%/}/${script_name%%.*}_$$"
 /bin/mkdir "$tmp_dir"
-exit_code=$?
-if [ $exit_code -ne 0 ]; then
-	echo "ERROR: $script_name failed ($exit_code) to create tmp dir $tmp_dir" >&2
+status_code=$?
+if [ $status_code -ne 0 ]; then
+	echo "ERROR: $script_name failed ($status_code) to create tmp dir $tmp_dir" >&2
 	exit 2
 fi
 
@@ -219,49 +216,47 @@ csv_file2="${tmp_dir}/$csv_filename2"
 
 print_log_message -I "$script_name BEGIN"
 
-# XML metadata
-md_location=http://md.incommon.org/InCommon/InCommon-metadata-export.xml
-print_log_message -D "$script_name using metadata file: $md_location"
-
 # get a fresh metadata file
-print_log_message -D "$script_name writing XML file: $xml_file"
-$cget $local_opts -F $md_location > $xml_file
-exit_code=$?
-if [ $exit_code -eq 1 ]; then
+md_location=http://md.incommon.org/InCommon/InCommon-metadata-export.xml
+print_log_message -I "$script_name requesting metadata file: $md_location"
+conditional_get $local_opts -F -d "$CACHE_DIR" -T "$tmp_dir" "$md_location" > "$xml_file"
+status_code=$?
+if [ $status_code -eq 1 ]; then
 	# short-circuit if 304 response
 	print_log_message -I "$script_name END"
 	clean_up_and_exit -d "$tmp_dir" 0
 fi
-if [ $exit_code -gt 1 ]; then
-	print_log_message -E "$script_name: cget.sh failed ($exit_code) on location: $md_location"
-    clean_up_and_exit -d "$tmp_dir" $exit_code
+if [ $status_code -gt 1 ]; then
+	print_log_message -E "$script_name: cget.sh failed ($status_code) on location: $md_location"
+    clean_up_and_exit -d "$tmp_dir" $status_code
 fi
+print_log_message -D "$script_name using XML file: $xml_file"
 
 # create the first output file
-print_log_message -D "$script_name writing output file: $csv_file1"
+print_log_message -I "$script_name writing output file: $csv_file1"
 /usr/bin/xsltproc $xsl_file1 $xml_file > $csv_file1
-exit_code=$?
-if [ $exit_code -ne 0 ]; then
-	print_log_message -E "$script_name: xsltproc failed ($exit_code) on stylesheet: $xsl_file1"
-    clean_up_and_exit -d "$tmp_dir" $exit_code
+status_code=$?
+if [ $status_code -ne 0 ]; then
+	print_log_message -E "$script_name: xsltproc failed ($status_code) on stylesheet: $xsl_file1"
+    clean_up_and_exit -d "$tmp_dir" $status_code
 fi
 
 # create the second output file
-print_log_message -D "$script_name writing output file: $csv_file2"
+print_log_message -I "$script_name writing output file: $csv_file2"
 /usr/bin/xsltproc $xsl_file2 $xml_file > $csv_file2
-exit_code=$?
-if [ $exit_code -ne 0 ]; then
-	print_log_message -E "$script_name: xsltproc failed ($exit_code) on stylesheet: $xsl_file2"
-    clean_up_and_exit -d "$tmp_dir" $exit_code
+status_code=$?
+if [ $status_code -ne 0 ]; then
+	print_log_message -E "$script_name: xsltproc failed ($status_code) on stylesheet: $xsl_file2"
+    clean_up_and_exit -d "$tmp_dir" $status_code
 fi
 
 # move the output files to the web directory
-print_log_message -D "$script_name moving output files to dir: $out_dir"
+print_log_message -I "$script_name moving output files to dir: $out_dir"
 /bin/mv $csv_file1 $csv_file2 $out_dir
-exit_code=$?
-if [ $exit_code -ne 0 ]; then
-	print_log_message -E "$script_name: mv failed ($exit_code) to dir: $out_dir"
-    clean_up_and_exit -d "$tmp_dir" $exit_code
+status_code=$?
+if [ $status_code -ne 0 ]; then
+	print_log_message -E "$script_name: mv failed ($status_code) to dir: $out_dir"
+    clean_up_and_exit -d "$tmp_dir" $status_code
 fi
 
 print_log_message -I "$script_name END"
diff --git a/bin/process_main_aggregate.sh b/bin/process_main_aggregate.sh
index b38831b..1c35e9c 100755
--- a/bin/process_main_aggregate.sh
+++ b/bin/process_main_aggregate.sh
@@ -51,11 +51,10 @@ display_help () {
 	
 	ENVIRONMENT
 	
-	This script leverages numerous environment variables:
+	This script leverages a handful of environment variables:
 	
-	  BIN_DIR    A directory of executable scripts
 	  LIB_DIR    A source library directory
-	  CACHE_DIR  An HTTP cache
+	  CACHE_DIR  A persistent HTTP cache
 	  TMPDIR     A temporary directory
 	  LOG_FILE   A persistent log file
 	  LOG_LEVEL  The global log level [0..5]
@@ -77,8 +76,7 @@ HELP_MSG
 script_name=${0##*/}  # equivalent to basename $0
 
 # required directories
-env_vars="BIN_DIR
-LIB_DIR
+env_vars="LIB_DIR
 CACHE_DIR
 TMPDIR"
 
@@ -111,6 +109,10 @@ if [ -z "$LOG_LEVEL" ]; then
 	LOG_LEVEL=3
 fi
 
+# library filenames (always list core_lib first)
+LIB_FILENAMES="core_lib.sh
+http_tools.sh"
+
 # output filenames
 out_filenames[1]="all-idp-displaynames.csv"
 out_filenames[2]="all-idps-rands.csv"
@@ -181,25 +183,20 @@ fi
 # Initialization
 #######################################################################
 
-# executable script must exist
-cget="$BIN_DIR/cget.sh"
-if [ ! -f "$cget" ]; then
-	echo "ERROR: $script_name: executable file does not exist: $cget" >&2
-	exit 2
-fi
-
-# source core library
-lib_file="$LIB_DIR/core_lib.sh"
-if [ ! -f "$lib_file" ]; then
-	echo "ERROR: $script_name: lib file does not exist: $lib_file" >&2
-	exit 2
-fi
-source "$lib_file"
-status_code=$?
-if [ $status_code -ne 0 ]; then
-	echo "ERROR: $script_name failed ($status_code) to source lib file $lib_file" >&2
-	exit 2
-fi
+# source lib files
+for lib_filename in $LIB_FILENAMES; do
+	lib_file="$LIB_DIR/$lib_filename"
+	if [ ! -f "$lib_file" ]; then
+		echo "ERROR: $script_name: lib file does not exist: $lib_file" >&2
+		exit 2
+	fi
+	source "$lib_file"
+	status_code=$?
+	if [ $status_code -ne 0 ]; then
+		echo "ERROR: $script_name failed ($status_code) to source lib file $lib_file" >&2
+		exit 2
+	fi
+done
 
 # XSL stylesheets must exist
 for i in ${!xsl_filenames[*]}; do
@@ -213,9 +210,9 @@ done
 # create a temporary subdirectory
 tmp_dir="${TMPDIR%%/}/${script_name%%.*}_$$"
 /bin/mkdir "$tmp_dir"
-exit_code=$?
-if [ $exit_code -ne 0 ]; then
-	echo "ERROR: $script_name failed ($exit_code) to create tmp dir $tmp_dir" >&2
+status_code=$?
+if [ $status_code -ne 0 ]; then
+	echo "ERROR: $script_name failed ($status_code) to create tmp dir $tmp_dir" >&2
 	exit 2
 fi
 
@@ -233,42 +230,40 @@ done
 
 print_log_message -I "$script_name BEGIN"
 
-# XML metadata
-md_location=http://md.incommon.org/InCommon/InCommon-metadata.xml
-print_log_message -D "$script_name using metadata file: $md_location"
-
 # get a fresh metadata file
-print_log_message -D "$script_name writing XML file: $xml_file"
-$cget $local_opts -F $md_location > $xml_file
-exit_code=$?
-if [ $exit_code -eq 1 ]; then
+md_location=http://md.incommon.org/InCommon/InCommon-metadata.xml
+print_log_message -I "$script_name requesting metadata file: $md_location"
+conditional_get $local_opts -F -d "$CACHE_DIR" -T "$tmp_dir" "$md_location" > "$xml_file"
+status_code=$?
+if [ $status_code -eq 1 ]; then
 	# short-circuit if 304 response
 	print_log_message -I "$script_name END"
 	clean_up_and_exit -d "$tmp_dir" 0
 fi
-if [ $exit_code -gt 1 ]; then
-	print_log_message -E "$script_name: cget.sh failed ($exit_code) on location: $md_location"
-    clean_up_and_exit -d "$tmp_dir" $exit_code
+if [ $status_code -gt 1 ]; then
+	print_log_message -E "$script_name: conditional_get failed ($status_code) on location: $md_location"
+    clean_up_and_exit -d "$tmp_dir" $status_code
 fi
+print_log_message -D "$script_name using XML file: $xml_file"
 
 # create the output files
 for i in ${!out_files[*]}; do
 	print_log_message -I "$script_name writing output file: ${out_filenames[$i]}"
 	/usr/bin/xsltproc ${xsl_files[$i]} $xml_file > ${out_files[$i]}
-	exit_code=$?
-	if [ $exit_code -ne 0 ]; then
-		print_log_message -E "$script_name: xsltproc failed ($exit_code) on stylesheet: ${xsl_files[$i]}"
-		clean_up_and_exit -d "$tmp_dir" $exit_code
+	status_code=$?
+	if [ $status_code -ne 0 ]; then
+		print_log_message -E "$script_name: xsltproc failed ($status_code) on stylesheet: ${xsl_files[$i]}"
+		clean_up_and_exit -d "$tmp_dir" $status_code
 	fi
 done
 
 # move the output files to the web directory
 print_log_message -I "$script_name moving output files to dir: $out_dir"
 /bin/mv $( echo -n ${out_files[*]} ) $out_dir
-exit_code=$?
-if [ $exit_code -ne 0 ]; then
-	print_log_message -E "$script_name: mv failed ($exit_code) to dir: $out_dir"
-    clean_up_and_exit -d "$tmp_dir" $exit_code
+status_code=$?
+if [ $status_code -ne 0 ]; then
+	print_log_message -E "$script_name: mv failed ($status_code) to dir: $out_dir"
+    clean_up_and_exit -d "$tmp_dir" $status_code
 fi
 
 print_log_message -I "$script_name END"