Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
Remove dependency on cget.sh
Tom Scavo committed May 7, 2017
1 parent a275c96 commit 7876348
Showing 2 changed files with 89 additions and 99 deletions.
101 changes: 48 additions & 53 deletions bin/process_export_aggregate.sh
View file
@@ -47,11 +47,10 @@ display_help () {
ENVIRONMENT
This script leverages numerous environment variables:
This script leverages a handful of environment variables:
BIN_DIR A directory of executable scripts
LIB_DIR A source library directory
CACHE_DIR An HTTP cache
CACHE_DIR A persistent HTTP cache
TMPDIR A temporary directory
LOG_FILE A persistent log file
LOG_LEVEL The global log level [0..5]
@@ -73,8 +72,7 @@ HELP_MSG
script_name=${0##*/} # equivalent to basename $0

# required directories
env_vars="BIN_DIR
LIB_DIR
env_vars="LIB_DIR
CACHE_DIR
TMPDIR"

@@ -107,6 +105,10 @@ if [ -z "$LOG_LEVEL" ]; then
LOG_LEVEL=3
fi

# library filenames (always list core_lib first)
LIB_FILENAMES="core_lib.sh
http_tools.sh"

# output filenames
csv_filename1="all-idps-exported.csv"
csv_filename2="all-sps-exported.csv"
@@ -165,25 +167,20 @@ fi
# Initialization
#######################################################################

# executable script must exist
cget="$BIN_DIR/cget.sh"
if [ ! -f "$cget" ]; then
echo "ERROR: $script_name: executable file does not exist: $cget" >&2
exit 2
fi

# source core library
lib_file="$LIB_DIR/core_lib.sh"
if [ ! -f "$lib_file" ]; then
echo "ERROR: $script_name: lib file does not exist: $lib_file" >&2
exit 2
fi
source "$lib_file"
status_code=$?
if [ $status_code -ne 0 ]; then
echo "ERROR: $script_name failed ($status_code) to source lib file $lib_file" >&2
exit 2
fi
# source lib files
for lib_filename in $LIB_FILENAMES; do
lib_file="$LIB_DIR/$lib_filename"
if [ ! -f "$lib_file" ]; then
echo "ERROR: $script_name: lib file does not exist: $lib_file" >&2
exit 2
fi
source "$lib_file"
status_code=$?
if [ $status_code -ne 0 ]; then
echo "ERROR: $script_name failed ($status_code) to source lib file $lib_file" >&2
exit 2
fi
done

# XSL stylesheets must exist
xsl_file1=$LIB_DIR/list_all_IdPs_csv.xsl
@@ -200,9 +197,9 @@ fi
# create a temporary subdirectory
tmp_dir="${TMPDIR%%/}/${script_name%%.*}_$$"
/bin/mkdir "$tmp_dir"
exit_code=$?
if [ $exit_code -ne 0 ]; then
echo "ERROR: $script_name failed ($exit_code) to create tmp dir $tmp_dir" >&2
status_code=$?
if [ $status_code -ne 0 ]; then
echo "ERROR: $script_name failed ($status_code) to create tmp dir $tmp_dir" >&2
exit 2
fi

@@ -219,49 +216,47 @@ csv_file2="${tmp_dir}/$csv_filename2"

print_log_message -I "$script_name BEGIN"

# XML metadata
md_location=http://md.incommon.org/InCommon/InCommon-metadata-export.xml
print_log_message -D "$script_name using metadata file: $md_location"

# get a fresh metadata file
print_log_message -D "$script_name writing XML file: $xml_file"
$cget $local_opts -F $md_location > $xml_file
exit_code=$?
if [ $exit_code -eq 1 ]; then
md_location=http://md.incommon.org/InCommon/InCommon-metadata-export.xml
print_log_message -I "$script_name requesting metadata file: $md_location"
conditional_get $local_opts -F -d "$CACHE_DIR" -T "$tmp_dir" "$md_location" > "$xml_file"
status_code=$?
if [ $status_code -eq 1 ]; then
# short-circuit if 304 response
print_log_message -I "$script_name END"
clean_up_and_exit -d "$tmp_dir" 0
fi
if [ $exit_code -gt 1 ]; then
print_log_message -E "$script_name: cget.sh failed ($exit_code) on location: $md_location"
clean_up_and_exit -d "$tmp_dir" $exit_code
if [ $status_code -gt 1 ]; then
print_log_message -E "$script_name: cget.sh failed ($status_code) on location: $md_location"
clean_up_and_exit -d "$tmp_dir" $status_code
fi
print_log_message -D "$script_name using XML file: $xml_file"

# create the first output file
print_log_message -D "$script_name writing output file: $csv_file1"
print_log_message -I "$script_name writing output file: $csv_file1"
/usr/bin/xsltproc $xsl_file1 $xml_file > $csv_file1
exit_code=$?
if [ $exit_code -ne 0 ]; then
print_log_message -E "$script_name: xsltproc failed ($exit_code) on stylesheet: $xsl_file1"
clean_up_and_exit -d "$tmp_dir" $exit_code
status_code=$?
if [ $status_code -ne 0 ]; then
print_log_message -E "$script_name: xsltproc failed ($status_code) on stylesheet: $xsl_file1"
clean_up_and_exit -d "$tmp_dir" $status_code
fi

# create the second output file
print_log_message -D "$script_name writing output file: $csv_file2"
print_log_message -I "$script_name writing output file: $csv_file2"
/usr/bin/xsltproc $xsl_file2 $xml_file > $csv_file2
exit_code=$?
if [ $exit_code -ne 0 ]; then
print_log_message -E "$script_name: xsltproc failed ($exit_code) on stylesheet: $xsl_file2"
clean_up_and_exit -d "$tmp_dir" $exit_code
status_code=$?
if [ $status_code -ne 0 ]; then
print_log_message -E "$script_name: xsltproc failed ($status_code) on stylesheet: $xsl_file2"
clean_up_and_exit -d "$tmp_dir" $status_code
fi

# move the output files to the web directory
print_log_message -D "$script_name moving output files to dir: $out_dir"
print_log_message -I "$script_name moving output files to dir: $out_dir"
/bin/mv $csv_file1 $csv_file2 $out_dir
exit_code=$?
if [ $exit_code -ne 0 ]; then
print_log_message -E "$script_name: mv failed ($exit_code) to dir: $out_dir"
clean_up_and_exit -d "$tmp_dir" $exit_code
status_code=$?
if [ $status_code -ne 0 ]; then
print_log_message -E "$script_name: mv failed ($status_code) to dir: $out_dir"
clean_up_and_exit -d "$tmp_dir" $status_code
fi

print_log_message -I "$script_name END"
87 changes: 41 additions & 46 deletions bin/process_main_aggregate.sh
View file
@@ -51,11 +51,10 @@ display_help () {
ENVIRONMENT
This script leverages numerous environment variables:
This script leverages a handful of environment variables:
BIN_DIR A directory of executable scripts
LIB_DIR A source library directory
CACHE_DIR An HTTP cache
CACHE_DIR A persistent HTTP cache
TMPDIR A temporary directory
LOG_FILE A persistent log file
LOG_LEVEL The global log level [0..5]
@@ -77,8 +76,7 @@ HELP_MSG
script_name=${0##*/} # equivalent to basename $0

# required directories
env_vars="BIN_DIR
LIB_DIR
env_vars="LIB_DIR
CACHE_DIR
TMPDIR"

@@ -111,6 +109,10 @@ if [ -z "$LOG_LEVEL" ]; then
LOG_LEVEL=3
fi

# library filenames (always list core_lib first)
LIB_FILENAMES="core_lib.sh
http_tools.sh"

# output filenames
out_filenames[1]="all-idp-displaynames.csv"
out_filenames[2]="all-idps-rands.csv"
@@ -181,25 +183,20 @@ fi
# Initialization
#######################################################################

# executable script must exist
cget="$BIN_DIR/cget.sh"
if [ ! -f "$cget" ]; then
echo "ERROR: $script_name: executable file does not exist: $cget" >&2
exit 2
fi

# source core library
lib_file="$LIB_DIR/core_lib.sh"
if [ ! -f "$lib_file" ]; then
echo "ERROR: $script_name: lib file does not exist: $lib_file" >&2
exit 2
fi
source "$lib_file"
status_code=$?
if [ $status_code -ne 0 ]; then
echo "ERROR: $script_name failed ($status_code) to source lib file $lib_file" >&2
exit 2
fi
# source lib files
for lib_filename in $LIB_FILENAMES; do
lib_file="$LIB_DIR/$lib_filename"
if [ ! -f "$lib_file" ]; then
echo "ERROR: $script_name: lib file does not exist: $lib_file" >&2
exit 2
fi
source "$lib_file"
status_code=$?
if [ $status_code -ne 0 ]; then
echo "ERROR: $script_name failed ($status_code) to source lib file $lib_file" >&2
exit 2
fi
done

# XSL stylesheets must exist
for i in ${!xsl_filenames[*]}; do
@@ -213,9 +210,9 @@ done
# create a temporary subdirectory
tmp_dir="${TMPDIR%%/}/${script_name%%.*}_$$"
/bin/mkdir "$tmp_dir"
exit_code=$?
if [ $exit_code -ne 0 ]; then
echo "ERROR: $script_name failed ($exit_code) to create tmp dir $tmp_dir" >&2
status_code=$?
if [ $status_code -ne 0 ]; then
echo "ERROR: $script_name failed ($status_code) to create tmp dir $tmp_dir" >&2
exit 2
fi

@@ -233,42 +230,40 @@ done

print_log_message -I "$script_name BEGIN"

# XML metadata
md_location=http://md.incommon.org/InCommon/InCommon-metadata.xml
print_log_message -D "$script_name using metadata file: $md_location"

# get a fresh metadata file
print_log_message -D "$script_name writing XML file: $xml_file"
$cget $local_opts -F $md_location > $xml_file
exit_code=$?
if [ $exit_code -eq 1 ]; then
md_location=http://md.incommon.org/InCommon/InCommon-metadata.xml
print_log_message -I "$script_name requesting metadata file: $md_location"
conditional_get $local_opts -F -d "$CACHE_DIR" -T "$tmp_dir" "$md_location" > "$xml_file"
status_code=$?
if [ $status_code -eq 1 ]; then
# short-circuit if 304 response
print_log_message -I "$script_name END"
clean_up_and_exit -d "$tmp_dir" 0
fi
if [ $exit_code -gt 1 ]; then
print_log_message -E "$script_name: cget.sh failed ($exit_code) on location: $md_location"
clean_up_and_exit -d "$tmp_dir" $exit_code
if [ $status_code -gt 1 ]; then
print_log_message -E "$script_name: conditional_get failed ($status_code) on location: $md_location"
clean_up_and_exit -d "$tmp_dir" $status_code
fi
print_log_message -D "$script_name using XML file: $xml_file"

# create the output files
for i in ${!out_files[*]}; do
print_log_message -I "$script_name writing output file: ${out_filenames[$i]}"
/usr/bin/xsltproc ${xsl_files[$i]} $xml_file > ${out_files[$i]}
exit_code=$?
if [ $exit_code -ne 0 ]; then
print_log_message -E "$script_name: xsltproc failed ($exit_code) on stylesheet: ${xsl_files[$i]}"
clean_up_and_exit -d "$tmp_dir" $exit_code
status_code=$?
if [ $status_code -ne 0 ]; then
print_log_message -E "$script_name: xsltproc failed ($status_code) on stylesheet: ${xsl_files[$i]}"
clean_up_and_exit -d "$tmp_dir" $status_code
fi
done

# move the output files to the web directory
print_log_message -I "$script_name moving output files to dir: $out_dir"
/bin/mv $( echo -n ${out_files[*]} ) $out_dir
exit_code=$?
if [ $exit_code -ne 0 ]; then
print_log_message -E "$script_name: mv failed ($exit_code) to dir: $out_dir"
clean_up_and_exit -d "$tmp_dir" $exit_code
status_code=$?
if [ $status_code -ne 0 ]; then
print_log_message -E "$script_name: mv failed ($status_code) to dir: $out_dir"
clean_up_and_exit -d "$tmp_dir" $status_code
fi

print_log_message -I "$script_name END"

0 comments on commit 7876348

Please sign in to comment.