From 2613570349a9b150e2875906f5b987859c62918f Mon Sep 17 00:00:00 2001 From: Pavol Mederly Date: Mon, 24 Sep 2018 23:56:47 +0200 Subject: [PATCH] Improve logging and TIER beacon support Changes in the midPoint Docker container: 1. Logging is improved: now we provide TIER-style logging for both midpoint.log content and plain console output. (Note that not all entry timestamps are in UTC yet.) 2. Reverted some of TIER beacon changes to make it more compliant with TIER standards: shell file names, opt-out environment variable, collector server name+port. --- midpoint/.env | 3 +-- midpoint/docker-compose.yml | 1 + midpoint/midpoint-server/Dockerfile | 26 ++++++++++++------- .../supervisor/supervisord.conf | 6 ++--- ...{send-tier-beacon.sh => sendtierbeacon.sh} | 12 ++++----- .../usr-local-bin/set-timezone.sh | 8 ++++++ .../usr-local-bin/setup-cron.sh | 12 +++------ .../{start-httpd-shib.sh => start-httpd.sh} | 0 .../usr-local-bin/start-midpoint.sh | 8 +++--- .../{start-all.sh => startup.sh} | 9 ++++++- 10 files changed, 52 insertions(+), 33 deletions(-) rename midpoint/midpoint-server/container_files/usr-local-bin/{send-tier-beacon.sh => sendtierbeacon.sh} (72%) create mode 100755 midpoint/midpoint-server/container_files/usr-local-bin/set-timezone.sh rename midpoint/midpoint-server/container_files/usr-local-bin/{start-httpd-shib.sh => start-httpd.sh} (100%) rename midpoint/midpoint-server/container_files/usr-local-bin/{start-all.sh => startup.sh} (58%) diff --git a/midpoint/.env b/midpoint/.env index a038dbc..03f48af 100644 --- a/midpoint/.env +++ b/midpoint/.env @@ -1,6 +1,6 @@ AUTHENTICATION=internal ENV=demo -USERTOKEN=3.9-SNAPSHOT +USERTOKEN= REPO_DATABASE_TYPE=mariadb REPO_JDBC_URL=default REPO_HOST=midpoint-data @@ -12,4 +12,3 @@ KEYSTORE_PASSWORD_FILE=/run/secrets/m_keystore_password.txt MEM=2048m LOGOUT_URL=https://localhost:8443/Shibboleth.sso/Logout SSO_HEADER=uid -TIER_BEACON_ENABLED=true diff --git a/midpoint/docker-compose.yml b/midpoint/docker-compose.yml index f2684c1..6605429 100644 --- a/midpoint/docker-compose.yml +++ b/midpoint/docker-compose.yml @@ -40,6 +40,7 @@ services: - MEM - SSO_HEADER - LOGOUT_URL + - TIER_BEACON_OPT_OUT networks: - back secrets: diff --git a/midpoint/midpoint-server/Dockerfile b/midpoint/midpoint-server/Dockerfile index ebb5539..5f0c06c 100644 --- a/midpoint/midpoint-server/Dockerfile +++ b/midpoint/midpoint-server/Dockerfile @@ -27,11 +27,11 @@ COPY container_files/usr-local-bin/* /usr/local/bin/ COPY container_files/opt-tier/* /opt/tier/ RUN chmod 755 /opt/tier/setenv.sh \ - && chmod 755 /usr/local/bin/send-tier-beacon.sh \ + && chmod 755 /usr/local/bin/sendtierbeacon.sh \ && chmod 755 /usr/local/bin/setup-cron.sh \ && chmod 755 /usr/local/bin/start-midpoint.sh \ - && chmod 755 /usr/local/bin/start-httpd-shib.sh \ - && chmod 755 /usr/local/bin/start-all.sh + && chmod 755 /usr/local/bin/start-httpd.sh \ + && chmod 755 /usr/local/bin/startup.sh RUN cp /dev/null /etc/httpd/conf.d/ssl.conf \ && sed -i 's/LogFormat "/LogFormat "httpd;access_log;%{ENV}e;%{USERTOKEN}e;/g' /etc/httpd/conf/httpd.conf \ @@ -71,7 +71,7 @@ ENV REPO_DATABASE_TYPE mariadb # Logging parameters ENV ENV demo -ENV USERTOKEN $MP_VERSION +ENV USERTOKEN "" # Authentication/web @@ -85,14 +85,22 @@ ENV LOGOUT_URL https://localhost:8443/Shibboleth.sso/Logout ENV KEYSTORE_PASSWORD_FILE /run/secrets/m_keystore_password.txt ENV MEM 2048m - -ENV TIER_RELEASE=test-non-release +ENV TIER_RELEASE=not-released-yet ENV TIER_MAINTAINER=tier -ENV TIER_BEACON_ENABLED=true -RUN pwd +# TIER Beacon Opt-out +# Completely uncomment the following ENV line to prevent the containers from sending analytics information to Internet2. +# With the default/release configuration, it will only send product (Shibb/Grouper/COmanage/midPoint) and version (3.9, etc) +# once daily between midnight and 4am. There is no configuration or private information collected or sent. +# This data helps with the scaling and funding of TIER. Please do not disable it if you find the TIER tools useful. +# To keep it commented, keep multiple comments on the following line (to prevent other scripts from processing it). +##### ENV TIER_BEACON_OPT_OUT true # requires MP_VERSION and TIER_xyz variables so we have to execute it here + +# TODO JUST FOR TESTING -- REMOVE BEFORE RELEASE +RUN /usr/local/bin/set-timezone.sh + RUN /opt/tier/setenv.sh -CMD ["/usr/local/bin/start-all.sh"] +CMD ["/usr/local/bin/startup.sh"] diff --git a/midpoint/midpoint-server/container_files/supervisor/supervisord.conf b/midpoint/midpoint-server/container_files/supervisor/supervisord.conf index c64ebb4..d0d91e2 100644 --- a/midpoint/midpoint-server/container_files/supervisor/supervisord.conf +++ b/midpoint/midpoint-server/container_files/supervisor/supervisord.conf @@ -5,8 +5,8 @@ loglevel=error nodaemon=true user=root -[program:httpd-shib] -command=/bin/bash -c "/usr/local/bin/start-httpd-shib.sh" +[program:httpd] +command=/bin/bash -c "/usr/local/bin/start-httpd.sh" stdout_logfile=/tmp/loghttpd stdout_logfile_maxbytes=0 redirect_stderr=true @@ -17,7 +17,7 @@ stdout_logfile=/dev/fd/2 stdout_logfile_maxbytes=0 redirect_stderr=true -[program:tier-beacon] +[program:crond] command=/usr/sbin/crond -n -i -m off stdout_logfile=/tmp/logcrond stdout_logfile_maxbytes=0 diff --git a/midpoint/midpoint-server/container_files/usr-local-bin/send-tier-beacon.sh b/midpoint/midpoint-server/container_files/usr-local-bin/sendtierbeacon.sh similarity index 72% rename from midpoint/midpoint-server/container_files/usr-local-bin/send-tier-beacon.sh rename to midpoint/midpoint-server/container_files/usr-local-bin/sendtierbeacon.sh index 8548075..2f263bb 100755 --- a/midpoint/midpoint-server/container_files/usr-local-bin/send-tier-beacon.sh +++ b/midpoint/midpoint-server/container_files/usr-local-bin/sendtierbeacon.sh @@ -1,7 +1,7 @@ #!/bin/bash -LOGHOST="localhost" -LOGPORT="80" +LOGHOST="collector.testbed.tier.internet2.edu" +LOGPORT="5001" if [ -s /opt/tier/env.bash ]; then . /opt/tier/env.bash @@ -15,21 +15,21 @@ if [ -z "$TIER_BEACON_OPT_OUT" ]; then "msgType" : "TIERBEACON", "msgName" : "TIER", "msgVersion" : "1.0", - "tbProduct" : "MIDPOINT", + "tbProduct" : "midPoint", "tbProductVersion" : "$MP_VERSION", "tbTIERRelease" : "$TIER_RELEASE", "tbMaintainer" : "$TIER_MAINTAINER" } EOF -# echo `date`": going to send TIER beacon to ${LOGHOST}:${LOGPORT}:" +# echo "going to send TIER beacon to ${LOGHOST}:${LOGPORT}:" # cat $messagefile curl -s -XPOST "${LOGHOST}:${LOGPORT}/" -H 'Content-Type: application/json' -T $messagefile 1>/dev/null 2>&1 if [ $? -eq 0 ]; then - echo `date`": TIER beacon sent" + echo "TIER beacon sent" else - echo `date`": Failed to send TIER beacon" + echo "Failed to send TIER beacon" fi rm -f $messagefile 1>/dev/null 2>&1 diff --git a/midpoint/midpoint-server/container_files/usr-local-bin/set-timezone.sh b/midpoint/midpoint-server/container_files/usr-local-bin/set-timezone.sh new file mode 100755 index 0000000..3de53ff --- /dev/null +++ b/midpoint/midpoint-server/container_files/usr-local-bin/set-timezone.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +# JUST FOR TESTING - REMOVE BEFORE RELEASE + +rm /etc/localtime +ln -s /usr/share/zoneinfo/Europe/Bratislava /etc/localtime +date + diff --git a/midpoint/midpoint-server/container_files/usr-local-bin/setup-cron.sh b/midpoint/midpoint-server/container_files/usr-local-bin/setup-cron.sh index 4d96158..9ec9705 100755 --- a/midpoint/midpoint-server/container_files/usr-local-bin/setup-cron.sh +++ b/midpoint/midpoint-server/container_files/usr-local-bin/setup-cron.sh @@ -1,14 +1,10 @@ #!/bin/bash -CRONFILE=/opt/tier/cronfile +CRONFILE=/opt/tier/tier-cron -if [ "$TIER_BEACON_ENABLED" == "true" ]; then - echo "#send daily \"beacon\" to central" > ${CRONFILE} -# echo $(expr $RANDOM % 59) $(expr $RANDOM % 3) "* * * /usr/local/bin/send-tier-beacon.sh >> /tmp/logcrond 2>&1" >> ${CRONFILE} - echo "* * * * * /usr/local/bin/send-tier-beacon.sh >> /tmp/logcrond 2>&1" >> ${CRONFILE} # for testing -else - echo "#beacon is disabled" > ${CRONFILE} -fi +echo "#send daily \"beacon\" to central" > ${CRONFILE} +echo $(expr $RANDOM % 60) $(expr $RANDOM % 4) "* * * /usr/local/bin/sendtierbeacon.sh >> /tmp/logcrond 2>&1" >> ${CRONFILE} +#echo "* * * * * /usr/local/bin/sendtierbeacon.sh >> /tmp/logcrond 2>&1" >> ${CRONFILE} # for testing chmod 644 ${CRONFILE} crontab ${CRONFILE} diff --git a/midpoint/midpoint-server/container_files/usr-local-bin/start-httpd-shib.sh b/midpoint/midpoint-server/container_files/usr-local-bin/start-httpd.sh similarity index 100% rename from midpoint/midpoint-server/container_files/usr-local-bin/start-httpd-shib.sh rename to midpoint/midpoint-server/container_files/usr-local-bin/start-httpd.sh diff --git a/midpoint/midpoint-server/container_files/usr-local-bin/start-midpoint.sh b/midpoint/midpoint-server/container_files/usr-local-bin/start-midpoint.sh index bc00b84..856f34e 100755 --- a/midpoint/midpoint-server/container_files/usr-local-bin/start-midpoint.sh +++ b/midpoint/midpoint-server/container_files/usr-local-bin/start-midpoint.sh @@ -16,13 +16,13 @@ java -Xmx$MEM -Xms2048m -Dfile.encoding=UTF8 \ -Dmidpoint.repository.missingSchemaAction=create \ -Dmidpoint.repository.initializationFailTimeout=60000 \ -Dmidpoint.keystore.keyStorePassword_FILE=$KEYSTORE_PASSWORD_FILE \ - -Dmidpoint.logging.console.enabled=true \ - -Dmidpoint.logging.console.prefix="midpoint;midpoint.log;$ENV;$USERTOKEN;" \ - -Dmidpoint.logging.console.timezone=UTC \ + -Dmidpoint.logging.alt.enabled=true \ + -Dmidpoint.logging.alt.filename=/tmp/logmidpoint \ + -Dmidpoint.logging.alt.timezone=UTC \ -Dspring.profiles.active="`$MP_DIR/active-spring-profiles`" \ -Dauth.sso.header=$SSO_HEADER \ $LOGOUT_URL_DIRECTIVE \ -Dserver.tomcat.ajp.enabled=$AJP_ENABLED \ -Dserver.tomcat.ajp.port=$AJP_PORT \ -Dlogging.path=/tmp/logtomcat \ - -jar $MP_DIR/lib/midpoint.war + -jar $MP_DIR/lib/midpoint.war &>/tmp/logmidpoint-console diff --git a/midpoint/midpoint-server/container_files/usr-local-bin/start-all.sh b/midpoint/midpoint-server/container_files/usr-local-bin/startup.sh similarity index 58% rename from midpoint/midpoint-server/container_files/usr-local-bin/start-all.sh rename to midpoint/midpoint-server/container_files/usr-local-bin/startup.sh index 8c96fe4..fe880d0 100755 --- a/midpoint/midpoint-server/container_files/usr-local-bin/start-all.sh +++ b/midpoint/midpoint-server/container_files/usr-local-bin/startup.sh @@ -4,6 +4,7 @@ export ENV=${ENV//[; ]/_} export USERTOKEN=${USERTOKEN//[; ]/_} +# this is to be executed at run time, not at build time -- to ensure sufficient variability of execution times /usr/local/bin/setup-cron.sh # generic console logging pipe for anyone @@ -17,7 +18,7 @@ mkfifo -m 666 /tmp/logshib (cat <> /tmp/logshib | awk '{printf "%s\n", $0; fflush()}' 1>/tmp/logpipe) & mkfifo -m 666 /tmp/logcrond -(cat <> /tmp/logcrond | awk -v ENV="$ENV" -v USERTOKEN="$USERTOKEN" '{printf "crond;console;%s;%s;%s\n", ENV, USERTOKEN, $0; fflush()}' 1>/tmp/logpipe) & +(cat <> /tmp/logcrond | awk -v ENV="$ENV" -v USERTOKEN="$USERTOKEN" '{line=sprintf ("crond;console;%s;%s;%s:%s", ENV, USERTOKEN, strftime("%F %T%z", systime(), 1), $0); print line >> "/tmp/logpipe"; print line >> "/var/log/cron.log"; fflush()}') & mkfifo -m 666 /tmp/logsuperd (cat <> /tmp/logsuperd | awk -v ENV="$ENV" -v USERTOKEN="$USERTOKEN" '{printf "supervisord;console;%s;%s;%s\n", ENV, USERTOKEN, $0; fflush()}' 1>/tmp/logpipe) & @@ -25,4 +26,10 @@ mkfifo -m 666 /tmp/logsuperd mkfifo -m 666 /tmp/logtomcat (cat <> /tmp/logtomcat | awk -v ENV="$ENV" -v USERTOKEN="$USERTOKEN" '{printf "tomcat;console;%s;%s;%s\n", ENV, USERTOKEN, $0; fflush()}' 1>/tmp/logpipe) & +mkfifo -m 666 /tmp/logmidpoint +(cat <> /tmp/logmidpoint | awk -v ENV="$ENV" -v USERTOKEN="$USERTOKEN" '{printf "midpoint;midpoint.log;%s;%s;%s\n", ENV, USERTOKEN, $0; fflush()}' 1>/tmp/logpipe) & + +mkfifo -m 666 /tmp/logmidpoint-console +(cat <> /tmp/logmidpoint-console | awk -v ENV="$ENV" -v USERTOKEN="$USERTOKEN" '{printf "midpoint;console;%s;%s;%s\n", ENV, USERTOKEN, $0; fflush()}' 1>/tmp/logpipe) & + /usr/bin/supervisord -c /etc/supervisor/supervisord.conf