Improve logging and TIER beacon support

Changes in the midPoint Docker container:
1. Logging is improved: now we provide TIER-style logging for
   both midpoint.log content and plain console output. (Note that
   not all entry timestamps are in UTC yet.)
2. Reverted some of TIER beacon changes to make it more compliant
   with TIER standards: shell file names, opt-out environment variable,
   collector server name+port.

midpoint/.env

@@ -1,6 +1,6 @@
 AUTHENTICATION=internal
 ENV=demo
-USERTOKEN=3.9-SNAPSHOT
+USERTOKEN=
 REPO_DATABASE_TYPE=mariadb
 REPO_JDBC_URL=default
 REPO_HOST=midpoint-data
@@ -12,4 +12,3 @@ KEYSTORE_PASSWORD_FILE=/run/secrets/m_keystore_password.txt
 MEM=2048m
 LOGOUT_URL=https://localhost:8443/Shibboleth.sso/Logout
 SSO_HEADER=uid
-TIER_BEACON_ENABLED=true

midpoint/docker-compose.yml

@@ -40,6 +40,7 @@ services:
      - MEM
      - SSO_HEADER
      - LOGOUT_URL
+     - TIER_BEACON_OPT_OUT
     networks:
      - back
     secrets:

midpoint/midpoint-server/Dockerfile

@@ -27,11 +27,11 @@ COPY container_files/usr-local-bin/* /usr/local/bin/
 COPY container_files/opt-tier/* /opt/tier/
 
 RUN chmod 755 /opt/tier/setenv.sh \
-    && chmod 755 /usr/local/bin/send-tier-beacon.sh \
+    && chmod 755 /usr/local/bin/sendtierbeacon.sh \
     && chmod 755 /usr/local/bin/setup-cron.sh \
     && chmod 755 /usr/local/bin/start-midpoint.sh \
-    && chmod 755 /usr/local/bin/start-httpd-shib.sh \
-    && chmod 755 /usr/local/bin/start-all.sh
+    && chmod 755 /usr/local/bin/start-httpd.sh \
+    && chmod 755 /usr/local/bin/startup.sh
 
 RUN cp /dev/null /etc/httpd/conf.d/ssl.conf \
     && sed -i 's/LogFormat "/LogFormat "httpd;access_log;%{ENV}e;%{USERTOKEN}e;/g' /etc/httpd/conf/httpd.conf \
@@ -71,7 +71,7 @@ ENV REPO_DATABASE_TYPE mariadb
 # Logging parameters
 
 ENV ENV demo
-ENV USERTOKEN $MP_VERSION
+ENV USERTOKEN ""
 
 # Authentication/web
 
@@ -85,14 +85,22 @@ ENV LOGOUT_URL https://localhost:8443/Shibboleth.sso/Logout
 
 ENV KEYSTORE_PASSWORD_FILE /run/secrets/m_keystore_password.txt
 ENV MEM 2048m
-
-ENV TIER_RELEASE=test-non-release
+ENV TIER_RELEASE=not-released-yet
 ENV TIER_MAINTAINER=tier
-ENV TIER_BEACON_ENABLED=true
 
-RUN pwd
+# TIER Beacon Opt-out
+# Completely uncomment the following ENV line to prevent the containers from sending analytics information to Internet2.
+# With the default/release configuration, it will only send product (Shibb/Grouper/COmanage/midPoint) and version (3.9, etc)
+# once daily between midnight and 4am.  There is no configuration or private information collected or sent.
+# This data helps with the scaling and funding of TIER.  Please do not disable it if you find the TIER tools useful.
+# To keep it commented, keep multiple comments on the following line (to prevent other scripts from processing it).
+#####     ENV TIER_BEACON_OPT_OUT true
 
 # requires MP_VERSION and TIER_xyz variables so we have to execute it here
+
+# TODO JUST FOR TESTING -- REMOVE BEFORE RELEASE
+RUN /usr/local/bin/set-timezone.sh
+
 RUN /opt/tier/setenv.sh
 
-CMD ["/usr/local/bin/start-all.sh"]
+CMD ["/usr/local/bin/startup.sh"]

midpoint/midpoint-server/container_files/supervisor/supervisord.conf

@@ -5,8 +5,8 @@ loglevel=error
 nodaemon=true
 user=root
 
-[program:httpd-shib]
-command=/bin/bash -c "/usr/local/bin/start-httpd-shib.sh"
+[program:httpd]
+command=/bin/bash -c "/usr/local/bin/start-httpd.sh"
 stdout_logfile=/tmp/loghttpd
 stdout_logfile_maxbytes=0
 redirect_stderr=true
@@ -17,7 +17,7 @@ stdout_logfile=/dev/fd/2
 stdout_logfile_maxbytes=0
 redirect_stderr=true
 
-[program:tier-beacon]
+[program:crond]
 command=/usr/sbin/crond -n -i -m off
 stdout_logfile=/tmp/logcrond
 stdout_logfile_maxbytes=0

midpoint/midpoint-server/container_files/usr-local-bin/send-tier-beacon.sh → midpoint/midpoint-server/container_files/usr-local-bin/sendtierbeacon.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 
-LOGHOST="localhost"
-LOGPORT="80"
+LOGHOST="collector.testbed.tier.internet2.edu"
+LOGPORT="5001"
 
 if [ -s /opt/tier/env.bash ]; then
   . /opt/tier/env.bash
@@ -15,21 +15,21 @@ if [ -z "$TIER_BEACON_OPT_OUT" ]; then
     "msgType"          : "TIERBEACON",
     "msgName"          : "TIER",
     "msgVersion"       : "1.0",
-    "tbProduct"        : "MIDPOINT",
+    "tbProduct"        : "midPoint",
     "tbProductVersion" : "$MP_VERSION",
     "tbTIERRelease"    : "$TIER_RELEASE",
     "tbMaintainer"     : "$TIER_MAINTAINER"
 }
 EOF
 
-#    echo `date`": going to send TIER beacon to ${LOGHOST}:${LOGPORT}:"
+#    echo "going to send TIER beacon to ${LOGHOST}:${LOGPORT}:"
 #    cat $messagefile
 
     curl -s -XPOST "${LOGHOST}:${LOGPORT}/" -H 'Content-Type: application/json' -T $messagefile 1>/dev/null 2>&1
     if [ $? -eq 0 ]; then
-        echo `date`": TIER beacon sent"
+        echo "TIER beacon sent"
     else
-        echo `date`": Failed to send TIER beacon"
+        echo "Failed to send TIER beacon"
     fi
 
     rm -f $messagefile 1>/dev/null 2>&1

midpoint/midpoint-server/container_files/usr-local-bin/set-timezone.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+# JUST FOR TESTING - REMOVE BEFORE RELEASE
+
+rm /etc/localtime
+ln -s /usr/share/zoneinfo/Europe/Bratislava /etc/localtime
+date
+

midpoint/midpoint-server/container_files/usr-local-bin/setup-cron.sh

@@ -1,14 +1,10 @@
 #!/bin/bash
 
-CRONFILE=/opt/tier/cronfile
+CRONFILE=/opt/tier/tier-cron
 
-if [ "$TIER_BEACON_ENABLED" == "true" ]; then
-    echo "#send daily \"beacon\" to central" > ${CRONFILE}
-#    echo $(expr $RANDOM % 59) $(expr $RANDOM % 3) "* * * /usr/local/bin/send-tier-beacon.sh >> /tmp/logcrond 2>&1" >> ${CRONFILE}
-    echo "* * * * * /usr/local/bin/send-tier-beacon.sh >> /tmp/logcrond 2>&1" >> ${CRONFILE}		# for testing
-else
-    echo "#beacon is disabled" > ${CRONFILE}
-fi
+echo "#send daily \"beacon\" to central" > ${CRONFILE}
+echo $(expr $RANDOM % 60) $(expr $RANDOM % 4) "* * * /usr/local/bin/sendtierbeacon.sh >> /tmp/logcrond 2>&1" >> ${CRONFILE}
+#echo "* * * * * /usr/local/bin/sendtierbeacon.sh >> /tmp/logcrond 2>&1" >> ${CRONFILE}		# for testing
 
 chmod 644 ${CRONFILE}
 crontab ${CRONFILE}

midpoint/midpoint-server/container_files/usr-local-bin/start-midpoint.sh

@@ -16,13 +16,13 @@ java -Xmx$MEM -Xms2048m -Dfile.encoding=UTF8 \
        -Dmidpoint.repository.missingSchemaAction=create \
        -Dmidpoint.repository.initializationFailTimeout=60000 \
        -Dmidpoint.keystore.keyStorePassword_FILE=$KEYSTORE_PASSWORD_FILE \
-       -Dmidpoint.logging.console.enabled=true \
-       -Dmidpoint.logging.console.prefix="midpoint;midpoint.log;$ENV;$USERTOKEN;" \
-       -Dmidpoint.logging.console.timezone=UTC \
+       -Dmidpoint.logging.alt.enabled=true \
+       -Dmidpoint.logging.alt.filename=/tmp/logmidpoint \
+       -Dmidpoint.logging.alt.timezone=UTC \
        -Dspring.profiles.active="`$MP_DIR/active-spring-profiles`" \
        -Dauth.sso.header=$SSO_HEADER \
        $LOGOUT_URL_DIRECTIVE \
        -Dserver.tomcat.ajp.enabled=$AJP_ENABLED \
        -Dserver.tomcat.ajp.port=$AJP_PORT \
        -Dlogging.path=/tmp/logtomcat \
-       -jar $MP_DIR/lib/midpoint.war
+       -jar $MP_DIR/lib/midpoint.war &>/tmp/logmidpoint-console

midpoint/midpoint-server/container_files/usr-local-bin/start-all.sh → midpoint/midpoint-server/container_files/usr-local-bin/startup.sh

@@ -4,6 +4,7 @@
 export ENV=${ENV//[; ]/_}
 export USERTOKEN=${USERTOKEN//[; ]/_}
 
+# this is to be executed at run time, not at build time -- to ensure sufficient variability of execution times
 /usr/local/bin/setup-cron.sh
 
 # generic console logging pipe for anyone
@@ -17,12 +18,18 @@ mkfifo -m 666 /tmp/logshib
 (cat <> /tmp/logshib  | awk '{printf "%s\n", $0; fflush()}' 1>/tmp/logpipe) &
 
 mkfifo -m 666 /tmp/logcrond
-(cat <> /tmp/logcrond  | awk -v ENV="$ENV" -v USERTOKEN="$USERTOKEN" '{printf "crond;console;%s;%s;%s\n", ENV, USERTOKEN, $0; fflush()}' 1>/tmp/logpipe) &
+(cat <> /tmp/logcrond  | awk -v ENV="$ENV" -v USERTOKEN="$USERTOKEN" '{line=sprintf ("crond;console;%s;%s;%s:%s", ENV, USERTOKEN, strftime("%F %T%z", systime(), 1), $0); print line >> "/tmp/logpipe"; print line >> "/var/log/cron.log"; fflush()}') &
 
 mkfifo -m 666 /tmp/logsuperd
 (cat <> /tmp/logsuperd | awk -v ENV="$ENV" -v USERTOKEN="$USERTOKEN" '{printf "supervisord;console;%s;%s;%s\n", ENV, USERTOKEN, $0; fflush()}' 1>/tmp/logpipe) &
 
 mkfifo -m 666 /tmp/logtomcat
 (cat <> /tmp/logtomcat | awk -v ENV="$ENV" -v USERTOKEN="$USERTOKEN" '{printf "tomcat;console;%s;%s;%s\n", ENV, USERTOKEN, $0; fflush()}' 1>/tmp/logpipe) &
 
+mkfifo -m 666 /tmp/logmidpoint
+(cat <> /tmp/logmidpoint | awk -v ENV="$ENV" -v USERTOKEN="$USERTOKEN" '{printf "midpoint;midpoint.log;%s;%s;%s\n", ENV, USERTOKEN, $0; fflush()}' 1>/tmp/logpipe) &
+
+mkfifo -m 666 /tmp/logmidpoint-console
+(cat <> /tmp/logmidpoint-console | awk -v ENV="$ENV" -v USERTOKEN="$USERTOKEN" '{printf "midpoint;console;%s;%s;%s\n", ENV, USERTOKEN, $0; fflush()}' 1>/tmp/logpipe) &
+
 /usr/bin/supervisord -c /etc/supervisor/supervisord.conf