finishing touches on logging for now

.dockerignore

@@ -2,4 +2,5 @@
 test-compose/
 *.md
 manualBuild.sh
+Jenkinsfile
 LICENSE

README.md

@@ -235,7 +235,23 @@ $ docker run -it --rm \
   tier/grouper gsh -registry -check -runscript -noprompt
 ```
 
-Note: a less privileged database user maybe used when running the typical Grouper roles. This user need SELECT, INSERT, UPDATE, and DELETE privileges on the schema objects.
+Note: a less privileged database user maybe used when running the typical Grouper roles. This user needs SELECT, INSERT, UPDATE, and DELETE privileges on the schema objects.
+
+# Logging
+
+This image outputs logs in a manner that is consistent with Docker Logging. Each log entry is prefaced with the submodule name (e.g. shibd, httpd, tomcat, grouper), the logfile name (e.g. access_log, grouper_error.log, catalina.out) and user definable environment name and a user definable token. Content found after the preface will be specific to the application ands its logging configuration.
+
+> Note: If customizing a particular component's logging, it is recommended that the file be source from the image (`docker container cp`) or from the image's source repository. 
+
+To assign the "environment" string, set the environment variable `ENV` when defining the Docker service. For the "user defined token" string, use the environment variable of `USERTOKEN`.
+
+An example might look like the following, with the env of "dev" and the usertoken of "build-2"
+
+```text
+shibd shibd.log dev build-2 2018-03-27 20:42:22 INFO Shibboleth.Listener : listener service starting
+grouper-api grouper_event.log dev build-2 2018-03-27 21:10:00,046: [DefaultQuartzScheduler_Worker-1] INFO  EventLog.info(156) -  - [fdbb0099fe9e46e5be4371eb11250d39,'GrouperSystem','application'] session: start (0ms)
+tomcat console dev build-2 Grouper starting up: version: 2.3.0, build date: null, env: <no label configured>
+``` 
 
 # Misc Notes
 

container_files/tier-support/supervisord-tomcat.conf

@@ -1,5 +1,5 @@
 [supervisord]
-logfile=/tmp/logpipe                               ; supervisord log file
+logfile=/tmp/logsuperd                              ; supervisord log file
 logfile_maxbytes=0                           ; maximum size of logfile before rotation
 loglevel=error                                  ; info, debug, warn, trace
 nodaemon=true                                  ; run supervisord as a daemon
@@ -16,24 +16,24 @@ serverurl=unix:///tmp/supervisor.sock         ; use a unix:// URL  for a unix so
 
 [program:httpd]
 command=httpd -DFOREGROUND
-stderr_logfile = /tmp/logpipe
+stderr_logfile = /tmp/loghttpd
 stderr_logfile_maxbytes=0
-stdout_logfile = /tmp/logpipe
+stdout_logfile = /tmp/loghttpd
 stdout_logfile_maxbytes=0
 
 [program:shibbolethsp]
 user=shibd
 command=/usr/sbin/shibd -f -F
-stderr_logfile = /tmp/logpipe
+stderr_logfile = /tmp/logshidb
 stderr_logfile_maxbytes=0
-stdout_logfile = /tmp/logpipe
+stdout_logfile = /tmp/logshidb
 stdout_logfile_maxbytes=0
 
 [program:tomcat]
 user=tomcat
 command=/opt/tomcat/bin/catalina.sh run 
-stderr_logfile = /tmp/logpipe
+stderr_logfile = /tmp/logtomcat
 stderr_logfile_maxbytes=0
-stdout_logfile = /tmp/logpipe
+stdout_logfile = /tmp/logtomcat
 stdout_logfile_maxbytes=0
 

container_files/tomcat/conf/log4j2.xml

@@ -1,16 +1,15 @@
 <?xml version="1.0" encoding="utf-8"?>
 <Configuration status="info">
     <Properties>
-        <Property name="logdir">${sys:catalina.base}/logs</Property>
         <Property name="layout">%d [%t] %-5p %c- %m%n</Property>
     </Properties>
     <Appenders>
         <File name="CATALINA"
-                     fileName="${logdir}/catalina.out">
+                     fileName="/tmp/logpipe">
             <PatternLayout pattern="tomcat catalina.out ${env:ENV} ${env:USERTOKEN} ${layout}"/>
         </File>
         <File name="LOCALHOST"
-                     fileName="${logdir}/localhost.log">
+                     fileName="/tmp/logpipe">
             <PatternLayout pattern="tomcat localhost.log ${env:ENV} ${env:USERTOKEN} ${layout}"/>
         </File>
 

container_files/tomee/conf/log4j2.xml

@@ -1,16 +1,15 @@
 <?xml version="1.0" encoding="utf-8"?>
 <Configuration status="info">
     <Properties>
-        <Property name="logdir">${sys:catalina.base}/logs</Property>
         <Property name="layout">%d [%t] %-5p %c- %m%n</Property>
     </Properties>
     <Appenders>
         <File name="CATALINA"
-                     fileName="${logdir}/catalina.out">
+                     fileName="/tmp/logpipe">
             <PatternLayout pattern="tomee catalina.out ${env:ENV} ${env:USERTOKEN} ${layout}"/>
         </File>
         <File name="LOCALHOST"
-                     fileName="${logdir}/localhost.log">
+                     fileName="/tmp/logpipe">
             <PatternLayout pattern="tomee localhost.log ${env:ENV} ${env:USERTOKEN} ${layout}"/>
         </File>
 

container_files/usr-local-bin/daemon

@@ -6,4 +6,4 @@ prepDaemon
 
 export GSH_JVMARGS="-DENV=$ENV -DUSERTOKEN=$USERTOKEN"
 
-exec bin/gsh -loader
+exec bin/gsh -loader > /tmp/loggrouper

container_files/usr-local-bin/gsh

@@ -4,4 +4,4 @@
 
 prepDaemon
 
-exec bin/gsh "$@"
+exec bin/gsh "$@" | tee /tmp/loggrouper

container_files/usr-local-bin/library.sh

@@ -4,6 +4,23 @@
 mkfifo -m 666 /tmp/logpipe
 cat <> /tmp/logpipe 1>&2 &
 
+# Make loggers pipes for the supervisord connected apps' console, so that we can prepend the streams.
+mkfifo -m 666 /tmp/loggrouper
+(cat <> /tmp/loggrouper | awk -v ENV="$ENV" -v UT="$USERTOKEN" '{printf "grouper console %s %s %s\n", ENV, UT, $0; fflush()}' 1>/tmp/logpipe) &
+
+mkfifo -m 666 /tmp/loghttpd
+(cat <> /tmp/loghttpd  | awk -v ENV="$ENV" -v UT="$USERTOKEN" '{printf "httpd console %s %s %s\n", ENV, UT, $0; fflush()}' 1>/tmp/logpipe) &
+
+mkfifo -m 666 /tmp/logshibd
+(cat <> /tmp/logshibd | awk -v ENV="$ENV" -v UT="$USERTOKEN" '{printf "shibd console %s %s %s", ENV, UT, $0; fflush()}' 1>/tmp/logpipe) &
+
+mkfifo -m 666 /tmp/logtomcat
+(cat <> /tmp/logtomcat | awk -v ENV="$ENV" -v UT="$USERTOKEN" '{printf "tomcat console %s %s %s\n", ENV, UT, $0; fflush()}' 1>/tmp/logpipe) &
+
+mkfifo -m 666 /tmp/logsuperd
+(cat <> /tmp/logsuperd | awk -v ENV="$ENV" -v UT="$USERTOKEN" '{printf "supervisord console %s %s %s\n", ENV, UT, $0; fflush()}' 1>/tmp/logpipe) &
+
+
 linkGrouperSecrets() {
     for filepath in /run/secrets/*; do
         local label_file=`basename $filepath`

test-compose/data/Dockerfile

@@ -11,20 +11,20 @@ RUN yum install -y epel-release \
     && yum clean all
 
 RUN mysql_install_db \
-  && chown -R mysql:mysql /var/lib/mysql/ \
-  && sed -i 's/^\(bind-address\s.*\)/# \1/' /etc/my.cnf \
-  && sed -i 's/^\(log_error\s.*\)/# \1/' /etc/my.cnf \
-  && sed -i 's/\[mysqld\]/\[mysqld\]\ncharacter_set_server = utf8/' /etc/my.cnf \
-  && sed -i 's/\[mysqld\]/\[mysqld\]\ncollation_server = utf8_general_ci/' /etc/my.cnf \
-  && sed -i 's/\[mysqld\]/\[mysqld\]\nport = 3306/' /etc/my.cnf \
-  && cat  /etc/my.cnf \
-  && echo "/usr/bin/mysqld_safe &" > /tmp/config \
-  && echo "mysqladmin --silent --wait=30 ping || exit 1" >> /tmp/config \
-  && echo "mysql -e 'GRANT ALL PRIVILEGES ON *.* TO \"root\"@\"%\" WITH GRANT OPTION;'" >> /tmp/config \
-  && echo "mysql -e 'CREATE DATABASE grouper CHARACTER SET utf8 COLLATE utf8_bin;'" >> /tmp/config \
-  && bash /tmp/config \
-  && rm -f /tmp/config \
-  && mysql grouper < /seed-data/sisData.sql
+    && chown -R mysql:mysql /var/lib/mysql/ \
+    && sed -i 's/^\(bind-address\s.*\)/# \1/' /etc/my.cnf \
+    && sed -i 's/^\(log_error\s.*\)/# \1/' /etc/my.cnf \
+    && sed -i 's/\[mysqld\]/\[mysqld\]\ncharacter_set_server = utf8/' /etc/my.cnf \
+    && sed -i 's/\[mysqld\]/\[mysqld\]\ncollation_server = utf8_general_ci/' /etc/my.cnf \
+    && sed -i 's/\[mysqld\]/\[mysqld\]\nport = 3306/' /etc/my.cnf \
+    && cat  /etc/my.cnf \
+    && echo "/usr/bin/mysqld_safe &" > /tmp/config \
+    && echo "mysqladmin --silent --wait=30 ping || exit 1" >> /tmp/config \
+    && echo "mysql -e 'GRANT ALL PRIVILEGES ON *.* TO \"root\"@\"%\" WITH GRANT OPTION;'" >> /tmp/config \
+    && echo "mysql -e 'CREATE DATABASE grouper CHARACTER SET utf8 COLLATE utf8_bin;'" >> /tmp/config \
+    && bash /tmp/config \
+    && rm -f /tmp/config \
+    && mysql grouper < /seed-data/sisData.sql
 
 RUN useradd ldapadmin \
     && rm -fr /var/lock /usr/lib/systemd/system \
@@ -42,7 +42,7 @@ RUN useradd ldapadmin \
 RUN (/usr/sbin/ns-slapd -D /etc/dirsrv/slapd-dir &) \
     && while ! curl -s ldap://localhost:389 > /dev/null; do echo waiting for ldap to start; sleep 1; done; \
     (mysqld_safe & ) \
-    && while ! curl -s localhost:3306 > /dev/null; do echo waiting for mysqld to start; sleep 3; done; \
+    && while ! curl -s localhost:3306 > /dev/null; do echo waiting for mysqld to start; sleep 1; done; \
     bin/gsh -registry -check -runscript -noprompt \
     && bin/gsh /seed-data/bootstrap.gsh