Adding tier's logging requirements

.dockerignore

@@ -2,4 +2,5 @@
 test-compose/
 *.md
 manualBuild.sh
+Jenkinsfile
 LICENSE

Dockerfile

@@ -54,24 +54,30 @@ COPY --from=installing /opt/grouper/$GROUPER_VERSION/grouper.ws-$GROUPER_VERSION
 COPY --from=installing /opt/grouper/$GROUPER_VERSION/apache-tomcat-$TOMCAT_VERSION/ /opt/tomcat/
 COPY --from=installing /opt/grouper/$GROUPER_VERSION/apache-tomee-webprofile-$TOMEE_VERSION/ /opt/tomee/
 
+ADD http://central.maven.org/maven2/org/apache/logging/log4j/log4j-core/2.11.0/log4j-core-2.11.0.jar /opt/tomcat/bin
+ADD http://central.maven.org/maven2/org/apache/logging/log4j/log4j-api/2.11.0/log4j-api-2.11.0.jar /opt/tomcat/bin
+ADD http://central.maven.org/maven2/org/apache/logging/log4j/log4j-jul/2.11.0/log4j-jul-2.11.0.jar /opt/tomcat/bin
+
+ADD http://central.maven.org/maven2/org/apache/logging/log4j/log4j-core/2.11.0/log4j-core-2.11.0.jar /opt/tomee/bin
+ADD http://central.maven.org/maven2/org/apache/logging/log4j/log4j-api/2.11.0/log4j-api-2.11.0.jar /opt/tomee/bin
+ADD http://central.maven.org/maven2/org/apache/logging/log4j/log4j-jul/2.11.0/log4j-jul-2.11.0.jar /opt/tomee/bin
+
 RUN cd /opt/grouper/grouper.apiBinary/; \
     rm -fr ddlScripts/ grouper.lck grouper.log grouper.script grouper.tmp/ gshAddGrouperSystemWsGroup.gsh logs/
 
 RUN cd /opt/tomcat/; \
-    rm -fr webapps/docs/ webapps/examples/ webapps/host-manager/ webapps/manager/ logs/* temp/* work/* \
-    && mkdir -p logs/grouperUi logs/grouperWs
+    chmod +r bin/log4j-*.jar; \
+    rm -fr webapps/docs/ webapps/examples/ webapps/host-manager/ webapps/manager/ logs/* temp/* work/* conf/logging.properties
 
 RUN cd /opt/tomee/; \
-    rm -fr webapps/docs/ webapps/host-manager/ webapps/manager/ logs/* temp/* work/*
-
-RUN sed -i "s/\/opt\/grouper\/$GROUPER_VERSION\/apache-tomcat-$TOMCAT_VERSION/\/opt\/tomcat/g" /opt/grouper/grouper.ui/WEB-INF/classes/log4j.properties \
-    && sed -i "s/\/opt\/grouper\/$GROUPER_VERSION\/apache-tomcat-$TOMCAT_VERSION/\/opt\/tomcat/g" /opt/grouper/grouper.ws/WEB-INF/classes/log4j.properties \
-    && sed -i 's/${grouper.home}/\/opt\/tomee\//g' /opt/grouper/grouper.scim/WEB-INF/classes/log4j.properties
+    chmod +r bin/log4j-*.jar; \
+    rm -fr webapps/docs/ webapps/host-manager/ webapps/manager/ logs/* temp/* work/* conf/logging.properties
 
+COPY container_files/api/* /opt/grouper/grouper.apiBinary/conf/
+COPY container_files/ui/ /opt/grouper/grouper.ui/WEB-INF/
+COPY container_files/ws/ /opt/grouper/grouper.ws/WEB-INF/
 COPY container_files/tomcat/ /opt/tomcat/
 COPY container_files/tomee/ /opt/tomee/
-COPY container_files/ui/* /opt/grouper/grouper.ui/WEB-INF/
-
 
 
 FROM tier/shibboleth_sp
@@ -106,12 +112,17 @@ RUN groupadd -r tomcat \
     && chown -R tomcat:tomcat /opt/tomee/logs/ /opt/tomee/temp/ /opt/tomee/work/
 
 COPY container_files/tier-support/ /opt/tier-support/
-COPY container_files/usr-local-bin /usr/local/bin/
+COPY container_files/usr-local-bin/ /usr/local/bin/
 COPY container_files/httpd/* /etc/httpd/conf.d/
 COPY container_files/shibboleth/* /etc/shibboleth/
 
 RUN cp /dev/null /etc/httpd/conf.d/ssl.conf \
-    && touch /etc/pki/tls/certs/cachain.pem
+    && sed -i 's/LogFormat "/LogFormat "httpd access_log %{ENV}e %{USERTOKEN}e /g' /etc/httpd/conf/httpd.conf \
+    && echo -e "\nErrorLogFormat \"httpd error_log %{ENV}e %{USERTOKEN}e [%{u}t] [%-m:%l] [pid %P:tid %T] %7F: %E: [client\ %a] %M% ,\ referer\ %{Referer}i\"" >> /etc/httpd/conf/httpd.conf \
+    && sed -i 's/CustomLog "logs\/access_log"/CustomLog "\/tmp\/logpipe"/g' /etc/httpd/conf/httpd.conf \
+    && sed -i 's/ErrorLog "logs\/error_log"/ErrorLog "\/tmp\/logpipe"/g' /etc/httpd/conf/httpd.conf \
+    && echo -e "\nPassEnv ENV" >> /etc/httpd/conf/httpd.conf \
+    && echo -e "\nPassEnv USERTOKEN" >> /etc/httpd/conf/httpd.conf
 
 WORKDIR /opt/grouper/grouper.apiBinary/
 

README.md

@@ -235,7 +235,23 @@ $ docker run -it --rm \
   tier/grouper gsh -registry -check -runscript -noprompt
 ```
 
-Note: a less privileged database user maybe used when running the typical Grouper roles. This user need SELECT, INSERT, UPDATE, and DELETE privileges on the schema objects.
+Note: a less privileged database user maybe used when running the typical Grouper roles. This user needs SELECT, INSERT, UPDATE, and DELETE privileges on the schema objects.
+
+# Logging
+
+This image outputs logs in a manner that is consistent with Docker Logging. Each log entry is prefaced with the submodule name (e.g. shibd, httpd, tomcat, grouper), the logfile name (e.g. access_log, grouper_error.log, catalina.out) and user definable environment name and a user definable token. Content found after the preface will be specific to the application ands its logging configuration.
+
+> Note: If customizing a particular component's logging, it is recommended that the file be source from the image (`docker container cp`) or from the image's source repository. 
+
+To assign the "environment" string, set the environment variable `ENV` when defining the Docker service. For the "user defined token" string, use the environment variable of `USERTOKEN`.
+
+An example might look like the following, with the env of "dev" and the usertoken of "build-2"
+
+```text
+shibd shibd.log dev build-2 2018-03-27 20:42:22 INFO Shibboleth.Listener : listener service starting
+grouper-api grouper_event.log dev build-2 2018-03-27 21:10:00,046: [DefaultQuartzScheduler_Worker-1] INFO  EventLog.info(156) -  - [fdbb0099fe9e46e5be4371eb11250d39,'GrouperSystem','application'] session: start (0ms)
+tomcat console dev build-2 Grouper starting up: version: 2.3.0, build date: null, env: <no label configured>
+``` 
 
 # Misc Notes
 

container_files/api/log4j.properties

@@ -0,0 +1,144 @@
+
+#
+# Copyright 2014 Internet2
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#${grouper.home} will be substituted with the System property "grouper.home", which must have a trailing \ or / 
+# depending on your OS. Of course you can use absolute paths if you prefer 
+
+
+#
+# log4j Configuration
+# $Id: log4j.example.properties,v 1.13 2009-12-18 13:56:51 tzeller Exp $
+#
+
+# Appenders
+
+## Grouper API event logging
+log4j.appender.grouper_event                            = org.apache.log4j.FileAppender
+log4j.appender.grouper_event.file                       = /tmp/logpipe
+log4j.appender.grouper_event.append                     = true
+log4j.appender.grouper_event.layout                     = org.apache.log4j.PatternLayout
+log4j.appender.grouper_event.layout.ConversionPattern   = grouper-api grouper_event.log ${ENV} ${USERTOKEN} %d{ISO8601}: [%t] %-5p %C{1}.%M(%L) - %x - %m%n
+
+## Grouper API error logging
+log4j.appender.grouper_error                            = org.apache.log4j.FileAppender
+log4j.appender.grouper_error.file                       = /tmp/logpipe
+log4j.appender.grouper_errot.append                     = true
+log4j.appender.grouper_error.layout                     = org.apache.log4j.PatternLayout
+log4j.appender.grouper_error.layout.ConversionPattern   = grouper-api grouper_error.log ${ENV} ${USERTOKEN} %d{ISO8601}: [%t] %-5p %C{1}.%M(%L) - %x - %m%n
+#log4j.appender.grouper_error.layout.ConversionPattern   = %d{ISO8601}: %m%n
+
+# Debug logging (Or: logging that I haven't cleaned up yet to send elsewhere)
+log4j.appender.grouper_debug                            = org.apache.log4j.FileAppender
+log4j.appender.grouper_debug.file                       = /tmp/logpipe
+log4j.appender.grouper_debug.append                     = true
+log4j.appender.grouper_debug.layout                     = org.apache.log4j.PatternLayout
+#log4j.appender.grouper_debug.layout.ConversionPattern   = %d{ISO8601} %5p %c{2}: %m%n
+log4j.appender.grouper_debug.layout.ConversionPattern   = grouper-api grouper_debug.log ${ENV} ${USERTOKEN} %d{ISO8601}: [%t] %-5p %C{1}.%M(%L) - %x - %m%n
+
+## Benchmark logging
+log4j.appender.grouper_gb                               = org.apache.log4j.FileAppender
+log4j.appender.grouper_gb.file                          = /tmp/logpipe
+log4j.appender.grouper_gb.append                        = true
+log4j.appender.grouper_gb.layout                        = org.apache.log4j.PatternLayout
+#log4j.appender.grouper_gb.layout.ConversionPattern      = %d{ISO8601} %5p %c{2}: %m%n
+log4j.appender.grouper_gb.layout.ConversionPattern      = grouper-api grouper_bench.log ${ENV} ${USERTOKEN} %d{ISO8601}: [%t] %-5p %C{1}.%M(%L) - %x - %m%n
+
+# Loggers
+
+## Default logger; will log *everything*
+log4j.rootLogger  = ERROR, grouper_error
+
+## All Internet2 (warn to grouper_error per default logger)
+log4j.logger.edu.internet2.middleware = WARN
+
+
+# Provisioning : PSP (version 2.1+)
+log4j.logger.edu.internet2.middleware.psp = INFO
+
+# Provisioning : vt-ldap
+# log4j.logger.edu.vt.middleware.ldap = INFO
+
+# Provisioning : Grouper plugin to Shibboleth attribute resolver
+# log4j.logger.edu.internet2.middleware.grouper.shibboleth = INFO
+
+
+# For more precise (or verbose) logging, enable one or more of the
+# following logging directives.  To remove duplicate entries, just change the 
+# level, and not where to send the logs
+# http://robertmarkbramprogrammer.blogspot.com/2007/06/log4j-duplicate-lines-in-output.html
+
+## Grouper Event Logging
+## * Logs at _info_ only
+log4j.logger.edu.internet2.middleware.grouper.log.EventLog        = INFO, grouper_event
+log4j.logger.edu.internet2.middleware.grouper.RegistryInstall = INFO, grouper_event
+
+## Grouper Error Logging
+## * Logs at _warn_, _fatal_ and _error_ only (by default this is WARN due to internet2 below)
+#log4j.logger.edu.internet2.middleware.grouper              = WARN, grouper_error
+
+## Grouper Debug Logging
+## * NOTE: There is currently VERY LITTLE (useful) information sent to this.
+## * Logs at _info_ only currently
+#log4j.logger.edu.internet2.middleware.grouper              = INFO, grouper_debug
+
+## Grouper XML Export + Import Logging
+## TODO Integrate with normal logging
+log4j.logger.edu.internet2.middleware.grouper.xml.XmlExporter           = INFO, grouper_event
+log4j.logger.edu.internet2.middleware.grouper.xml.XmlImporter           = INFO, grouper_event
+
+## Grouper Benchmark Logging
+log4j.logger.edu.internet2.middleware.grouper.bench                 = INFO, grouper_gb
+
+## Grouper script to add missing group sets
+log4j.logger.edu.internet2.middleware.grouper.misc.AddMissingGroupSets   = INFO, grouper_event
+
+## Grouper Sync Point in Time Tables
+log4j.logger.edu.internet2.middleware.grouper.misc.SyncPITTables   = INFO, grouper_event
+
+## Grouper Sync Stem Set Table
+log4j.logger.edu.internet2.middleware.grouper.misc.SyncStemSets      = INFO, grouper_event
+
+## Grouper Migrate Legacy Attributes
+log4j.logger.edu.internet2.middleware.grouper.misc.MigrateLegacyAttributes = INFO, grouper_event
+
+### Subject API
+#log4j.logger.edu.internet2.middleware.subject                       = ERROR, grouper_error
+#log4j.logger.edu.internet2.middleware.subject.provider              = ERROR, grouper_error
+### Hibernate 
+#log4j.logger.org.hibernate                                          = ERROR, grouper_error
+### ehcache
+#log4j.logger.net.sf.ehcache                                         = ERROR, grouper_error
+### Spring
+#log4j.logger.org.springframework                                    = ERROR, grouper_error
+
+## Grouper Stress Testing
+log4j.logger.edu.internet2.middleware.grouper.stress                = INFO, grouper_debug
+
+
+#######################################################
+##Optional settings for debug logs
+#######################################################
+
+## Hooks debug info
+#log4j.logger.edu.internet2.middleware.grouper.hooks.examples.GroupTypeTupleIncludeExcludeHook = DEBUG
+#log4j.logger.edu.internet2.middleware.grouper.Group = DEBUG
+
+#log4j.logger.edu.internet2.middleware.grouper.hooks.examples.GroupTypeSecurityHook = DEBUG
+
+
+# added by grouper-installer
+log4j.logger.org.apache.tools.ant = WARN

container_files/shibboleth/shibd.logger

@@ -0,0 +1,59 @@
+# set overall behavior
+log4j.rootCategory=INFO, shibd_log
+
+# fairly verbose for DEBUG, so generally leave at INFO
+log4j.category.XMLTooling.XMLObject=INFO
+log4j.category.XMLTooling.KeyInfoResolver=INFO
+log4j.category.Shibboleth.IPRange=INFO
+log4j.category.Shibboleth.PropertySet=INFO
+
+# raise for low-level tracing of SOAP client HTTP/SSL behavior
+log4j.category.XMLTooling.libcurl=INFO
+
+# useful categories to tune independently:
+#
+# tracing of SAML messages and security policies
+#log4j.category.OpenSAML.MessageDecoder=DEBUG
+#log4j.category.OpenSAML.MessageEncoder=DEBUG
+#log4j.category.OpenSAML.SecurityPolicyRule=DEBUG
+#log4j.category.XMLTooling.SOAPClient=DEBUG
+# interprocess message remoting
+#log4j.category.Shibboleth.Listener=DEBUG
+# mapping of requests to applicationId
+#log4j.category.Shibboleth.RequestMapper=DEBUG
+# high level session cache operations
+#log4j.category.Shibboleth.SessionCache=DEBUG
+# persistent storage and caching
+#log4j.category.XMLTooling.StorageService=DEBUG
+
+# logs XML being signed or verified if set to DEBUG
+log4j.category.XMLTooling.Signature.Debugger=INFO, sig_log
+log4j.additivity.XMLTooling.Signature.Debugger=false
+
+# the tran log blocks the "default" appender(s) at runtime
+# Level should be left at INFO for this category
+log4j.category.Shibboleth-TRANSACTION=INFO, tran_log
+log4j.additivity.Shibboleth-TRANSACTION=false
+# uncomment to suppress particular event types
+#log4j.category.Shibboleth-TRANSACTION.AuthnRequest=WARN
+#log4j.category.Shibboleth-TRANSACTION.Login=WARN
+#log4j.category.Shibboleth-TRANSACTION.Logout=WARN
+
+# define the appenders
+
+log4j.appender.shibd_log=org.apache.log4j.FileAppender
+log4j.appender.shibd_log.fileName=/tmp/logpipe
+log4j.appender.shibd_log.maxFileSize=0
+log4j.appender.shibd_log.layout=org.apache.log4j.PatternLayout
+log4j.appender.shibd_log.layout.ConversionPattern=shibd shibd.log ${ENV} ${USERTOKEN} %d{%Y-%m-%d %H:%M:%S} %p %c %x: %m%n
+
+log4j.appender.tran_log=org.apache.log4j.FileAppender
+log4j.appender.tran_log.fileName=/tmp/logpipe
+log4j.appender.tran_log.maxFileSize=0
+log4j.appender.tran_log.layout=org.apache.log4j.PatternLayout
+log4j.appender.tran_log.layout.ConversionPattern=shibd transaction.log ${ENV} ${USERTOKEN} %d{%Y-%m-%d %H:%M:%S} %p %c %x: %m%n
+
+log4j.appender.sig_log=org.apache.log4j.FileAppender
+log4j.appender.sig_log.fileName=/tmp/logpipe
+log4j.appender.sig_log.layout=org.apache.log4j.PatternLayout
+log4j.appender.sig_log.layout.ConversionPattern=shibd signature.log ${ENV} ${USERTOKEN} %m

container_files/tier-support/supervisord-tomcat.conf

@@ -1,5 +1,5 @@
 [supervisord]
-logfile=/dev/fd/1                               ; supervisord log file
+logfile=/tmp/logsuperd                              ; supervisord log file
 logfile_maxbytes=0                           ; maximum size of logfile before rotation
 loglevel=error                                  ; info, debug, warn, trace
 nodaemon=true                                  ; run supervisord as a daemon
@@ -16,24 +16,24 @@ serverurl=unix:///tmp/supervisor.sock         ; use a unix:// URL  for a unix so
 
 [program:httpd]
 command=httpd -DFOREGROUND
-stderr_logfile = /dev/fd/2
+stderr_logfile = /tmp/loghttpd
 stderr_logfile_maxbytes=0
-stdout_logfile = /dev/fd/1
+stdout_logfile = /tmp/loghttpd
 stdout_logfile_maxbytes=0
 
 [program:shibbolethsp]
 user=shibd
 command=/usr/sbin/shibd -f -F
-stderr_logfile = /dev/fd/2
+stderr_logfile = /tmp/logshidb
 stderr_logfile_maxbytes=0
-stdout_logfile = /dev/fd/1
+stdout_logfile = /tmp/logshidb
 stdout_logfile_maxbytes=0
 
 [program:tomcat]
 user=tomcat
 command=/opt/tomcat/bin/catalina.sh run 
-stderr_logfile = /dev/fd/2
+stderr_logfile = /tmp/logtomcat
 stderr_logfile_maxbytes=0
-stdout_logfile = /dev/fd/1
+stdout_logfile = /tmp/logtomcat
 stdout_logfile_maxbytes=0
 

container_files/tier-support/supervisord-tomee.conf

@@ -1,5 +1,5 @@
 [supervisord]
-logfile=/dev/fd/1                               ; supervisord log file
+logfile=/tmp/logpipe                               ; supervisord log file
 logfile_maxbytes=0                           ; maximum size of logfile before rotation
 loglevel=error                                  ; info, debug, warn, trace
 nodaemon=true                                  ; run supervisord as a daemon
@@ -16,16 +16,16 @@ serverurl=unix:///tmp/supervisor.sock         ; use a unix:// URL  for a unix so
 
 [program:httpd]
 command=httpd -DFOREGROUND
-stderr_logfile = /dev/fd/2
+stderr_logfile = /tmp/logpipe
 stderr_logfile_maxbytes=0
-stdout_logfile = /dev/fd/1
+stdout_logfile = /tmp/logpipe
 stdout_logfile_maxbytes=0
 
 [program:tomee]
 user=tomcat
 command=/opt/tomee/bin/catalina.sh run 
-stderr_logfile = /dev/fd/2
+stderr_logfile = /tmp/logpipe
 stderr_logfile_maxbytes=0
-stdout_logfile = /dev/fd/1
+stdout_logfile = /tmp/logpipe
 stdout_logfile_maxbytes=0
 

container_files/tomcat/bin/setenv.sh

@@ -0,0 +1,3 @@
+CLASSPATH=/opt/tomcat/bin/*
+JAVA_OPTS="-Dlog4j.configurationFile=/opt/tomcat/conf/log4j2.xml -DENV=$ENV -DUSERTOKEN=$USERTOKEN"
+LOGGING_MANAGER=-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager

container_files/tomcat/conf/log4j2.xml

@@ -0,0 +1,26 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Configuration status="info">
+    <Properties>
+        <Property name="layout">%d [%t] %-5p %c- %m%n</Property>
+    </Properties>
+    <Appenders>
+        <File name="CATALINA"
+                     fileName="/tmp/logpipe">
+            <PatternLayout pattern="tomcat catalina.out ${env:ENV} ${env:USERTOKEN} ${layout}"/>
+        </File>
+        <File name="LOCALHOST"
+                     fileName="/tmp/logpipe">
+            <PatternLayout pattern="tomcat localhost.log ${env:ENV} ${env:USERTOKEN} ${layout}"/>
+        </File>
+
+    </Appenders>
+    <Loggers>
+        <Root level="info">
+            <AppenderRef ref="CATALINA"/>
+        </Root>
+        <Logger name="org.apache.catalina.core.ContainerBase.[Catalina].[localhost]"
+                level="info" additivity="false">
+            <AppenderRef ref="LOCALHOST"/>
+        </Logger>
+    </Loggers>
+</Configuration>