Skip to content

Commit

Permalink
Support for TIER standard logging format
Browse files Browse the repository at this point in the history
Added support for the TIER standard logging format including
the ability to inject using the ENV and USERTOKEN environment
variables.
  • Loading branch information
skoranda committed May 3, 2018
1 parent 53ca9f8 commit 1ec2ec9
Show file tree
Hide file tree
Showing 9 changed files with 171 additions and 47 deletions.
13 changes: 9 additions & 4 deletions comanage-registry-internet2-tier/000-comanage.conf
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,8 @@

Listen 443
ServerName %%SERVER_NAME%%
PassEnv ENV
PassEnv USERTOKEN

<VirtualHost *:80>
ServerName http://%%SERVER_NAME%%:80
Expand All @@ -31,16 +33,19 @@ RewriteRule ^ https://%{HTTP_HOST}:443%{REQUEST_URI} [R=302,L,QSA]

ServerName https://%%SERVER_NAME%%:443

PassEnv ENV
PassEnv USERTOKEN

DocumentRoot /var/www/html

RedirectMatch ^/$ /registry/

ErrorLogFormat "httpd ssl_error_log [%{u}t] [%-m:%l] [pid %P:tid %T] %7F: %E: [client\ %a] %M% ,\ referer\ %{Referer}i"
ErrorLog logs/ssl_error_log
ErrorLogFormat "httpd;ssl_error_log;%{ENV}e;%{USERTOKEN}e;[%{u}t] [%-m:%l] [pid %P:tid %T] %7F: %E: [client\ %a] %M% ,\ referer\ %{Referer}i"
ErrorLog /tmp/logpipe
LogLevel warn

LogFormat "httpd ssl_access_log %h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" ssl_combined
CustomLog logs/ssl_access_log ssl_combined
LogFormat "httpd;ssl_access_log;%{ENV}e;%{USERTOKEN}e;%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" ssl_combined
CustomLog /tmp/logpipe ssl_combined

Header always set Strict-Transport-Security "max-age=63072000; includeSubDomains"

Expand Down
18 changes: 8 additions & 10 deletions comanage-registry-internet2-tier/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -98,10 +98,13 @@ RUN yum -y update && yum -y install \
mod_ssl \
mariadb-libs \
postgresql \
python-pip \
shibboleth \
supervisor \
wget \
zlib
zlib \
&& pip install --upgrade pip \
&& pip install supervisor \
&& yum clean -y all

COPY --from=php-build /usr/lib64/httpd/modules/libphp7.so /usr/lib64/httpd/modules/
COPY --from=php-build /usr/local/lib/php /usr/local/lib/php/
Expand Down Expand Up @@ -138,13 +141,7 @@ COPY shibd.logger /etc/shibboleth/
COPY native.logger /etc/shibboleth/
COPY httpd.conf /etc/httpd/conf/

RUN ln -sfT /dev/stdout /var/log/httpd/access_log \
&& ln -sfT /dev/stdout /var/log/httpd/error_log \
&& ln -sfT /dev/stdout /var/log/httpd/ssl_error_log \
&& ln -sfT /dev/stdout /var/log/httpd/ssl_access_log \
&& ln -sfT /dev/stdout /var/log/shibboleth-www/native.log \
&& ln -sfT /dev/stdout /var/log/shibboleth-www/native_warn.log \
&& cd /etc/httpd/conf.d \
RUN cd /etc/httpd/conf.d \
&& rm -f autoindex.conf ssl.conf userdir.conf welcome.conf \
&& cd /etc/httpd \
&& ln -s /etc/pki/tls/certs/localhost.crt cert.pem \
Expand Down Expand Up @@ -173,5 +170,6 @@ VOLUME ["/etc/shibboleth"]

COPY docker-comanage-entrypoint /usr/local/bin/
COPY docker-comanage-shibboleth-sp-entrypoint /usr/local/bin/
COPY docker-supervisord-entrypoint /usr/local/bin/

ENTRYPOINT ["/usr/bin/supervisord", "-c", "/usr/local/etc/supervisord.conf"]
ENTRYPOINT ["docker-supervisord-entrypoint"]
34 changes: 30 additions & 4 deletions comanage-registry-internet2-tier/docker-comanage-entrypoint
Original file line number Diff line number Diff line change
Expand Up @@ -43,9 +43,11 @@ injectable_config_vars=(
COMANAGE_REGISTRY_EMAIL_ACCOUNT_PASSWORD
COMANAGE_REGISTRY_SECURITY_SALT
COMANAGE_REGISTRY_SECURITY_SEED
ENV
HTTPS_CERT_FILE
HTTPS_PRIVKEY_FILE
SERVER_NAME
USERTOKEN
)

# If the file associated with a configuration variable is present then
Expand Down Expand Up @@ -312,13 +314,37 @@ fi
# Configure Apache HTTP Server with the server name.
sed -i -e s@%%SERVER_NAME%%@"${SERVER_NAME:-unknown}"@g /etc/httpd/conf.d/000-comanage.conf

# If ENV or USERTOKEN as injected by the deployer contain a semi-colon remove it.
if [[ $ENV =~ .*";".* ]]; then
ENV=`echo $ENV | tr -d ';'`
export ENV
fi

if [[ $USERTOKEN =~ .*";".* ]]; then
USERTOKEN=`echo $USERTOKEN | tr -d ';'`
export USERTOKEN
fi

# If ENV or USERTOKEN as injected by the deployer contain a space remove it.
if [[ $ENV =~ [[:space:]] ]]; then
ENV=`echo $ENV | tr -d [:space:]`
export ENV
fi

if [[ $USERTOKEN =~ [[:space:]] ]]; then
USERTOKEN=`echo $USERTOKEN | tr -d [:space:]`
export USERTOKEN
fi

# Create pipes to use for COmanage Registry instead of standard log files.
mkfifo -m 666 /srv/comanage-registry/app/tmp/logs/error.log
mkfifo -m 666 /srv/comanage-registry/app/tmp/logs/debug.log
rm -f "$COMANAGE_REGISTRY_DIR/app/tmp/logs/error.log" > "$OUTPUT" 2>&1
rm -f "$COMANAGE_REGISTRY_DIR/app/tmp/logs/debug.log" > "$OUTPUT" 2>&1
mkfifo -m 666 "$COMANAGE_REGISTRY_DIR/app/tmp/logs/error.log" > "$OUTPUT" 2>&1
mkfifo -m 666 "$COMANAGE_REGISTRY_DIR/app/tmp/logs/debug.log" > "$OUTPUT" 2>&1

# Format any output from COmanange Registry into standard TIER form.
(cat <> /srv/comanage-registry/app/tmp/logs/error.log | awk -v ENV="$ENV" -v UT="$USERTOKEN" '{printf "COmanage Registry error.log %s %s %s\n", ENV, UT, $0; fflush()}' 1>/dev/stdout)&
(cat <> /srv/comanage-registry/app/tmp/logs/debug.log | awk -v ENV="$ENV" -v UT="$USERTOKEN" '{printf "COmanage Registry debug.log %s %s %s\n", ENV, UT, $0; fflush()}' 1>/dev/stdout)&
(cat <> "$COMANAGE_REGISTRY_DIR/app/tmp/logs/error.log" | awk -v ENV="$ENV" -v UT="$USERTOKEN" '{printf "comanage_registry;error.log;%s;%s;%s\n", ENV, UT, $0; fflush()}' 1>/tmp/logpipe)&
(cat <> "$COMANAGE_REGISTRY_DIR/app/tmp/logs/debug.log" | awk -v ENV="$ENV" -v UT="$USERTOKEN" '{printf "comanage_registry;debug.log;%s;%s;%s\n", ENV, UT, $0; fflush()}' 1>/tmp/logpipe)&

# Start Apache HTTP Server
exec /usr/sbin/httpd -DFOREGROUND
Original file line number Diff line number Diff line change
Expand Up @@ -100,15 +100,27 @@ if [ -n "$SHIBBOLETH_SP_PRIVKEY" ]; then
chmod 0600 /etc/shibboleth/sp-key.pem
fi

# Send logging to container stdout.
pushd /var/log/shibboleth
/bin/rm -f shibd.log transaction.log shibd_warn.log signature.log
/bin/ln -sfT /dev/stdout shibd.log
/bin/ln -sfT /dev/stdout shibd_warn.log
/bin/ln -sfT /dev/stdout transaction.log
/bin/ln -sfT /dev/stdout signature.log
chown shibd:shibd shibd.log transaction.log shibd_warn.log signature.log
popd
# If ENV or USERTOKEN as injected by the deployer contain a semi-colon remove it.
if [[ $ENV =~ .*";".* ]]; then
ENV=`echo $ENV | tr -d ';'`
export ENV
fi

if [[ $USERTOKEN =~ .*";".* ]]; then
USERTOKEN=`echo $USERTOKEN | tr -d ';'`
export USERTOKEN
fi

# If ENV or USERTOKEN as injected by the deployer contain a space remove it.
if [[ $ENV =~ [[:space:]] ]]; then
ENV=`echo $ENV | tr -d [:space:]`
export ENV
fi

if [[ $USERTOKEN =~ [[:space:]] ]]; then
USERTOKEN=`echo $USERTOKEN | tr -d [:space:]`
export USERTOKEN
fi

# Start the daemon.
export LD_LIBRARY_PATH=/opt/shibboleth/lib64
Expand Down
78 changes: 78 additions & 0 deletions comanage-registry-internet2-tier/docker-supervisord-entrypoint
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
#!/bin/bash

# COmanage Registry Dockerfile entrypoint
#
# Portions licensed to the University Corporation for Advanced Internet
# Development, Inc. ("UCAID") under one or more contributor license agreements.
# See the NOTICE file distributed with this work for additional information
# regarding copyright ownership.
#
# UCAID licenses this file to you under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with the
# License. You may obtain a copy of the License at:
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

if [ -n "$DEBUG" ]
then
OUTPUT=/dev/stdout
else
OUTPUT=/dev/null
fi

# If ENV or USERTOKEN as injected by the deployer contain a semi-colon remove it.
if [[ $ENV =~ .*";".* ]]; then
ENV=`echo $ENV | tr -d ';'`
export ENV
fi

if [[ $USERTOKEN =~ .*";".* ]]; then
USERTOKEN=`echo $USERTOKEN | tr -d ';'`
export USERTOKEN
fi

# If ENV or USERTOKEN as injected by the deployer contain a space remove it.
if [[ $ENV =~ [[:space:]] ]]; then
ENV=`echo $ENV | tr -d [:space:]`
export ENV
fi

if [[ $USERTOKEN =~ [[:space:]] ]]; then
USERTOKEN=`echo $USERTOKEN | tr -d [:space:]`
export USERTOKEN
fi

# Make a "console" logging pipe that anyone can write to regardless of who owns the process.
rm -f /tmp/logpipe > "$OUTPUT" 2>&1
mkfifo -m 666 /tmp/logpipe > "$OUTPUT" 2>&1
cat <> /tmp/logpipe &

# Format any console output from httpd into standard TIER form.
rm -f /tmp/loghttpd > "$OUTPUT" 2>&1
mkfifo -m 666 /tmp/loghttpd > "$OUTPUT" 2>&1
(cat <> /tmp/loghttpd | awk -v ENV="$ENV" -v UT="$USERTOKEN" '{printf "httpd;console;%s;%s;%s\n", ENV, UT, $0; fflush()}' 1>/tmp/logpipe 2>&1)&

# Format any console output from shibd into standard TIER form.
rm -f /tmp/logshibd > "$OUTPUT" 2>&1
mkfifo -m 666 /tmp/logshibd > "$OUTPUT" 2>&1
(cat <> /tmp/logshibd | awk -v ENV="$ENV" -v UT="$USERTOKEN" '{printf "httpd;console;%s;%s;%s\n", ENV, UT, $0; fflush()}' 1>/tmp/logpipe 2>&1)&

# Format any console output from supervisord into standard TIER form.
rm -f /tmp/logsuperd > "$OUTPUT" 2>&1
mkfifo -m 666 /tmp/logsuperd > "$OUTPUT" 2>&1
(cat <> /tmp/logsuperd | awk -v ENV="$ENV" -v UT="$USERTOKEN" '{printf "supervisord;console;%s;%s;%s\n", ENV, UT, $0; fflush()}' 1>/tmp/logpipe 2>&1)&

# Close stdout and stderr for this process since supervisord will write
# to its logfile and its children are configured to write to different
# pipes.
exec 1<&-
exec 2<&-

# Start supervisord
exec /usr/bin/supervisord -c /usr/local/etc/supervisord.conf
11 changes: 7 additions & 4 deletions comanage-registry-internet2-tier/httpd.conf
Original file line number Diff line number Diff line change
Expand Up @@ -49,12 +49,15 @@ DirectoryIndex index.html
Require all denied
</Files>

ErrorLogFormat "httpd error_log [%{u}t] [%-m:%l] [pid %P:tid %T] %7F: %E: [client\ %a] %M% ,\ referer\ %{Referer}i"
ErrorLog "logs/error_log"
PassEnv ENV
PassEnv USERTOKEN

ErrorLogFormat "httpd;error_log;%{ENV}e;%{USERTOKEN}e;[%{u}t] [%-m:%l] [pid %P:tid %T] %7F: %E: [client\ %a] %M% ,\ referer\ %{Referer}i"
ErrorLog "/tmp/logpipe"
LogLevel warn

LogFormat "httpd access_log %h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" combined
CustomLog "logs/access_log" combined
LogFormat "httpd;access_log;%{ENV}e;%{USERTOKEN}e;%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" combined
CustomLog "/tmp/logpipe" combined

TypesConfig /etc/mime.types
AddType application/x-compress .Z
Expand Down
8 changes: 4 additions & 4 deletions comanage-registry-internet2-tier/native.logger
Original file line number Diff line number Diff line change
Expand Up @@ -28,12 +28,12 @@ log4j.category.XMLTooling.libcurl=INFO
# define the appender

log4j.appender.native_log=org.apache.log4j.FileAppender
log4j.appender.native_log.fileName=/var/log/shibboleth-www/native.log
log4j.appender.native_log.fileName=/tmp/logpipe
log4j.appender.native_log.layout=org.apache.log4j.PatternLayout
log4j.appender.native_log.layout.ConversionPattern=shibd native.log %d{%Y-%m-%d %H:%M:%S} %p %c %x: %m%n
log4j.appender.native_log.layout.ConversionPattern=shibd;native.log;${ENV};${USERTOKEN};%d{%Y-%m-%d %H:%M:%S} %p %c %x: %m%n

log4j.appender.warn_log=org.apache.log4j.FileAppender
log4j.appender.warn_log.fileName=/var/log/shibboleth-www/native_warn.log
log4j.appender.warn_log.fileName=/tmp/logpipe
log4j.appender.warn_log.layout=org.apache.log4j.PatternLayout
log4j.appender.warn_log.layout.ConversionPattern=shibd native_warn.log %d{%Y-%m-%d %H:%M:%S} %p %c %x: %m%n
log4j.appender.warn_log.layout.ConversionPattern=shibd;native_warn.log;${ENV};${USERTOKEN};%d{%Y-%m-%d %H:%M:%S} %p %c %x: %m%n
log4j.appender.warn_log.threshold=WARN
16 changes: 8 additions & 8 deletions comanage-registry-internet2-tier/shibd.logger
Original file line number Diff line number Diff line change
Expand Up @@ -42,22 +42,22 @@ log4j.additivity.Shibboleth-TRANSACTION=false
# define the appenders

log4j.appender.shibd_log=org.apache.log4j.FileAppender
log4j.appender.shibd_log.fileName=/var/log/shibboleth/shibd.log
log4j.appender.shibd_log.fileName=/tmp/logpipe
log4j.appender.shibd_log.layout=org.apache.log4j.PatternLayout
log4j.appender.shibd_log.layout.ConversionPattern=shibd shibd.log %d{%Y-%m-%d %H:%M:%S} %p %c %x: %m%n
log4j.appender.shibd_log.layout.ConversionPattern=shibd;shibd.log;${ENV};${USERTOKEN};%d{%Y-%m-%d %H:%M:%S} %p %c %x: %m%n

log4j.appender.warn_log=org.apache.log4j.FileAppender
log4j.appender.warn_log.fileName=/var/log/shibboleth/shibd_warn.log
log4j.appender.warn_log.fileName=/tmp/logpipe
log4j.appender.warn_log.layout=org.apache.log4j.PatternLayout
log4j.appender.warn_log.layout.ConversionPattern=shibd shibd_warn.log %d{%Y-%m-%d %H:%M:%S} %p %c %x: %m%n
log4j.appender.warn_log.layout.ConversionPattern=shibd;shibd_warn.log;${ENV};${USERTOKEN};%d{%Y-%m-%d %H:%M:%S} %p %c %x: %m%n
log4j.appender.warn_log.threshold=WARN

log4j.appender.tran_log=org.apache.log4j.FileAppender
log4j.appender.tran_log.fileName=/var/log/shibboleth/transaction.log
log4j.appender.tran_log.fileName=/tmp/logpipe
log4j.appender.tran_log.layout=org.apache.log4j.PatternLayout
log4j.appender.tran_log.layout.ConversionPattern=shibd transaction.log %d{%Y-%m-%d %H:%M:%S} %p %c %x: %m%n
log4j.appender.tran_log.layout.ConversionPattern=shibd;transaction.log;${ENV};${USERTOKEN};%d{%Y-%m-%d %H:%M:%S} %p %c %x: %m%n

log4j.appender.sig_log=org.apache.log4j.FileAppender
log4j.appender.sig_log.fileName=/var/log/shibboleth/signature.log
log4j.appender.sig_log.fileName=/tmp/logpipe
log4j.appender.sig_log.layout=org.apache.log4j.PatternLayout
log4j.appender.sig_log.layout.ConversionPattern=shibd signature.log %m
log4j.appender.sig_log.layout.ConversionPattern=shibd;signature.log${ENV};${USERTOKEN};%m
10 changes: 6 additions & 4 deletions comanage-registry-internet2-tier/supervisord.conf
Original file line number Diff line number Diff line change
Expand Up @@ -18,19 +18,21 @@
; limitations under the License.

[supervisord]
logfile=/tmp/logsuperd
logfile_maxbytes=0
nodaemon=true
user=root

[program:httpd]
command=/usr/local/bin/docker-comanage-entrypoint
stdout_logfile=/dev/stdout
stdout_logfile=/tmp/loghttpd
stdout_logfile_maxbytes=0
stderr_logfile=/dev/stderr
stderr_logfile=/tmp/loghttpd
stderr_logfile_maxbytes=0

[program:shibd]
command=/usr/local/bin/docker-comanage-shibboleth-sp-entrypoint
stdout_logfile=/dev/stdout
stdout_logfile=/tmp/logshibd
stdout_logfile_maxbytes=0
stderr_logfile=/dev/stderr
stderr_logfile=/tmp/logshibd
stderr_logfile_maxbytes=0

0 comments on commit 1ec2ec9

Please sign in to comment.