Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
Support for TIER standard logging format
Added support for the TIER standard logging format including
the ability to inject using the ENV and USERTOKEN environment
variables.
skoranda committed May 3, 2018
1 parent 53ca9f8 commit 1ec2ec9
Showing 9 changed files with 171 additions and 47 deletions.
13 changes: 9 additions & 4 deletions comanage-registry-internet2-tier/000-comanage.conf
@@ -19,6 +19,8 @@

Listen 443
ServerName %%SERVER_NAME%%
PassEnv ENV
PassEnv USERTOKEN

<VirtualHost *:80>
ServerName http://%%SERVER_NAME%%:80
@@ -31,16 +33,19 @@ RewriteRule ^ https://%{HTTP_HOST}:443%{REQUEST_URI} [R=302,L,QSA]

ServerName https://%%SERVER_NAME%%:443

PassEnv ENV
PassEnv USERTOKEN

DocumentRoot /var/www/html

RedirectMatch ^/$ /registry/

ErrorLogFormat "httpd ssl_error_log [%{u}t] [%-m:%l] [pid %P:tid %T] %7F: %E: [client\ %a] %M% ,\ referer\ %{Referer}i"
ErrorLog logs/ssl_error_log
ErrorLogFormat "httpd;ssl_error_log;%{ENV}e;%{USERTOKEN}e;[%{u}t] [%-m:%l] [pid %P:tid %T] %7F: %E: [client\ %a] %M% ,\ referer\ %{Referer}i"
ErrorLog /tmp/logpipe
LogLevel warn

LogFormat "httpd ssl_access_log %h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" ssl_combined
CustomLog logs/ssl_access_log ssl_combined
LogFormat "httpd;ssl_access_log;%{ENV}e;%{USERTOKEN}e;%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" ssl_combined
CustomLog /tmp/logpipe ssl_combined

Header always set Strict-Transport-Security "max-age=63072000; includeSubDomains"

18 changes: 8 additions & 10 deletions comanage-registry-internet2-tier/Dockerfile
@@ -98,10 +98,13 @@ RUN yum -y update && yum -y install \
mod_ssl \
mariadb-libs \
postgresql \
python-pip \
shibboleth \
supervisor \
wget \
zlib
zlib \
&& pip install --upgrade pip \
&& pip install supervisor \
&& yum clean -y all

COPY --from=php-build /usr/lib64/httpd/modules/libphp7.so /usr/lib64/httpd/modules/
COPY --from=php-build /usr/local/lib/php /usr/local/lib/php/
@@ -138,13 +141,7 @@ COPY shibd.logger /etc/shibboleth/
COPY native.logger /etc/shibboleth/
COPY httpd.conf /etc/httpd/conf/

RUN ln -sfT /dev/stdout /var/log/httpd/access_log \
&& ln -sfT /dev/stdout /var/log/httpd/error_log \
&& ln -sfT /dev/stdout /var/log/httpd/ssl_error_log \
&& ln -sfT /dev/stdout /var/log/httpd/ssl_access_log \
&& ln -sfT /dev/stdout /var/log/shibboleth-www/native.log \
&& ln -sfT /dev/stdout /var/log/shibboleth-www/native_warn.log \
&& cd /etc/httpd/conf.d \
RUN cd /etc/httpd/conf.d \
&& rm -f autoindex.conf ssl.conf userdir.conf welcome.conf \
&& cd /etc/httpd \
&& ln -s /etc/pki/tls/certs/localhost.crt cert.pem \
@@ -173,5 +170,6 @@ VOLUME ["/etc/shibboleth"]

COPY docker-comanage-entrypoint /usr/local/bin/
COPY docker-comanage-shibboleth-sp-entrypoint /usr/local/bin/
COPY docker-supervisord-entrypoint /usr/local/bin/

ENTRYPOINT ["/usr/bin/supervisord", "-c", "/usr/local/etc/supervisord.conf"]
ENTRYPOINT ["docker-supervisord-entrypoint"]
34 changes: 30 additions & 4 deletions comanage-registry-internet2-tier/docker-comanage-entrypoint
@@ -43,9 +43,11 @@ injectable_config_vars=(
COMANAGE_REGISTRY_EMAIL_ACCOUNT_PASSWORD
COMANAGE_REGISTRY_SECURITY_SALT
COMANAGE_REGISTRY_SECURITY_SEED
ENV
HTTPS_CERT_FILE
HTTPS_PRIVKEY_FILE
SERVER_NAME
USERTOKEN
)

# If the file associated with a configuration variable is present then
@@ -312,13 +314,37 @@ fi
# Configure Apache HTTP Server with the server name.
sed -i -e s@%%SERVER_NAME%%@"${SERVER_NAME:-unknown}"@g /etc/httpd/conf.d/000-comanage.conf

# If ENV or USERTOKEN as injected by the deployer contain a semi-colon remove it.
if [[ $ENV =~ .*";".* ]]; then
ENV=`echo $ENV | tr -d ';'`
export ENV
fi

if [[ $USERTOKEN =~ .*";".* ]]; then
USERTOKEN=`echo $USERTOKEN | tr -d ';'`
export USERTOKEN
fi

# If ENV or USERTOKEN as injected by the deployer contain a space remove it.
if [[ $ENV =~ [[:space:]] ]]; then
ENV=`echo $ENV | tr -d [:space:]`
export ENV
fi

if [[ $USERTOKEN =~ [[:space:]] ]]; then
USERTOKEN=`echo $USERTOKEN | tr -d [:space:]`
export USERTOKEN
fi

# Create pipes to use for COmanage Registry instead of standard log files.
mkfifo -m 666 /srv/comanage-registry/app/tmp/logs/error.log
mkfifo -m 666 /srv/comanage-registry/app/tmp/logs/debug.log
rm -f "$COMANAGE_REGISTRY_DIR/app/tmp/logs/error.log" > "$OUTPUT" 2>&1
rm -f "$COMANAGE_REGISTRY_DIR/app/tmp/logs/debug.log" > "$OUTPUT" 2>&1
mkfifo -m 666 "$COMANAGE_REGISTRY_DIR/app/tmp/logs/error.log" > "$OUTPUT" 2>&1
mkfifo -m 666 "$COMANAGE_REGISTRY_DIR/app/tmp/logs/debug.log" > "$OUTPUT" 2>&1

# Format any output from COmanange Registry into standard TIER form.
(cat <> /srv/comanage-registry/app/tmp/logs/error.log | awk -v ENV="$ENV" -v UT="$USERTOKEN" '{printf "COmanage Registry error.log %s %s %s\n", ENV, UT, $0; fflush()}' 1>/dev/stdout)&
(cat <> /srv/comanage-registry/app/tmp/logs/debug.log | awk -v ENV="$ENV" -v UT="$USERTOKEN" '{printf "COmanage Registry debug.log %s %s %s\n", ENV, UT, $0; fflush()}' 1>/dev/stdout)&
(cat <> "$COMANAGE_REGISTRY_DIR/app/tmp/logs/error.log" | awk -v ENV="$ENV" -v UT="$USERTOKEN" '{printf "comanage_registry;error.log;%s;%s;%s\n", ENV, UT, $0; fflush()}' 1>/tmp/logpipe)&
(cat <> "$COMANAGE_REGISTRY_DIR/app/tmp/logs/debug.log" | awk -v ENV="$ENV" -v UT="$USERTOKEN" '{printf "comanage_registry;debug.log;%s;%s;%s\n", ENV, UT, $0; fflush()}' 1>/tmp/logpipe)&

# Start Apache HTTP Server
exec /usr/sbin/httpd -DFOREGROUND
@@ -100,15 +100,27 @@ if [ -n "$SHIBBOLETH_SP_PRIVKEY" ]; then
chmod 0600 /etc/shibboleth/sp-key.pem
fi

# Send logging to container stdout.
pushd /var/log/shibboleth
/bin/rm -f shibd.log transaction.log shibd_warn.log signature.log
/bin/ln -sfT /dev/stdout shibd.log
/bin/ln -sfT /dev/stdout shibd_warn.log
/bin/ln -sfT /dev/stdout transaction.log
/bin/ln -sfT /dev/stdout signature.log
chown shibd:shibd shibd.log transaction.log shibd_warn.log signature.log
popd
# If ENV or USERTOKEN as injected by the deployer contain a semi-colon remove it.
if [[ $ENV =~ .*";".* ]]; then
ENV=`echo $ENV | tr -d ';'`
export ENV
fi

if [[ $USERTOKEN =~ .*";".* ]]; then
USERTOKEN=`echo $USERTOKEN | tr -d ';'`
export USERTOKEN
fi

# If ENV or USERTOKEN as injected by the deployer contain a space remove it.
if [[ $ENV =~ [[:space:]] ]]; then
ENV=`echo $ENV | tr -d [:space:]`
export ENV
fi

if [[ $USERTOKEN =~ [[:space:]] ]]; then
USERTOKEN=`echo $USERTOKEN | tr -d [:space:]`
export USERTOKEN
fi

# Start the daemon.
export LD_LIBRARY_PATH=/opt/shibboleth/lib64
78 changes: 78 additions & 0 deletions comanage-registry-internet2-tier/docker-supervisord-entrypoint
@@ -0,0 +1,78 @@
#!/bin/bash

# COmanage Registry Dockerfile entrypoint
#
# Portions licensed to the University Corporation for Advanced Internet
# Development, Inc. ("UCAID") under one or more contributor license agreements.
# See the NOTICE file distributed with this work for additional information
# regarding copyright ownership.
#
# UCAID licenses this file to you under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with the
# License. You may obtain a copy of the License at:
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

if [ -n "$DEBUG" ]
then
OUTPUT=/dev/stdout
else
OUTPUT=/dev/null
fi

# If ENV or USERTOKEN as injected by the deployer contain a semi-colon remove it.
if [[ $ENV =~ .*";".* ]]; then
ENV=`echo $ENV | tr -d ';'`
export ENV
fi

if [[ $USERTOKEN =~ .*";".* ]]; then
USERTOKEN=`echo $USERTOKEN | tr -d ';'`
export USERTOKEN
fi

# If ENV or USERTOKEN as injected by the deployer contain a space remove it.
if [[ $ENV =~ [[:space:]] ]]; then
ENV=`echo $ENV | tr -d [:space:]`
export ENV
fi

if [[ $USERTOKEN =~ [[:space:]] ]]; then
USERTOKEN=`echo $USERTOKEN | tr -d [:space:]`
export USERTOKEN
fi

# Make a "console" logging pipe that anyone can write to regardless of who owns the process.
rm -f /tmp/logpipe > "$OUTPUT" 2>&1
mkfifo -m 666 /tmp/logpipe > "$OUTPUT" 2>&1
cat <> /tmp/logpipe &

# Format any console output from httpd into standard TIER form.
rm -f /tmp/loghttpd > "$OUTPUT" 2>&1
mkfifo -m 666 /tmp/loghttpd > "$OUTPUT" 2>&1
(cat <> /tmp/loghttpd | awk -v ENV="$ENV" -v UT="$USERTOKEN" '{printf "httpd;console;%s;%s;%s\n", ENV, UT, $0; fflush()}' 1>/tmp/logpipe 2>&1)&

# Format any console output from shibd into standard TIER form.
rm -f /tmp/logshibd > "$OUTPUT" 2>&1
mkfifo -m 666 /tmp/logshibd > "$OUTPUT" 2>&1
(cat <> /tmp/logshibd | awk -v ENV="$ENV" -v UT="$USERTOKEN" '{printf "httpd;console;%s;%s;%s\n", ENV, UT, $0; fflush()}' 1>/tmp/logpipe 2>&1)&

# Format any console output from supervisord into standard TIER form.
rm -f /tmp/logsuperd > "$OUTPUT" 2>&1
mkfifo -m 666 /tmp/logsuperd > "$OUTPUT" 2>&1
(cat <> /tmp/logsuperd | awk -v ENV="$ENV" -v UT="$USERTOKEN" '{printf "supervisord;console;%s;%s;%s\n", ENV, UT, $0; fflush()}' 1>/tmp/logpipe 2>&1)&

# Close stdout and stderr for this process since supervisord will write
# to its logfile and its children are configured to write to different
# pipes.
exec 1<&-
exec 2<&-

# Start supervisord
exec /usr/bin/supervisord -c /usr/local/etc/supervisord.conf
11 changes: 7 additions & 4 deletions comanage-registry-internet2-tier/httpd.conf
@@ -49,12 +49,15 @@ DirectoryIndex index.html
Require all denied
</Files>

ErrorLogFormat "httpd error_log [%{u}t] [%-m:%l] [pid %P:tid %T] %7F: %E: [client\ %a] %M% ,\ referer\ %{Referer}i"
ErrorLog "logs/error_log"
PassEnv ENV
PassEnv USERTOKEN

ErrorLogFormat "httpd;error_log;%{ENV}e;%{USERTOKEN}e;[%{u}t] [%-m:%l] [pid %P:tid %T] %7F: %E: [client\ %a] %M% ,\ referer\ %{Referer}i"
ErrorLog "/tmp/logpipe"
LogLevel warn

LogFormat "httpd access_log %h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" combined
CustomLog "logs/access_log" combined
LogFormat "httpd;access_log;%{ENV}e;%{USERTOKEN}e;%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" combined
CustomLog "/tmp/logpipe" combined

TypesConfig /etc/mime.types
AddType application/x-compress .Z
8 changes: 4 additions & 4 deletions comanage-registry-internet2-tier/native.logger
@@ -28,12 +28,12 @@ log4j.category.XMLTooling.libcurl=INFO
# define the appender

log4j.appender.native_log=org.apache.log4j.FileAppender
log4j.appender.native_log.fileName=/var/log/shibboleth-www/native.log
log4j.appender.native_log.fileName=/tmp/logpipe
log4j.appender.native_log.layout=org.apache.log4j.PatternLayout
log4j.appender.native_log.layout.ConversionPattern=shibd native.log %d{%Y-%m-%d %H:%M:%S} %p %c %x: %m%n
log4j.appender.native_log.layout.ConversionPattern=shibd;native.log;${ENV};${USERTOKEN};%d{%Y-%m-%d %H:%M:%S} %p %c %x: %m%n

log4j.appender.warn_log=org.apache.log4j.FileAppender
log4j.appender.warn_log.fileName=/var/log/shibboleth-www/native_warn.log
log4j.appender.warn_log.fileName=/tmp/logpipe
log4j.appender.warn_log.layout=org.apache.log4j.PatternLayout
log4j.appender.warn_log.layout.ConversionPattern=shibd native_warn.log %d{%Y-%m-%d %H:%M:%S} %p %c %x: %m%n
log4j.appender.warn_log.layout.ConversionPattern=shibd;native_warn.log;${ENV};${USERTOKEN};%d{%Y-%m-%d %H:%M:%S} %p %c %x: %m%n
log4j.appender.warn_log.threshold=WARN
16 changes: 8 additions & 8 deletions comanage-registry-internet2-tier/shibd.logger
@@ -42,22 +42,22 @@ log4j.additivity.Shibboleth-TRANSACTION=false
# define the appenders

log4j.appender.shibd_log=org.apache.log4j.FileAppender
log4j.appender.shibd_log.fileName=/var/log/shibboleth/shibd.log
log4j.appender.shibd_log.fileName=/tmp/logpipe
log4j.appender.shibd_log.layout=org.apache.log4j.PatternLayout
log4j.appender.shibd_log.layout.ConversionPattern=shibd shibd.log %d{%Y-%m-%d %H:%M:%S} %p %c %x: %m%n
log4j.appender.shibd_log.layout.ConversionPattern=shibd;shibd.log;${ENV};${USERTOKEN};%d{%Y-%m-%d %H:%M:%S} %p %c %x: %m%n

log4j.appender.warn_log=org.apache.log4j.FileAppender
log4j.appender.warn_log.fileName=/var/log/shibboleth/shibd_warn.log
log4j.appender.warn_log.fileName=/tmp/logpipe
log4j.appender.warn_log.layout=org.apache.log4j.PatternLayout
log4j.appender.warn_log.layout.ConversionPattern=shibd shibd_warn.log %d{%Y-%m-%d %H:%M:%S} %p %c %x: %m%n
log4j.appender.warn_log.layout.ConversionPattern=shibd;shibd_warn.log;${ENV};${USERTOKEN};%d{%Y-%m-%d %H:%M:%S} %p %c %x: %m%n
log4j.appender.warn_log.threshold=WARN

log4j.appender.tran_log=org.apache.log4j.FileAppender
log4j.appender.tran_log.fileName=/var/log/shibboleth/transaction.log
log4j.appender.tran_log.fileName=/tmp/logpipe
log4j.appender.tran_log.layout=org.apache.log4j.PatternLayout
log4j.appender.tran_log.layout.ConversionPattern=shibd transaction.log %d{%Y-%m-%d %H:%M:%S} %p %c %x: %m%n
log4j.appender.tran_log.layout.ConversionPattern=shibd;transaction.log;${ENV};${USERTOKEN};%d{%Y-%m-%d %H:%M:%S} %p %c %x: %m%n

log4j.appender.sig_log=org.apache.log4j.FileAppender
log4j.appender.sig_log.fileName=/var/log/shibboleth/signature.log
log4j.appender.sig_log.fileName=/tmp/logpipe
log4j.appender.sig_log.layout=org.apache.log4j.PatternLayout
log4j.appender.sig_log.layout.ConversionPattern=shibd signature.log %m
log4j.appender.sig_log.layout.ConversionPattern=shibd;signature.log${ENV};${USERTOKEN};%m
10 changes: 6 additions & 4 deletions comanage-registry-internet2-tier/supervisord.conf
@@ -18,19 +18,21 @@
; limitations under the License.

[supervisord]
logfile=/tmp/logsuperd
logfile_maxbytes=0
nodaemon=true
user=root

[program:httpd]
command=/usr/local/bin/docker-comanage-entrypoint
stdout_logfile=/dev/stdout
stdout_logfile=/tmp/loghttpd
stdout_logfile_maxbytes=0
stderr_logfile=/dev/stderr
stderr_logfile=/tmp/loghttpd
stderr_logfile_maxbytes=0

[program:shibd]
command=/usr/local/bin/docker-comanage-shibboleth-sp-entrypoint
stdout_logfile=/dev/stdout
stdout_logfile=/tmp/logshibd
stdout_logfile_maxbytes=0
stderr_logfile=/dev/stderr
stderr_logfile=/tmp/logshibd
stderr_logfile_maxbytes=0

0 comments on commit 1ec2ec9

Please sign in to comment.