diff --git a/Dockerfile b/Dockerfile index 6723853c..6fdf32bd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,17 +3,25 @@ FROM centos:centos7 as installing RUN yum update -y \ && yum install -y wget tar unzip dos2unix \ && yum clean all - -ENV GROUPER_VERSION=2.3.0 \ - JAVA_HOME=/opt/java - -RUN java_version=8.0.131; \ - zulu_version=8.21.0.1; \ - echo 'Downloading the OpenJDK Zulu...' \ - && wget -q http://cdn.azul.com/zulu/bin/zulu$zulu_version-jdk$java_version-linux_x64.tar.gz \ - && echo "1931ed3beedee0b16fb7fd37e069b162 zulu$zulu_version-jdk$java_version-linux_x64.tar.gz" | md5sum -c - \ - && tar -zxvf zulu$zulu_version-jdk$java_version-linux_x64.tar.gz -C /opt \ - && ln -s /opt/zulu$zulu_version-jdk$java_version-linux_x64 $JAVA_HOME + +ARG GROUPER_CONTAINER_VERSION + +ENV GROUPER_VERSION=2.4.0 \ + JAVA_HOME=/usr/lib/jvm/zulu-8/ \ + GROUPER_CONTAINER_VERSION=$GROUPER_CONTAINER_VERSION + +# use Zulu package +RUN rpm --import http://repos.azulsystems.com/RPM-GPG-KEY-azulsystems \ + && curl -o /etc/yum.repos.d/zulu.repo http://repos.azulsystems.com/rhel/zulu.repo \ + && yum -y install zulu-8 + +#RUN java_version=8.0.172; \ +# zulu_version=8.30.0.1; \ +# echo 'Downloading the OpenJDK Zulu...' \ +# && wget -q http://cdn.azul.com/zulu/bin/zulu$zulu_version-jdk$java_version-linux_x64.tar.gz \ +# && echo "0a101a592a177c1c7bc63738d7bc2930 zulu$zulu_version-jdk$java_version-linux_x64.tar.gz" | md5sum -c - \ +# && tar -zxvf zulu$zulu_version-jdk$java_version-linux_x64.tar.gz -C /opt \ +# && ln -s /opt/zulu$zulu_version-jdk$java_version-linux_x64 $JAVA_HOME #RUN java_version=8u151; \ # java_bnumber=12; \ @@ -31,6 +39,9 @@ RUN echo 'Downloading Grouper Installer...' \ && wget -q -O /opt/grouper/$GROUPER_VERSION/grouperInstaller.jar http://software.internet2.edu/grouper/release/$GROUPER_VERSION/grouperInstaller.jar COPY container_files/grouper.installer.properties /opt/grouper/$GROUPER_VERSION +# Temporary morphString file used for building, not used in production +COPY container_files/morphString.properties /opt/grouper/$GROUPER_VERSION + RUN echo 'Installing Grouper'; \ PATH=$PATH:$JAVA_HOME/bin; \ @@ -41,8 +52,8 @@ RUN echo 'Installing Grouper'; \ FROM centos:centos7 as cleanup -ENV GROUPER_VERSION=2.3.0 \ - TOMCAT_VERSION=8.5.12 \ +ENV GROUPER_VERSION=2.4.0 \ + TOMCAT_VERSION=8.5.42 \ TOMEE_VERSION=7.0.0 COPY --from=installing /opt/grouper/$GROUPER_VERSION/grouperInstaller.jar /opt/grouper/ @@ -53,21 +64,22 @@ COPY --from=installing /opt/grouper/$GROUPER_VERSION/grouper.ws-$GROUPER_VERSION #COPY --from=installing /opt/grouper/$GROUPER_VERSION/grouper.clientBinary-$GROUPER_VERSION/ /opt/grouper/grouper.clientBinary/ COPY --from=installing /opt/grouper/$GROUPER_VERSION/apache-tomcat-$TOMCAT_VERSION/ /opt/tomcat/ COPY --from=installing /opt/grouper/$GROUPER_VERSION/apache-tomee-webprofile-$TOMEE_VERSION/ /opt/tomee/ +COPY --from=installing /etc/alternatives/java /etc/alternatives/java -ADD http://central.maven.org/maven2/org/apache/logging/log4j/log4j-core/2.11.0/log4j-core-2.11.0.jar /opt/tomcat/bin -ADD http://central.maven.org/maven2/org/apache/logging/log4j/log4j-api/2.11.0/log4j-api-2.11.0.jar /opt/tomcat/bin -ADD http://central.maven.org/maven2/org/apache/logging/log4j/log4j-jul/2.11.0/log4j-jul-2.11.0.jar /opt/tomcat/bin +ADD https://repo1.maven.org/maven2/org/apache/logging/log4j/log4j-core/2.11.0/log4j-core-2.11.0.jar /opt/tomcat/bin +ADD https://repo1.maven.org/maven2/org/apache/logging/log4j/log4j-api/2.11.0/log4j-api-2.11.0.jar /opt/tomcat/bin +ADD https://repo1.maven.org/maven2/org/apache/logging/log4j/log4j-jul/2.11.0/log4j-jul-2.11.0.jar /opt/tomcat/bin -ADD http://central.maven.org/maven2/org/apache/logging/log4j/log4j-core/2.11.0/log4j-core-2.11.0.jar /opt/tomee/bin -ADD http://central.maven.org/maven2/org/apache/logging/log4j/log4j-api/2.11.0/log4j-api-2.11.0.jar /opt/tomee/bin -ADD http://central.maven.org/maven2/org/apache/logging/log4j/log4j-jul/2.11.0/log4j-jul-2.11.0.jar /opt/tomee/bin +ADD https://repo1.maven.org/maven2/org/apache/logging/log4j/log4j-core/2.11.0/log4j-core-2.11.0.jar /opt/tomee/bin +ADD https://repo1.maven.org/maven2/org/apache/logging/log4j/log4j-api/2.11.0/log4j-api-2.11.0.jar /opt/tomee/bin +ADD https://repo1.maven.org/maven2/org/apache/logging/log4j/log4j-jul/2.11.0/log4j-jul-2.11.0.jar /opt/tomee/bin RUN cd /opt/grouper/grouper.apiBinary/; \ - rm -fr ddlScripts/ grouper.lck grouper.log grouper.script grouper.tmp/ gshAddGrouperSystemWsGroup.gsh logs/ + rm -fr ddlScripts/ grouper.properties grouper.lck grouper.log grouper.script grouper.tmp/ gshAddGrouperSystemWsGroup.gsh logs/ RUN cd /opt/tomcat/; \ chmod +r bin/log4j-*.jar; \ - rm -fr webapps/docs/ webapps/examples/ webapps/host-manager/ webapps/manager/ logs/* temp/* work/* conf/logging.properties + rm -fr webapps/docs/ webapps/examples/ webapps/host-manager/ webapps/manager/ webapps/ROOT/ logs/* temp/* work/* conf/logging.properties RUN cd /opt/tomee/; \ chmod +r bin/log4j-*.jar; \ @@ -80,17 +92,20 @@ COPY container_files/tomcat/ /opt/tomcat/ COPY container_files/tomee/ /opt/tomee/ -FROM tier/shibboleth_sp +FROM tier/shibboleth_sp:3.0.4_03122019 LABEL author="tier-packaging@internet2.edu <tier-packaging@internet2.edu>" \ Vendor="TIER" \ ImageType="Grouper" \ ImageName=$imagename \ ImageOS=centos7 + +ARG GROUPER_CONTAINER_VERSION -ENV JAVA_HOME=/opt/java \ +ENV JAVA_HOME=/usr/lib/jvm/zulu-8/ \ PATH=$PATH:$JAVA_HOME/bin \ - GROUPER_HOME=/opt/grouper/grouper.apiBinary + GROUPER_HOME=/opt/grouper/grouper.apiBinary \ + GROUPER_CONTAINER_VERSION=$GROUPER_CONTAINER_VERSION RUN ln -sf /usr/share/zoneinfo/UTC /etc/localtime @@ -109,9 +124,11 @@ RUN groupadd -r tomcat \ && useradd -r -m -s /sbin/nologin -g tomcat tomcat \ && mkdir -p /opt/tomcat/logs/ /opt/tomcat/temp/ /opt/tomcat/work/ \ && chown -R tomcat:tomcat /opt/tomcat/logs/ /opt/tomcat/temp/ /opt/tomcat/work/ \ - && chown -R tomcat:tomcat /opt/tomee/logs/ /opt/tomee/temp/ /opt/tomee/work/ + && chown -R tomcat:tomcat /opt/tomee/logs/ /opt/tomee/temp/ /opt/tomee/work/ \ + && ln -s $JAVA_HOME/bin/java /etc/alternatives/java -RUN rm /etc/shibboleth/sp-key.pem /etc/shibboleth/sp-cert.pem +# does shib sp3 not generate these files? +# RUN rm /etc/shibboleth/sp-key.pem /etc/shibboleth/sp-cert.pem COPY container_files/tier-support/ /opt/tier-support/ COPY container_files/usr-local-bin/ /usr/local/bin/ @@ -130,4 +147,8 @@ WORKDIR /opt/grouper/grouper.apiBinary/ EXPOSE 80 443 +HEALTHCHECK NONE + +ENTRYPOINT ["/usr/local/bin/entrypoint.sh"] + CMD ["bin/gsh", "-loader"] diff --git a/Jenkinsfile b/Jenkinsfile index e9c2315c..3ed439e5 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -21,17 +21,19 @@ pipeline { echo "You must define an imagename in common.bash" currentBuild.result = 'FAILURE' } - sh 'mkdir -p bin' - sh 'mkdir -p tmp' + sh 'mkdir -p tmp && mkdir -p bin' dir('tmp'){ git([ url: "https://github.internet2.edu/docker/util.git", credentialsId: "jenkins-github-access-token" ]) - sh 'ls' - sh 'mv bin/* ../bin/.' + sh 'rm -rf ../bin/*' + sh 'mv ./bin/* ../bin/.' } + // Build and test scripts expect that 'tag' is present in common.bash. This is necessary for both Jenkins and standalone testing. + // We don't care if there are more 'tag' assignments there. The latest one wins. + sh "echo >> common.bash ; echo \"tag=\\\"${tag}\\\"\" >> common.bash ; echo common.bash ; cat common.bash" } } } - stage('Build') { + stage('Clean') { steps { script { try{ @@ -45,14 +47,58 @@ pipeline { } } } + stage('Build') { + steps { + script { + try{ + docker.withRegistry('https://registry.hub.docker.com/', "dockerhub-$maintainer") { + baseImg = docker.build("$maintainer/$imagename", "--build-arg GROUPER_CONTAINER_VERSION=$tag --no-cache .") + } + } catch(error) { + def error_details = readFile('./debug'); + def message = "BUILD ERROR: There was a problem building ${imagename}:${tag}. \n\n ${error_details}" + sh "rm -f ./debug" + handleError(message) + } + } + } + } + stage('Test') { + steps { + script { + try { + sh 'bin/test.sh 2>&1 | tee debug ; test ${PIPESTATUS[0]} -eq 0' + } catch (error) { + def error_details = readFile('./debug') + def message = "BUILD ERROR: There was a problem testing ${imagename}:${tag}. \n\n ${error_details}" + sh "rm -f ./debug" + handleError(message) + } + } + } + } + stage('Push') { steps { script { - docker.withRegistry('https://registry.hub.docker.com/', "dockerhub-$maintainer") { - def baseImg = docker.build("$maintainer/$imagename", "--no-cache .") - baseImg.push("$tag") - } - } + //// scan the image with clair + // sh 'docker run -p 5432:5432 -d --name clairdb arminc/clair-db:latest' + // sh 'docker run -p 6060:6060 --link clairdb:postgres -d --name clair arminc/clair-local-scan:v2.0.5' + // sh 'curl -L -o clair-scanner https://github.com/arminc/clair-scanner/releases/download/v8/clair-scanner_linux_amd64' + // sh 'chmod 755 clair-scanner' + // sh "./clair-scanner --ip 172.17.0.1 -r test.out $maintainer/$imagename:latest" + //// test the environment + // sh 'docker kill clairdb' + // sh 'docker rm clairdb' + // sh 'docker kill clair' + // sh 'docker rm clair' + // sh 'cd test-compose && ./compose.sh' + //// bring down after testing + //sh 'cd test-compose && docker-compose down' + docker.withRegistry('https://registry.hub.docker.com/', "dockerhub-$maintainer") { + baseImg.push("$tag") + } + } } } stage('Notify') { diff --git a/README.md b/README.md index dc613c9e..691033e4 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,27 @@ +[](https://jenkins.testbed.tier.internet2.edu/buildStatus/icon?job=docker/grouper/master) + + + +This repository contains the source code used to create the InCommon Trusted Access Platform Grouper container. This standalone container is pushed to Dockerhub, various tags are available at the following URL: https://hub.docker.com/r/tier/grouper/tags. This repo can also be cloned and the container built locally. + +The test-compose directory contains an example Grouper environment that starts up the various Grouper components. This example demonstrates how one might go about customizing and deploying their Grouper containers, using the TIER Grouper image as a base image. If evaluating Grouper, this is a good place to start. + + +# Upgrading from 2.3 to 2.4 + +If upgrading from Grouper version 2.3 to 2.4 and using LDAP, modifications will be needed in subject.properties and grouper-loaders.proprties. Further details about this can be found at the following URL: +https://spaces.at.internet2.edu/display/Grouper/vt-ldap+to+ldaptive+migration+for+LDAP+access + +In particular, in subject.properties, *.param.base.value should be adjusted to only contain the RDN (Relative Distinguished Name), not the full DN. For example, "OU=People", not "OU=People,DC=domain,DC=edu" + +Additional upgrade information can be found at the following URL: https://spaces.at.internet2.edu/display/Grouper/v2.4+Upgrade+Instructions+from+v2.3 + + # Supported tags - latest -- patch specific tags* (i.e. 2.3.0-a97-u41-w11-p16) +- patch specific tags with date timestamp* (i.e. 2.4.0-80-u51-w10-p11-20191118) \* Patch builds are routinely produced, but not necessarily for each patch release. The following monikers are used to construct the tag name: @@ -10,6 +29,7 @@ - u = ui patch number - w = ws patch number - p = pspng patch number +- last field = the year, month and day the image was built # Quick reference @@ -43,7 +63,7 @@ While TIER recommends/supports using Docker Swarm for orchestrating the Grouper ### Daemon/Loader -Run the Grouper Daemon/Loader as a service. +Run the Grouper Daemon/Loader as a service. If the daemon/loader container dies unexpectedly, it may be due to memory contraints. Refer to the "Grouper Shell/Loader" section below for information on how to tweak memory settings. ```console $ docker service create --detach --name grouper-daemon tier/grouper:latest daemon @@ -156,7 +176,7 @@ For passing full files into the container, this container will make any secrets Docker Secrets can also be used to pass in strings, such as a database connection string password, into the component config. To pass in the Grouper database connection string, one might set the property and value as such: ```text -hibernate.connection.password.elConfig = ${java.lang.System.getenv().get('GROUPER_DATABASE_PASSWORD_FILE') != null ? org.apache.commons.io.FileUtils.readFileToString(java.lang.System.getenv().get('GROUPER_DATABASE_PASSWORD_FILE'), "utf-8") : java.lang.System.getenv().get('GROUPER_DATABASE_PASSWORD') } +hibernate.connection.password.elConfig = ${java.lang.System.getenv().get('GROUPER_DATABASE_PASSWORD_FILE') != null ? org.apache.commons.io.FileUtils.readFileToString(new("java.io.File", java.lang.System.getenv().get('GROUPER_DATABASE_PASSWORD_FILE')), "utf-8") : java.lang.System.getenv().get('GROUPER_DATABASE_PASSWORD') } ``` Note that the default property name has been changed by appending `.elConfig`. (This causes Grouper to evaluate the string before saving the value.) The expression allows deployers to use a file containing only the database password as a Docker Secret and reference the file name via the `GROUPER_DATABASE_PASSWORD_FILE` environment property. This allows the config files to be baked into the image, if desired. Also, but not recommended, the database password could just be set in the Docker Service definition as an environment variable, `GROUPER_DATABASE_PASSWORD`. (Technically the expression can be broken up and just the desired functionality used.) Of course, using Grouper's MorphString functionality is supported and likely is the best option, but does require more effort in setting it up. @@ -206,8 +226,8 @@ Deployers can set runtime variables to both the Grouper Shell and Loader/Daemon ### Grouper Shell/Loader The following environment variables are used by the Grouper Shell/Loader: -- MEM_START: corresponds to the java's `-Xms`. -- MEM_MAX: corresponds to java's `-Xmx`. +- MEM_START: corresponds to the java's `-Xms`. (default is 64m) +- MEM_MAX: corresponds to java's `-Xmx`. (default is 750m) ### Tomcat/TomEE @@ -229,6 +249,7 @@ Here is a list of significant directories and files that deployers should be awa - `/opt/tomcat/`: used to run Grouper UI and Grouper WS - `/opt/tomee/`: used to run the Grouper SCIM Server. - `/var/run/secrets`: location where Docker Secrets are mounted into the container. Secrets starting with `grouper_`, `shib_`, and `httpd_` have special meaning. See `Secrets/Configs` above. +- `/usr/lib/jvm/zulu-8/jre/lib/security/cacerts`: location of the Java trust store. To examine baseline image files, one might run `docker run --name=temp -it tier/grouper bash` and browse through these file system endpoints. While the container is running one may copy files out of the image/container using something like `docker cp containerId:/opt/grouper/grouper.api/conf/grouper.properties .`, which will copy the `grouper.properties` to the Docker client's present working directory. These files can then be edited and applied via the mechanisms outlined above. diff --git a/container_files/grouper.installer.properties b/container_files/grouper.installer.properties index dfd3ac5e..4e018728 100644 --- a/container_files/grouper.installer.properties +++ b/container_files/grouper.installer.properties @@ -1,7 +1,7 @@ # this should be before the version number download.server.url = https://software.internet2.edu/grouper # default version to install -grouper.version = 2.3.0 +grouper.version = 2.4.0 # print out autorun keys in prompts so you can easily see how to configure the autorun grouperInstaller.print.autorunKeys = true # default to install or upgrade (default is install) @@ -14,9 +14,12 @@ grouperInstaller.default.installOrUpgrade = install ## ############################## +grouperInstaller.autorun.forceInstallPatch = t grouperInstaller.autorun.installAllPatches = false grouperInstaller.autorun.installPatchesUpToACertainPatchLevel = true -grouperInstaller.autorun.installPatchesUpToThesePatchLevels = grouper_v2_3_0_api_patch_104,grouper_v2_3_0_ui_patch_42,grouper_v2_3_0_ws_patch_12,grouper_v2_3_0_pspng_patch_16 +# 2.4.0-a93-u56-w11-p12-20200214-rc1 +grouperInstaller.autorun.installPatchesUpToThesePatchLevels = grouper_v2_4_0_api_patch_93,grouper_v2_4_0_ui_patch_56,grouper_v2_4_0_ws_patch_11,grouper_v2_4_0_pspng_patch_12 + #### set this to true to try to use defaults for everything. Only things without default values will need to be set grouperInstaller.autorun.useDefaultsAsMuchAsAvailable = true @@ -29,11 +32,15 @@ grouperInstaller.autorun.deleteAndInitDatabase = t grouperInstaller.autorun.addQuickstartData = f grouperInstaller.autorun.installClient = f -grouperInstaller.autorun.installGrouperActiveMqMessaging = t -grouperInstaller.autorun.activeMqWhereInstalled = /opt/grouper/2.3.0/grouper.apiBinary-2.3.0/ +grouperInstaller.autorun.installGrouperActiveMqMessaging = f +grouperInstaller.autorun.activeMqWhereInstalled = /opt/grouper/2.4.0/grouper.apiBinary-2.4.0/ grouperInstaller.autorun.installGrouperAwsSqsMessaging = t -grouperInstaller.autorun.AwsSqsWhereInstalled = /opt/grouper/2.3.0/grouper.apiBinary-2.3.0/ +grouperInstaller.autorun.AwsSqsWhereInstalled = /opt/grouper/2.4.0/grouper.apiBinary-2.4.0/ grouperInstaller.autorun.installGrouperRabbitMqMessaging = t -grouperInstaller.autorun.rabbitMqWhereInstalled = /opt/grouper/2.3.0/grouper.apiBinary-2.3.0/ +grouperInstaller.autorun.rabbitMqWhereInstalled = /opt/grouper/2.4.0/grouper.apiBinary-2.4.0/ + +# disable installing pspng, for now +grouperInstaller.autorun.installPspng = t +grouperInstaller.autorun.installPsp = f diff --git a/container_files/httpd/grouper-www.conf b/container_files/httpd/grouper-www.conf index 368f7f6c..562e47c6 100644 --- a/container_files/httpd/grouper-www.conf +++ b/container_files/httpd/grouper-www.conf @@ -7,6 +7,10 @@ ProxyPass /grouper ajp://localhost:8009/grouper timeout=2400 ProxyPass /grouper-ws ajp://localhost:8009/grouper-ws timeout=2400 ProxyPass /grouper-ws-scim ajp://localhost:8009/grouper-ws-scim timeout=2400 +RewriteEngine on +RewriteCond %{REQUEST_URI} "^/$" +RewriteRule . %{REQUEST_SCHEME}://%{HTTP_HOST}/grouper/ [R=301,L] + <Location /grouper> AuthType shibboleth ShibRequestSetting requireSession 1 diff --git a/container_files/morphString.properties b/container_files/morphString.properties new file mode 100644 index 00000000..52479216 --- /dev/null +++ b/container_files/morphString.properties @@ -0,0 +1 @@ +encrypt.key=fh43IRJ4Nf5 diff --git a/container_files/shibboleth/native.logger b/container_files/shibboleth/native.logger new file mode 100644 index 00000000..0b01f32f --- /dev/null +++ b/container_files/shibboleth/native.logger @@ -0,0 +1,39 @@ +# set overall behavior +log4j.rootCategory=INFO, native_log, warn_log + +# fairly verbose for DEBUG, so generally leave at INFO +log4j.category.XMLTooling.XMLObject=INFO +log4j.category.XMLTooling.KeyInfoResolver=INFO +log4j.category.Shibboleth.IPRange=INFO +log4j.category.Shibboleth.PropertySet=INFO + +# raise for low-level tracing of SOAP client HTTP/SSL behavior +log4j.category.XMLTooling.libcurl=INFO + +# useful categories to tune independently: +# +# tracing of SAML messages and security policies +#log4j.category.OpenSAML.MessageDecoder=DEBUG +#log4j.category.OpenSAML.MessageEncoder=DEBUG +#log4j.category.OpenSAML.SecurityPolicyRule=DEBUG +# interprocess message remoting +#log4j.category.Shibboleth.Listener=DEBUG +# mapping of requests to applicationId +#log4j.category.Shibboleth.RequestMapper=DEBUG +# high level session cache operations +#log4j.category.Shibboleth.SessionCache=DEBUG +# persistent storage and caching +#log4j.category.XMLTooling.StorageService=DEBUG + +# define the appender + +log4j.appender.native_log=org.apache.log4j.FileAppender +log4j.appender.native_log.fileName=/tmp/logpipe +log4j.appender.native_log.layout=org.apache.log4j.PatternLayout +log4j.appender.native_log.layout.ConversionPattern=shibd;native.log;${ENV};${USERTOKEN};%d{%Y-%m-%d %H:%M:%S} %p %c %x: %m%n + +log4j.appender.warn_log=org.apache.log4j.FileAppender +log4j.appender.warn_log.fileName=/tmp/logpipe +log4j.appender.warn_log.layout=org.apache.log4j.PatternLayout +log4j.appender.warn_log.layout.ConversionPattern=shibd;native_warn.log;${ENV};${USERTOKEN};%d{%Y-%m-%d %H:%M:%S} %p %c %x: %m%n +log4j.appender.warn_log.threshold=WARN diff --git a/container_files/ui/classes/grouper-ui.properties b/container_files/ui/classes/grouper-ui.properties new file mode 100644 index 00000000..80fbee1e --- /dev/null +++ b/container_files/ui/classes/grouper-ui.properties @@ -0,0 +1,12 @@ +# +# Grouper UI configuration +# $Id: grouper.client.example.properties,v 1.24 2009-12-30 04:23:02 mchyzer Exp $ +# + +# The grouper-ui.properties uses Grouper Configuration Overlays (documented on wiki) +# By default the configuration is read from grouper-ui.base.properties +# (which should not be edited), and the grouper-ui.properties overlays +# the base settings. See the grouper-ui.base.properties for the possible +# settings that can be applied to the grouper-ui.properties + +grouperUi.logout.redirectToUrl=/Shibboleth.sso/Logout \ No newline at end of file diff --git a/container_files/ui/web.xml b/container_files/ui/web.xml index 92d4125d..f3aa302f 100644 --- a/container_files/ui/web.xml +++ b/container_files/ui/web.xml @@ -1,203 +1,89 @@ <?xml version="1.0" encoding="UTF-8"?> -<web-app xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:j2ee="http://java.sun.com/xml/ns/j2ee" version="2.4" xsi:schemaLocation="http://java.sun.com/xml/ns/j2ee http://java.s -un.com/xml/ns/j2ee/web-app_2_4.xsd"> -<!--DO NOT EDIT THIS FILE. IT WILL BE OVERWRITTEN. CHANGE YOUR FILE specified by the build.properties value [additional.web.xml]. The contents of that file are merged into ${grouper-ui}/w -ebapp/WEB-INF/web.core.xml--> -<!--In webapp--> -<!--Processing context-param--> -<!--Processing filter--> -<!--Inserting tag from merge file--> -<filter> +<web-app xmlns:j2ee="http://java.sun.com/xml/ns/j2ee" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://java.sun.com/xml/ns/j2ee http://java.sun.com/xml/ns/j2ee/web-app_2_4.xsd" + version="2.4"> + <filter> <filter-name>GrouperUi</filter-name> <filter-class>edu.internet2.middleware.grouper.ui.GrouperUiFilter</filter-class> -</filter> -<!--Inserting tag from merge file--> -<filter> - <filter-name>Error Catcher</filter-name> - <filter-class>edu.internet2.middleware.grouper.ui.ErrorFilter</filter-class> </filter> -<!--Inserting tag from base file. Merge file was file:/opt/grouper.ui-2.3.0/temp/99.web.core-filters.xml--> -<filter> - <filter-name>Login check</filter-name> - <filter-class>edu.internet2.middleware.grouper.ui.LoginCheckFilter</filter-class> - <init-param> - <param-name>failureUrl</param-name> - <param-value>/index.jsp</param-value> - </init-param> - <init-param> - <param-name>ignore</param-name> - <param-value>:/populateIndex.do:/callLogin.do:/error.do:/logout.do:/status:</param-value> - </init-param> - <init-param> - <param-name>grouperRole</param-name> - <param-value>*</param-value> - </init-param> - </filter> -<!--Inserting tag from base file. Merge file was file:/opt/grouper.ui-2.3.0/temp/99.web.core-filters.xml--> -<filter> - <filter-name>Caller page</filter-name> - <filter-class>edu.internet2.middleware.grouper.ui.CallerPageFilter</filter-class> - </filter> -<!--Inserting tag from base file. Merge file was file:/opt/grouper.ui-2.3.0/temp/99.web.core-filters.xml--> -<filter> + <filter> <filter-name>CSRFGuard</filter-name> <filter-class>org.owasp.csrfguard.CsrfGuardFilter</filter-class> </filter> -<!--Processing filter-mapping--> -<!--Inserting tag from merge file--> -<filter-mapping> - <filter-name>GrouperUi</filter-name> - <url-pattern>*.do</url-pattern> - </filter-mapping> -<!--Inserting tag from merge file--> -<filter-mapping> + <filter-mapping> <filter-name>GrouperUi</filter-name> <url-pattern>*.jsp</url-pattern> </filter-mapping> -<!--Inserting tag from merge file--> -<filter-mapping> - <filter-name>Error Catcher</filter-name> - <url-pattern>*.do</url-pattern> - </filter-mapping> -<!--Inserting tag from merge file--> -<filter-mapping> - <filter-name>Error Catcher</filter-name> - <url-pattern>/gotoCallerPage</url-pattern> - </filter-mapping> -<!--Inserting tag from base file. Merge file was file:/opt/grouper.ui-2.3.0/temp/99.web.core-filters.xml--> -<filter-mapping> + <filter-mapping> <filter-name>GrouperUi</filter-name> <url-pattern>/grouperUi/app/*</url-pattern> </filter-mapping> -<!--Inserting tag from base file. Merge file was file:/opt/grouper.ui-2.3.0/temp/99.web.core-filters.xml--> -<filter-mapping> + <filter-mapping> <filter-name>GrouperUi</filter-name> <url-pattern>/grouperUi/appHtml/*</url-pattern> </filter-mapping> -<!--Inserting tag from base file. Merge file was file:/opt/grouper.ui-2.3.0/temp/99.web.core-filters.xml--> -<filter-mapping> + <filter-mapping> <filter-name>GrouperUi</filter-name> <url-pattern>/grouperExternal/app/*</url-pattern> </filter-mapping> -<!--Inserting tag from base file. Merge file was file:/opt/grouper.ui-2.3.0/temp/99.web.core-filters.xml--> -<filter-mapping> - <filter-name>GrouperUi</filter-name> - <url-pattern>/grouperExternal/appHtml/*</url-pattern> - </filter-mapping> -<!--Inserting tag from base file. Merge file was file:/opt/grouper.ui-2.3.0/temp/99.web.core-filters.xml--> -<filter-mapping> + <filter-mapping> <filter-name>GrouperUi</filter-name> <url-pattern>/grouperExternal/public/UiV2Public.index</url-pattern> </filter-mapping> -<!--Inserting tag from base file. Merge file was file:/opt/grouper.ui-2.3.0/temp/99.web.core-filters.xml--> -<filter-mapping> + <filter-mapping> <filter-name>GrouperUi</filter-name> <url-pattern>/grouperExternal/public/UiV2Public.postIndex</url-pattern> </filter-mapping> -<!--Inserting tag from base file. Merge file was file:/opt/grouper.ui-2.3.0/temp/99.web.core-filters.xml--> -<filter-mapping> - <filter-name>Caller page</filter-name> - <url-pattern>/gotoCallerPage</url-pattern> - </filter-mapping> -<!--Inserting tag from base file. Merge file was file:/opt/grouper.ui-2.3.0/temp/99.web.core-filters.xml--> -<filter-mapping> - <filter-name>Login check</filter-name> - <url-pattern>*.do</url-pattern> - </filter-mapping> -<!--Inserting tag from base file. Merge file was file:/opt/grouper.ui-2.3.0/temp/99.web.core-filters.xml--> -<filter-mapping> + <filter-mapping> <filter-name>CSRFGuard</filter-name> <url-pattern>/*</url-pattern> </filter-mapping> -<!--Processing listener--> -<!--Inserting tag from base file. Merge file was file:/opt/grouper.ui-2.3.0/temp/99.web.core-filters.xml--> -<listener> - <listener-class>edu.internet2.middleware.grouper.ui.GrouperSessionAttributeListener</listener-class> -</listener> -<!--Inserting tag from base file. Merge file was file:/opt/grouper.ui-2.3.0/temp/99.web.core-filters.xml--> -<listener> + <listener> + <listener-class>edu.internet2.middleware.grouper.ui.GrouperSessionAttributeListener</listener-class> + </listener> + <listener> <listener-class>org.owasp.csrfguard.CsrfGuardServletContextListener</listener-class> </listener> -<!--Inserting tag from base file. Merge file was file:/opt/grouper.ui-2.3.0/temp/99.web.core-filters.xml--> -<listener> + <listener> <listener-class>org.owasp.csrfguard.CsrfGuardHttpSessionListener</listener-class> </listener> -<!--Processing servlet--> -<!--Inserting tag from base file. Merge file was file:/opt/grouper.ui-2.3.0/temp/99.web.core-filters.xml--> -<servlet> + <servlet> <servlet-name>StatusServlet</servlet-name> <display-name>Status Servlet</display-name> <servlet-class>edu.internet2.middleware.grouper.j2ee.status.GrouperStatusServlet</servlet-class> <load-on-startup>1</load-on-startup> </servlet> -<!--Inserting tag from base file. Merge file was file:/opt/grouper.ui-2.3.0/temp/99.web.core-filters.xml--> -<servlet> - <!-- Map the filter to a Servlet or URL --> - + <servlet> <servlet-name>UiServlet</servlet-name> <servlet-class>edu.internet2.middleware.grouper.j2ee.GrouperUiRestServlet</servlet-class> <load-on-startup>1</load-on-startup> </servlet> -<!--Inserting tag from base file. Merge file was file:/opt/grouper.ui-2.3.0/temp/99.web.core-filters.xml--> -<servlet> - <servlet-name>OwaspJavaScriptServlet</servlet-name> - <servlet-class>org.owasp.csrfguard.servlet.JavaScriptServlet</servlet-class> - </servlet> -<!--Inserting tag from base file. Merge file was file:/opt/grouper.ui-2.3.0/temp/99.web.core-filters.xml--> -<servlet> - <servlet-name>action</servlet-name> - <servlet-class>org.apache.struts.action.ActionServlet</servlet-class> - <init-param> - <param-name>config</param-name> - <param-value>/WEB-INF/struts-config.xml</param-value> - </init-param> - <load-on-startup>2</load-on-startup> + <servlet> + <servlet-name>OwaspJavaScriptServlet</servlet-name> + <servlet-class>org.owasp.csrfguard.servlet.JavaScriptServlet</servlet-class> </servlet> -<!--Processing servlet-mapping--> -<!--Inserting tag from base file. Merge file was file:/opt/grouper.ui-2.3.0/temp/99.web.core-filters.xml--> -<servlet-mapping> + <servlet-mapping> <servlet-name>StatusServlet</servlet-name> <url-pattern>/status</url-pattern> </servlet-mapping> -<!--Inserting tag from base file. Merge file was file:/opt/grouper.ui-2.3.0/temp/99.web.core-filters.xml--> -<servlet-mapping> + <servlet-mapping> <servlet-name>UiServlet</servlet-name> <url-pattern>/grouperUi/app/*</url-pattern> </servlet-mapping> -<!--Inserting tag from base file. Merge file was file:/opt/grouper.ui-2.3.0/temp/99.web.core-filters.xml--> -<servlet-mapping> + <servlet-mapping> <servlet-name>UiServlet</servlet-name> <url-pattern>/grouperExternal/app/*</url-pattern> </servlet-mapping> -<!--Inserting tag from base file. Merge file was file:/opt/grouper.ui-2.3.0/temp/99.web.core-filters.xml--> -<servlet-mapping> + <servlet-mapping> <servlet-name>UiServlet</servlet-name> <url-pattern>/grouperExternal/public/UiV2Public.index</url-pattern> </servlet-mapping> -<!--Inserting tag from base file. Merge file was file:/opt/grouper.ui-2.3.0/temp/99.web.core-filters.xml--> -<servlet-mapping> + <servlet-mapping> <servlet-name>UiServlet</servlet-name> <url-pattern>/grouperExternal/public/UiV2Public.postIndex</url-pattern> </servlet-mapping> -<!--Inserting tag from base file. Merge file was file:/opt/grouper.ui-2.3.0/temp/99.web.core-filters.xml--> -<servlet-mapping> - <servlet-name>OwaspJavaScriptServlet</servlet-name> - <url-pattern>/grouperExternal/public/OwaspJavaScriptServlet</url-pattern> - </servlet-mapping> -<!--Inserting tag from base file. Merge file was file:/opt/grouper.ui-2.3.0/temp/99.web.core-filters.xml--> -<servlet-mapping> - <servlet-name>action</servlet-name> - <url-pattern>*.do</url-pattern> + <servlet-mapping> + <servlet-name>OwaspJavaScriptServlet</servlet-name> + <url-pattern>/grouperExternal/public/OwaspJavaScriptServlet</url-pattern> </servlet-mapping> -<!--Processing mime-mapping--> -<!--Processing error-page--> -<!--Processing error-page--> -<!--Processing taglib--> -<!--Processing resource-env-ref--> -<!--Processing resource-ref--> -<!--Processing security-constraint--> -<!--Inserting tag from base file. Merge file was file:/opt/grouper.ui-2.3.0/temp/99.web.core-filters.xml--> -<!--Processing env-entry--> -<!--Processing ejb-ref--> -<!--Processing ejb-local-ref--> </web-app> diff --git a/container_files/usr-local-bin/daemon b/container_files/usr-local-bin/daemon index 27f3da63..da40d1e6 100755 --- a/container_files/usr-local-bin/daemon +++ b/container_files/usr-local-bin/daemon @@ -4,6 +4,6 @@ prepDaemon -export GSH_JVMARGS="-DENV=$ENV -DUSERTOKEN=$USERTOKEN" +export GSH_JVMARGS="$GSH_JVMARGS -DENV=$ENV -DUSERTOKEN=$USERTOKEN" exec bin/gsh -loader > /tmp/loggrouper diff --git a/container_files/usr-local-bin/entrypoint.sh b/container_files/usr-local-bin/entrypoint.sh new file mode 100755 index 00000000..83e985d6 --- /dev/null +++ b/container_files/usr-local-bin/entrypoint.sh @@ -0,0 +1,6 @@ +#!/bin/sh + +. /usr/local/bin/library.sh +prepConf + +exec "$@" \ No newline at end of file diff --git a/container_files/usr-local-bin/gsh b/container_files/usr-local-bin/gsh index 0539b40a..e65979ab 100755 --- a/container_files/usr-local-bin/gsh +++ b/container_files/usr-local-bin/gsh @@ -4,6 +4,6 @@ prepDaemon -export GSH_JVMARGS="-DENV=$ENV -DUSERTOKEN=$USERTOKEN" +export GSH_JVMARGS="$GSH_JVMARGS -DENV=$ENV -DUSERTOKEN=$USERTOKEN" exec bin/gsh "$@" | tee /tmp/loggrouper diff --git a/container_files/usr-local-bin/library.sh b/container_files/usr-local-bin/library.sh old mode 100644 new mode 100755 index 15cf545c..5d3790af --- a/container_files/usr-local-bin/library.sh +++ b/container_files/usr-local-bin/library.sh @@ -7,26 +7,37 @@ setupPipe() { mkfifo -m 666 $1 } -# Make a "console" logging pipe that anyone can write too regardless of who owns the process. -setupPipe /tmp/logpipe -cat <> /tmp/logpipe & +setupLoggingPipe() { + # Make a "console" logging pipe that anyone can write too regardless of who owns the process. + setupPipe /tmp/logpipe + cat <> /tmp/logpipe & +} # Make loggers pipes for the supervisord connected apps' console, so that we can prepend the streams. -setupPipe /tmp/loggrouper -(cat <> /tmp/loggrouper | awk -v ENV="$ENV" -v UT="$USERTOKEN" '{printf "grouper;console;%s;%s;%s\n", ENV, UT, $0; fflush()}' &>/tmp/logpipe) & - -setupPipe /tmp/loghttpd -(cat <> /tmp/loghttpd | awk -v ENV="$ENV" -v UT="$USERTOKEN" '{printf "httpd;console;%s;%s;%s\n", ENV, UT, $0; fflush()}' &>/tmp/logpipe) & +setupGrouperLogPipe() { + setupPipe /tmp/loggrouper + (cat <> /tmp/loggrouper | awk -v ENV="$ENV" -v UT="$USERTOKEN" '{printf "grouper;console;%s;%s;%s\n", ENV, UT, $0; fflush()}' &>/tmp/logpipe) & +} -setupPipe /tmp/logshibd -(cat <> /tmp/logshibd | awk -v ENV="$ENV" -v UT="$USERTOKEN" '{printf "shibd;console;%s;%s;%s", ENV, UT, $0; fflush()}' &>/tmp/logpipe) & +setupHttpdLogPipe() { + setupPipe /tmp/loghttpd + (cat <> /tmp/loghttpd | awk -v ENV="$ENV" -v UT="$USERTOKEN" '{printf "httpd;console;%s;%s;%s\n", ENV, UT, $0; fflush()}' &>/tmp/logpipe) & +} -setupPipe /tmp/logtomcat -(cat <> /tmp/logtomcat | awk -v ENV="$ENV" -v UT="$USERTOKEN" '{printf "tomcat;console;%s;%s;%s\n", ENV, UT, $0; fflush()}' &>/tmp/logpipe) & +setupShibdLogPipe() { + setupPipe /tmp/logshibd + (cat <> /tmp/logshibd | awk -v ENV="$ENV" -v UT="$USERTOKEN" '{printf "shibd;console;%s;%s;%s", ENV, UT, $0; fflush()}' &>/tmp/logpipe) & +} -setupPipe /tmp/logsuperd -(cat <> /tmp/logsuperd | awk -v ENV="$ENV" -v UT="$USERTOKEN" '{printf "supervisord;console;%s;%s;%s\n", ENV, UT, $0; fflush()}' &>/tmp/logpipe) & +setupTomcatLogPipe() { + setupPipe /tmp/logtomcat + (cat <> /tmp/logtomcat | awk -v ENV="$ENV" -v UT="$USERTOKEN" '{printf "tomcat;console;%s;%s;%s\n", ENV, UT, $0; fflush()}' &>/tmp/logpipe) & +} +setupSupervisordLogPipe() { + setupPipe /tmp/logsuperd + (cat <> /tmp/logsuperd | awk -v ENV="$ENV" -v UT="$USERTOKEN" '{printf "supervisord;console;%s;%s;%s\n", ENV, UT, $0; fflush()}' &>/tmp/logpipe) & +} linkGrouperSecrets() { for filepath in /run/secrets/*; do @@ -46,6 +57,11 @@ linkGrouperSecrets() { } prepDaemon() { + setupLoggingPipe + setupGrouperLogPipe +} + +prepDaemonConf() { local dest=/opt/grouper/grouper.apiBinary linkGrouperSecrets $dest/conf @@ -58,6 +74,16 @@ prepDaemon() { } prepSCIM() { + setupLoggingPipe + setupGrouperLogPipe + setupHttpdLogPipe + setupTomcatLogPipe + + + cp /opt/tier-support/grouper-ws-scim.xml /opt/tomee/conf/Catalina/localhost/ +} + +prepSCIMConf() { local dest=/opt/grouper/grouper.scim/WEB-INF linkGrouperSecrets $dest/classes @@ -66,12 +92,21 @@ prepSCIM() { fi if [ -d "/opt/grouper/lib" ]; then cp -r /opt/grouper/lib/* $dest/lib/ - fi - - cp /opt/tier-support/grouper-ws-scim.xml /opt/tomee/conf/Catalina/localhost/ + fi } prepUI() { + setupLoggingPipe + setupGrouperLogPipe + setupHttpdLogPipe + setupShibdLogPipe + setupTomcatLogPipe + setupSupervisordLogPipe + + cp /opt/tier-support/grouper.xml /opt/tomcat/conf/Catalina/localhost/ +} + +prepUIConf() { local dest=/opt/grouper/grouper.ui/WEB-INF linkGrouperSecrets $dest/classes @@ -81,11 +116,19 @@ prepUI() { if [ -d "/opt/grouper/lib" ]; then cp -r /opt/grouper/lib/* $dest/lib/ fi - - cp /opt/tier-support/grouper.xml /opt/tomcat/conf/Catalina/localhost/ } prepWS() { + setupLoggingPipe + setupGrouperLogPipe + setupHttpdLogPipe + setupTomcatLogPipe + setupSupervisordLogPipe + + cp /opt/tier-support/grouper-ws.xml /opt/tomcat/conf/Catalina/localhost/ +} + +prepWSConf() { local dest=/opt/grouper/grouper.ws/WEB-INF linkGrouperSecrets $dest/classes @@ -95,6 +138,12 @@ prepWS() { if [ -d "/opt/grouper/lib" ]; then cp -r /opt/grouper/lib/* $dest/lib/ fi - - cp /opt/tier-support/grouper-ws.xml /opt/tomcat/conf/Catalina/localhost/ } + + +prepConf() { + prepDaemonConf + prepSCIMConf + prepUIConf + prepWSConf +} \ No newline at end of file diff --git a/test-compose/README.md b/test-compose/README.md index 2dc53e0e..6bf62c51 100644 --- a/test-compose/README.md +++ b/test-compose/README.md @@ -53,7 +53,7 @@ Note that when accessing the Grouper UI, Grouper WS, or Shibboleth IdP, your bro - In this example, we use a variety of ways to pass in passwords (Grouper database, LDAP, Grouper Client, and RabbitMQ). The point is to demonstrate possibilities and not demonstrating what is required. (See the image readme for more details.) - Docker `configs` are not supported by Docker Compose (when run in a non-Swarm mode), so those are represented in the `docker-compose.yml` file as bind mount volumes. - The Grouper config files in the `data` image's `conf` directory are used to build the sample grouper database and ldap store. They are not used when the container is instantiated as there is no Grouper runtime in this container. -- The containers will use Docker Secrets and bind mounts for non-sensitive files that are read from the `configs-ans-secrets` directory in the `test-compose` directory. +- The containers will use Docker Secrets and bind mounts for non-sensitive files that are read from the `configs-and-secrets` directory in the `test-compose` directory. - With regard to RabbitMQ, the deployer must manually add a queue named `sampleQueue` to see Grouper messages in RabbitMQ. Messages will be dropped by RabbitMQ (and the Grouper Deamon will log errors) until this occurs. - In this example, we don't care about the IdP secrets. They are baked into the overlay instead of using Docker Secrets. (This is not best practice for an IdP configuration, but that isn't the focus of this example.) diff --git a/test-compose/configs-and-secrets/grouper/grouper-loader.properties b/test-compose/configs-and-secrets/grouper/grouper-loader.properties index 07c4f56a..68bef05f 100644 --- a/test-compose/configs-and-secrets/grouper/grouper-loader.properties +++ b/test-compose/configs-and-secrets/grouper/grouper-loader.properties @@ -4,17 +4,18 @@ # specify the ldap connection with user, pass, url # the string after "ldap." is the ID of the connection, and it should not have # spaces or other special chars in it. In this case is it "personLdap" - + #note the URL should start with ldap: or ldaps: if it is SSL. #It should contain the server and port (optional if not default), and baseDn, #e.g. ldaps://ldapserver.school.edu:636/dc=school,dc=edu -ldap.demo.url = ldap://data:389/dc=example,dc=edu +ldap.demo.url = ldap://data:389/dc=internet2,dc=edu #optional, if authenticated ldap.demo.user = cn=admin,dc=internet2,dc=edu #optional, if authenticated note the password can be stored encrypted in an external file -ldap.demo.pass = ${java.lang.System.getenv().get('SUBJECT_SOURCE_LDAP_PASSWORD_FILE') != null ? org.apache.commons.io.FileUtils.readFileToString(java.lang.System.getenv().get('SUBJECT_SOURCE_LDAP_PASSWORD_FILE'), "utf-8") : java.lang.System.getenv().get('SUBJECT_SOURCE_LDAP_PASSWORD')} +ldap.demo.pass.elConfig = ${java.lang.System.getenv().get('SUBJECT_SOURCE_LDAP_PASSWORD_FILE') != null ? org.apache.commons.io.FileUtils.readFileToString(java.lang.System.getenv().get('SUBJECT_SOURCE_LDAP_PASSWORD_FILE'), "utf-8") : java.lang.System.getenv().get('SUBJECT_SOURCE_LDAP_PASSWORD')} +#ldap.demo.pass = password #optional, if you are using tls, set this to true. Generally you will not be using an SSL URL to use TLS... ldap.demo.tls = false diff --git a/test-compose/configs-and-secrets/grouper/grouper.client.properties b/test-compose/configs-and-secrets/grouper/grouper.client.properties index 5169c718..dcc50ae7 100644 --- a/test-compose/configs-and-secrets/grouper/grouper.client.properties +++ b/test-compose/configs-and-secrets/grouper/grouper.client.properties @@ -55,7 +55,7 @@ grouperClient.webService.login = banderson # password for shared secret authentication to web service # or you can put a filename with an encrypted password -grouperClient.webService.password = ${java.lang.System.getenv().get('GROUPER_CLIENT_WEBSERVICE_PASSWORD_FILE') != null ? org.apache.commons.io.FileUtils.readFileToString(java.lang.System.getenv().get('GROUPER_CLIENT_WEBSERVICE_PASSWORD_FILE'), "utf-8") : java.lang.System.getenv().get('GROUPER_CLIENT_WEBSERVICE_PASSWORD') } +grouperClient.webService.password.elConfig = ${java.lang.System.getenv().get('GROUPER_CLIENT_WEBSERVICE_PASSWORD_FILE') != null ? org.apache.commons.io.FileUtils.readFileToString(java.lang.System.getenv().get('GROUPER_CLIENT_WEBSERVICE_PASSWORD_FILE'), "utf-8") : java.lang.System.getenv().get('GROUPER_CLIENT_WEBSERVICE_PASSWORD') } ################################ @@ -100,7 +100,7 @@ grouper.messaging.system.rabbitmq.defaultSystemName = rabbitmqSystem grouper.messaging.system.rabbitmq.user = guest #pass -grouper.messaging.system.rabbitmq.password = ${java.lang.System.getenv().get('RABBITMQ_PASSWORD_FILE') != null ? org.apache.commons.io.FileUtils.readFileToString(java.lang.System.getenv().get('RABBITMQ_PASSWORD_FILE'), "utf-8") : java.lang.System.getenv().get('RABBITMQ_PASSWORD') } +grouper.messaging.system.rabbitmq.password.elConfig = ${java.lang.System.getenv().get('RABBITMQ_PASSWORD_FILE') != null ? org.apache.commons.io.FileUtils.readFileToString(java.lang.System.getenv().get('RABBITMQ_PASSWORD_FILE'), "utf-8") : java.lang.System.getenv().get('RABBITMQ_PASSWORD') } # set the following three properties if you want to use TLS connection to rabbitmq. All three need to be populated. # TLS Version #grouper.messaging.system.rabbitmqSystem.tlsVersion = TLSv1.1 diff --git a/test-compose/configs-and-secrets/grouper/subject.properties b/test-compose/configs-and-secrets/grouper/subject.properties index fd5b25cd..c9329fdb 100644 --- a/test-compose/configs-and-secrets/grouper/subject.properties +++ b/test-compose/configs-and-secrets/grouper/subject.properties @@ -1,19 +1,25 @@ -subject.sources.xml.location = +#subject.sources.xml.location = + +subjectApi.source.ldap.param.ldapServerId.value = demo + subjectApi.source.ldap.id = ldap subjectApi.source.ldap.name = EDU Ldap subjectApi.source.ldap.types = person subjectApi.source.ldap.adapterClass = edu.internet2.middleware.grouper.subj.GrouperJndiSourceAdapter -subjectApi.source.ldap.param.INITIAL_CONTEXT_FACTORY.value = com.sun.jndi.ldap.LdapCtxFactory -subjectApi.source.ldap.param.PROVIDER_URL.value = ldap://data:389 -subjectApi.source.ldap.param.SECURITY_AUTHENTICATION.value = simple -subjectApi.source.ldap.param.SECURITY_PRINCIPAL.value = cn=admin,dc=internet2,dc=edu -subjectApi.source.ldap.param.SECURITY_CREDENTIALS.value.elConfig = ${java.lang.System.getenv().get('SUBJECT_SOURCE_LDAP_PASSWORD_FILE') != null ? org.apache.commons.io.FileUtils.readFileToString(java.lang.System.getenv().get('SUBJECT_SOURCE_LDAP_PASSWORD_FILE'), "utf-8") : java.lang.System.getenv().get('SUBJECT_SOURCE_LDAP_PASSWORD')} + +## replaced with new LDAPTIVE lib +#subjectApi.source.ldap.param.INITIAL_CONTEXT_FACTORY.value = com.sun.jndi.ldap.LdapCtxFactory +#subjectApi.source.ldap.param.PROVIDER_URL.value = ldap://data:389 +#subjectApi.source.ldap.param.SECURITY_AUTHENTICATION.value = simple +#subjectApi.source.ldap.param.SECURITY_PRINCIPAL.value = cn=admin,dc=internet2,dc=edu +#subjectApi.source.ldap.param.SECURITY_CREDENTIALS.value.elConfig = ${java.lang.System.getenv().get('SUBJECT_SOURCE_LDAP_PASSWORD_FILE') != null ? org.apache.commons.io.FileUtils.readFileToString(java.lang.System.getenv().get('SUBJECT_SOURCE_LDAP_PASSWORD_FILE'), "utf-8") : java.lang.System.getenv().get('SUBJECT_SOURCE_LDAP_PASSWORD')} +#subjectApi.source.ldap.param.VTLDAP_VALIDATOR.value = ConnectLdapValidator + subjectApi.source.ldap.param.SubjectID_AttributeType.value = uid subjectApi.source.ldap.param.SubjectID_formatToLowerCase.value = false subjectApi.source.ldap.param.Name_AttributeType.value = cn subjectApi.source.ldap.param.Description_AttributeType.value = cn -subjectApi.source.ldap.param.VTLDAP_VALIDATOR.value = ConnectLdapValidator subjectApi.source.ldap.param.subjectVirtualAttribute_0_searchAttribute0.value = ${subjectUtils.defaultIfBlank(subject.getAttributeValueOrCommaSeparated('uid'), "")},${subjectUtils.defaultIfBlank(subject.getAttributeValueOrCommaSeparated('cn'), "")},${subjectUtils.defaultIfBlank(subject.getAttributeValueOrCommaSeparated('exampleEduRegId'), "")} subjectApi.source.ldap.param.sortAttribute0.value = cn subjectApi.source.ldap.param.searchAttribute0.value = searchAttribute0 @@ -55,7 +61,7 @@ subjectApi.source.ldap.param.searchAttribute0.value = searchAttribute0 # Each subject has one and only on ID. Returns one result when searching for one ID. subjectApi.source.ldap.search.searchSubject.param.filter.value = (&(uid=%TERM%)(objectclass=person)) subjectApi.source.ldap.search.searchSubject.param.scope.value = SUBTREE_SCOPE -subjectApi.source.ldap.search.searchSubject.param.base.value = ou=people,dc=internet2,dc=edu +subjectApi.source.ldap.search.searchSubject.param.base.value = ou=people #searchSubjectByIdentifier: find a subject by identifier. Identifier is anything that uniquely # identifies the user, e.g. jsmith or jsmith@institution.edu. @@ -63,13 +69,13 @@ subjectApi.source.ldap.search.searchSubject.param.base.value = ou=people,dc=inte # even across sources. Returns one result when searching for one identifier. subjectApi.source.ldap.search.searchSubjectByIdentifier.param.filter.value = (&(|(uid=%TERM%)(employeeNumber=%TERM%))(objectclass=person)) subjectApi.source.ldap.search.searchSubjectByIdentifier.param.scope.value = SUBTREE_SCOPE -subjectApi.source.ldap.search.searchSubjectByIdentifier.param.base.value = ou=people,dc=internet2,dc=edu +subjectApi.source.ldap.search.searchSubjectByIdentifier.param.base.value = ou=people # search: find subjects by free form search. Returns multiple results. subjectApi.source.ldap.search.search.param.filter.value = (&(|(|(uid=%TERM%)(cn=*%TERM%*))(uid=%TERM%*))(objectclass=person)) subjectApi.source.ldap.search.search.param.scope.value = SUBTREE_SCOPE -subjectApi.source.ldap.search.search.param.base.value = ou=people,dc=internet2,dc=edu +subjectApi.source.ldap.search.search.param.base.value = ou=people subjectApi.source.ldap.attributes = givenName, sn, uid, mail, employeeNumber subjectApi.source.ldap.internalAttributes = searchAttribute0 diff --git a/test-compose/data/Dockerfile b/test-compose/data/Dockerfile index 35317837..e4ffb7f8 100644 --- a/test-compose/data/Dockerfile +++ b/test-compose/data/Dockerfile @@ -8,9 +8,10 @@ COPY container_files/conf/ /opt/grouper/grouper.apiBinary/conf/ RUN yum install -y epel-release \ && yum update -y \ && yum install -y 389-ds-base 389-admin 389-adminutil mariadb-server mariadb \ - && yum clean all + && yum clean all \ + && rm -rf /var/cache/yum -RUN mysql_install_db \ +RUN mysql_install_db --force \ && chown -R mysql:mysql /var/lib/mysql/ \ && sed -i 's/^\(bind-address\s.*\)/# \1/' /etc/my.cnf \ && sed -i 's/^\(log_error\s.*\)/# \1/' /etc/my.cnf \ @@ -36,8 +37,8 @@ RUN useradd ldapadmin \ && sed -i '/if (@errs = startServer($inf))/,/}/d' /usr/lib64/dirsrv/perl/* \ && setup-ds.pl --silent --file /seed-data/ds-setup.inf \ && /usr/sbin/ns-slapd -D /etc/dirsrv/slapd-dir \ - && sleep 3 \ - && ldapadd -H ldap:/// -f /seed-data/users.ldif -x -D "cn=Directory Manager" -w password + && while ! curl -s ldap://localhost:389 > /dev/null; do echo waiting for ldap to start; sleep 1; done; \ + ldapadd -H ldap:/// -f /seed-data/users.ldif -x -D "cn=Directory Manager" -w password RUN (/usr/sbin/ns-slapd -D /etc/dirsrv/slapd-dir &) \ && while ! curl -s ldap://localhost:389 > /dev/null; do echo waiting for ldap to start; sleep 1; done; \ diff --git a/test-compose/data/container_files/conf/grouper-loader.properties b/test-compose/data/container_files/conf/grouper-loader.properties new file mode 100644 index 00000000..c7d0bcbc --- /dev/null +++ b/test-compose/data/container_files/conf/grouper-loader.properties @@ -0,0 +1,64 @@ +################################# +## LDAP connections +################################# +# specify the ldap connection with user, pass, url +# the string after "ldap." is the ID of the connection, and it should not have +# spaces or other special chars in it. In this case is it "personLdap" + +#note the URL should start with ldap: or ldaps: if it is SSL. +#It should contain the server and port (optional if not default), and baseDn, +#e.g. ldaps://ldapserver.school.edu:636/dc=school,dc=edu +ldap.demo.url = ldap://localhost:389/dc=internet2,dc=edu + +#optional, if authenticated +ldap.demo.user = cn=admin,dc=internet2,dc=edu +#ldap.demo.user = cn=admin + +#optional, if authenticated note the password can be stored encrypted in an external file +#ldap.demo.pass = ${java.lang.System.getenv().get('SUBJECT_SOURCE_LDAP_PASSWORD_FILE') != null ? org.apache.commons.io.FileUtils.readFileToString(java.lang.System.getenv().get('SUBJECT_SOURCE_LDAP_PASSWORD_FILE'), "utf-8") : java.lang.System.getenv().get('SUBJECT_SOURCE_LDAP_PASSWORD')} +ldap.demo.pass = password + +#optional, if you are using tls, set this to true. Generally you will not be using an SSL URL to use TLS... +ldap.demo.tls = false + +#optional, if using sasl +#ldap.personLdap.saslAuthorizationId = +#ldap.personLdap.saslRealm = + +#optional (note, time limit is for search operations, timeout is for connection timeouts), +#most of these default to vt-ldap defaults. times are in millis +#validateOnCheckout defaults to true if all other validate methods are false +#ldap.personLdap.batchSize = +#ldap.personLdap.countLimit = +#ldap.personLdap.timeLimit = +#ldap.personLdap.timeout = +#ldap.personLdap.minPoolSize = +#ldap.personLdap.maxPoolSize = +#ldap.personLdap.validateOnCheckIn = +#ldap.personLdap.validateOnCheckOut = +#ldap.personLdap.validatePeriodically = +#ldap.personLdap.validateTimerPeriod = +#ldap.personLdap.pruneTimerPeriod = +#if connections expire after a certain amount of time, this is it, in millis, defaults to 300000 (5 minutes) +#ldap.personLdap.expirationTime = + +#make the paths fully qualified and not relative to the loader group. +loader.ldap.requireTopStemAsStemFromConfigGroup=false + +##################################### +## Messaging integration with change log +##################################### +changeLog.consumer.rabbitMqMessagingSample.quartzCron = 0 * * * * ? + +# note, change "messagingSample" in key to be the name of the consumer. e.g. changeLog.consumer.someNameAnyName.class +changeLog.consumer.rabbitMqMessagingSample.class = edu.internet2.middleware.grouper.changeLog.esb.consumer.EsbConsumer + +changeLog.consumer.rabbitMqMessagingSample.publisher.class = edu.internet2.middleware.grouper.changeLog.esb.consumer.EsbMessagingPublisher +changeLog.consumer.rabbitMqMessagingSample.publisher.messagingSystemName = rabbitmq +# note, routingKey property is valid only for rabbitmq. For other messaging systems, it is ignored. +changeLog.consumer.rabbitMqMessagingSample.publisher.routingKey = +## queue or topic +changeLog.consumer.rabbitMqMessagingSample.publisher.messageQueueType = queue +changeLog.consumer.rabbitMqMessagingSample.publisher.queueOrTopicName = sampleQueue +## this is optional if not using "id" for subjectId, need to be a subject attribute in the sources.xml +#changeLog.consumer.rabbitMqMessagingSample.publisher.addSubjectAttributes = email diff --git a/test-compose/data/container_files/conf/subject.properties b/test-compose/data/container_files/conf/subject.properties index a8231911..c9329fdb 100644 --- a/test-compose/data/container_files/conf/subject.properties +++ b/test-compose/data/container_files/conf/subject.properties @@ -1,19 +1,25 @@ -subject.sources.xml.location = +#subject.sources.xml.location = + +subjectApi.source.ldap.param.ldapServerId.value = demo + subjectApi.source.ldap.id = ldap subjectApi.source.ldap.name = EDU Ldap subjectApi.source.ldap.types = person subjectApi.source.ldap.adapterClass = edu.internet2.middleware.grouper.subj.GrouperJndiSourceAdapter -subjectApi.source.ldap.param.INITIAL_CONTEXT_FACTORY.value = com.sun.jndi.ldap.LdapCtxFactory -subjectApi.source.ldap.param.PROVIDER_URL.value = ldap://localhost:389 -subjectApi.source.ldap.param.SECURITY_AUTHENTICATION.value = simple -subjectApi.source.ldap.param.SECURITY_PRINCIPAL.value = cn=admin,dc=internet2,dc=edu -subjectApi.source.ldap.param.SECURITY_CREDENTIALS.value = password + +## replaced with new LDAPTIVE lib +#subjectApi.source.ldap.param.INITIAL_CONTEXT_FACTORY.value = com.sun.jndi.ldap.LdapCtxFactory +#subjectApi.source.ldap.param.PROVIDER_URL.value = ldap://data:389 +#subjectApi.source.ldap.param.SECURITY_AUTHENTICATION.value = simple +#subjectApi.source.ldap.param.SECURITY_PRINCIPAL.value = cn=admin,dc=internet2,dc=edu +#subjectApi.source.ldap.param.SECURITY_CREDENTIALS.value.elConfig = ${java.lang.System.getenv().get('SUBJECT_SOURCE_LDAP_PASSWORD_FILE') != null ? org.apache.commons.io.FileUtils.readFileToString(java.lang.System.getenv().get('SUBJECT_SOURCE_LDAP_PASSWORD_FILE'), "utf-8") : java.lang.System.getenv().get('SUBJECT_SOURCE_LDAP_PASSWORD')} +#subjectApi.source.ldap.param.VTLDAP_VALIDATOR.value = ConnectLdapValidator + subjectApi.source.ldap.param.SubjectID_AttributeType.value = uid subjectApi.source.ldap.param.SubjectID_formatToLowerCase.value = false subjectApi.source.ldap.param.Name_AttributeType.value = cn subjectApi.source.ldap.param.Description_AttributeType.value = cn -subjectApi.source.ldap.param.VTLDAP_VALIDATOR.value = ConnectLdapValidator subjectApi.source.ldap.param.subjectVirtualAttribute_0_searchAttribute0.value = ${subjectUtils.defaultIfBlank(subject.getAttributeValueOrCommaSeparated('uid'), "")},${subjectUtils.defaultIfBlank(subject.getAttributeValueOrCommaSeparated('cn'), "")},${subjectUtils.defaultIfBlank(subject.getAttributeValueOrCommaSeparated('exampleEduRegId'), "")} subjectApi.source.ldap.param.sortAttribute0.value = cn subjectApi.source.ldap.param.searchAttribute0.value = searchAttribute0 @@ -55,7 +61,7 @@ subjectApi.source.ldap.param.searchAttribute0.value = searchAttribute0 # Each subject has one and only on ID. Returns one result when searching for one ID. subjectApi.source.ldap.search.searchSubject.param.filter.value = (&(uid=%TERM%)(objectclass=person)) subjectApi.source.ldap.search.searchSubject.param.scope.value = SUBTREE_SCOPE -subjectApi.source.ldap.search.searchSubject.param.base.value = ou=people,dc=internet2,dc=edu +subjectApi.source.ldap.search.searchSubject.param.base.value = ou=people #searchSubjectByIdentifier: find a subject by identifier. Identifier is anything that uniquely # identifies the user, e.g. jsmith or jsmith@institution.edu. @@ -63,13 +69,13 @@ subjectApi.source.ldap.search.searchSubject.param.base.value = ou=people,dc=inte # even across sources. Returns one result when searching for one identifier. subjectApi.source.ldap.search.searchSubjectByIdentifier.param.filter.value = (&(|(uid=%TERM%)(employeeNumber=%TERM%))(objectclass=person)) subjectApi.source.ldap.search.searchSubjectByIdentifier.param.scope.value = SUBTREE_SCOPE -subjectApi.source.ldap.search.searchSubjectByIdentifier.param.base.value = ou=people,dc=internet2,dc=edu +subjectApi.source.ldap.search.searchSubjectByIdentifier.param.base.value = ou=people # search: find subjects by free form search. Returns multiple results. subjectApi.source.ldap.search.search.param.filter.value = (&(|(|(uid=%TERM%)(cn=*%TERM%*))(uid=%TERM%*))(objectclass=person)) subjectApi.source.ldap.search.search.param.scope.value = SUBTREE_SCOPE -subjectApi.source.ldap.search.search.param.base.value = ou=people,dc=internet2,dc=edu +subjectApi.source.ldap.search.search.param.base.value = ou=people subjectApi.source.ldap.attributes = givenName, sn, uid, mail, employeeNumber subjectApi.source.ldap.internalAttributes = searchAttribute0 diff --git a/test-compose/data/container_files/seed-data/bootstrap.gsh b/test-compose/data/container_files/seed-data/bootstrap.gsh index e2a018aa..4f902129 100644 --- a/test-compose/data/container_files/seed-data/bootstrap.gsh +++ b/test-compose/data/container_files/seed-data/bootstrap.gsh @@ -1,3 +1,10 @@ gs = GrouperSession.startRootSession() +addStem("","app", "app") +addStem("","basis", "basis") +addStem("","bundle", "bundle") +addStem("","org", "org") +addStem("","ref", "ref") +addStem("","test", "test") + addMember("etc:sysadmingroup","banderson"); diff --git a/test-compose/data/container_files/seed-data/ds-setup.inf b/test-compose/data/container_files/seed-data/ds-setup.inf index ae365ca4..9eef33c4 100644 --- a/test-compose/data/container_files/seed-data/ds-setup.inf +++ b/test-compose/data/container_files/seed-data/ds-setup.inf @@ -1,5 +1,5 @@ [General] -AdminDomain = example.edu +AdminDomain = internet2.edu ConfigDirectoryAdminID = admin ConfigDirectoryAdminPwd = admin ConfigDirectoryLdapURL = ldap://localhost:389/o=NetscapeRoot diff --git a/test-compose/docker-compose.yml b/test-compose/docker-compose.yml index b4547273..c0e21ed3 100644 --- a/test-compose/docker-compose.yml +++ b/test-compose/docker-compose.yml @@ -3,7 +3,7 @@ version: "3.3" services: daemon: build: ./daemon/ - command: bash -c "while ! curl -s data:3306 > /dev/null; do echo waiting for mysql to start; sleep 3; done; exec daemon" + command: bash -c "while ! curl -s data:3306 > /dev/null; do echo waiting for mysql to start; sleep 3; done; while ! curl -s ldap://data:389 > /dev/null; do echo waiting for ldap to start; sleep 3; done; exec daemon" depends_on: - data environment: @@ -13,9 +13,6 @@ services: - RABBITMQ_PASSWORD_FILE=/run/secrets/rabbitmq_password.txt - SUBJECT_SOURCE_LDAP_PASSWORD=password - USERTOKEN=build-2 - logging: - options: - tag: "grouper daemon" networks: - back secrets: @@ -46,9 +43,6 @@ services: - GROUPER_DATABASE_PASSWORD_FILE=/run/secrets/database_password.txt - SUBJECT_SOURCE_LDAP_PASSWORD=password - USERTOKEN=build-2 - logging: - options: - tag: "grouper ui" networks: - front - back @@ -100,9 +94,6 @@ services: - GROUPER_DATABASE_PASSWORD_FILE=/run/secrets/database_password.txt - SUBJECT_SOURCE_LDAP_PASSWORD=password - USERTOKEN=build-2 - logging: - options: - tag: "grouoer ws" networks: - front - back @@ -178,9 +169,6 @@ services: - GROUPER_DATABASE_PASSWORD_FILE=/run/secrets/database_password.txt - SUBJECT_SOURCE_LDAP_PASSWORD=password - USERTOKEN=build-2 - logging: - options: - tag: "grouper gsh" networks: - back secrets: @@ -210,6 +198,24 @@ services: ports: - "389:389" - "3306:3306" + secrets: + - database_password.txt + - rabbitmq_password.txt + - source: grouper.hibernate.properties + target: grouper_grouper.hibernate.properties + - source: grouper-loader.properties + target: grouper_grouper-loader.properties + - source: subject.properties + target: grouper_subject.properties + volumes: + - type: bind + source: ./configs-and-secrets/grouper/grouper.properties + target: /opt/grouper/conf/grouper.properties + - type: bind + source: ./configs-and-secrets/grouper/grouper.client.properties + target: /opt/grouper/conf/grouper.client.properties + - grouper_mysql:/var/lib/mysql + - grouper_ldap:/var/lib/dirsrv idp: @@ -261,3 +267,10 @@ secrets: file: ./configs-and-secrets/grouper/subject.properties sp-key.pem: file: ./configs-and-secrets/shibboleth/sp-key.pem + + +volumes: + grouper_mysql: + driver: local + grouper_ldap: + driver: local diff --git a/tests/clairscan.sh b/tests/clairscan.sh new file mode 100755 index 00000000..99e36141 --- /dev/null +++ b/tests/clairscan.sh @@ -0,0 +1,72 @@ +#!/bin/bash + +startsecs=$(date +'%s') +starttime=$(date +%H:%M:%S) + +echo 'starting:' ${starttime} + +#ensure clair-scanner +if [ ! -s ./clair-scanner ]; then + echo 'downloading curl-scanner...' + curl -s -L -o ./clair-scanner https://github.com/arminc/clair-scanner/releases/download/v8/clair-scanner_linux_amd64 + chmod 755 clair-scanner +else + echo 'using existing clair-scanner...' +fi + +#ensure DB container +echo 'ensuring a fresh clair-db container...' +docker ps | grep clair-db &>/dev/null +if [ $? == "0" ]; then + echo 'removing existing clair-db container...' + docker kill db &>/dev/null + docker rm db &>/dev/null + docker run -p 5432:5432 -d --name db arminc/clair-db:latest &>/dev/null +else + docker run -p 5432:5432 -d --name db arminc/clair-db:latest &>/dev/null +fi +sleep 30 + +#ensure clair-scan container +echo 'ensuring a fresh clair-scan container...' +docker ps | grep clair-local-scan &>/dev/null +if [ $? == "0" ]; then + echo 'removing existing clair-scan container...' + docker kill clair &>/dev/null + docker rm clair &>/dev/null + docker run -p 6060:6060 --link db:postgres -d --name clair arminc/clair-local-scan:v2.0.5 &>/dev/null +else + docker run -p 6060:6060 --link db:postgres -d --name clair arminc/clair-local-scan:v2.0.5 &>/dev/null +fi +sleep 30 + +#get ip where clair-scanner will listen +clairip=$(/sbin/ifconfig docker0 | grep 'inet ' | sed 's/^[[:space:]]*//g' | cut -f 2 -d ' ' | sed 's/^[[:space:]]*//g') +echo 'sending ip addr' ${clairip} 'to clair-scan server...' + +#run scan +echo 'running scan...' +./clair-scanner --ip ${clairip} $1 +retcode=$? + +#eval results +if [ $retcode == '0' ]; then + echo 'scan found nothing.' +else + echo 'scan found issues.' +fi + +#cleanup +echo 'removing temporary containers...' +docker kill clair &>/dev/null +docker rm clair &>/dev/null +docker kill db &>/dev/null +docker rm db &>/dev/null + +endsecs=$(date +'%s') +endtime=$(date +%H:%M:%S) +echo 'finished:' $endtime ' ('$((endsecs - startsecs)) 'seconds)' +echo "" + +#pass along return code from scan +exit $retcode diff --git a/tests/main.bats b/tests/main.bats new file mode 100644 index 00000000..0c18d122 --- /dev/null +++ b/tests/main.bats @@ -0,0 +1,16 @@ +#!/usr/bin/env bats + +load ../common + +@test "010 Image is present and healthy" { + docker image inspect ${maintainer}/${imagename} +} + +@test "030 Test Compose the environment" { + cd test-compose && ./compose.sh && docker-compose down +} + + +@test "070 There are no known security vulnerabilities" { + ./tests/clairscan.sh ${maintainer}/${imagename}:latest +}