Merge branch 'master' into 3.9-SNAPSHOT-stable

# Conflicts:
#	demo/complex/grouper-daemon/Dockerfile
#	demo/complex/grouper-ui/Dockerfile
#	download-midpoint
#	midpoint/README.md
#	midpoint/build.sh
#	midpoint/midpoint-server/container_files/httpd/conf/midpoint.conf.auth.shibboleth

.dockerignore

@@ -0,0 +1,3 @@
+demo
+.git
+tests

midpoint/midpoint-server/Dockerfile → Dockerfile

@@ -31,9 +31,14 @@ RUN chmod 755 /opt/tier/setenv.sh \
     && chmod 755 /usr/local/bin/setup-cron.sh \
     && chmod 755 /usr/local/bin/start-midpoint.sh \
     && chmod 755 /usr/local/bin/start-httpd.sh \
-    && chmod 755 /usr/local/bin/startup.sh
+    && chmod 755 /usr/local/bin/startup.sh \
+    && chmod 755 /usr/local/bin/healthcheck.sh
 
 RUN cp /dev/null /etc/httpd/conf.d/ssl.conf \
+    && mv /etc/httpd/conf.d/shib.conf /etc/httpd/conf.d/shib.conf.auth.shibboleth \
+    && touch /etc/httpd/conf.d/shib.conf.auth.internal \
+    && mv /etc/httpd/conf.modules.d/00-shib.conf /etc/httpd/conf.modules.d/00-shib.conf.auth.shibboleth \
+    && touch /etc/httpd/conf.modules.d/00-shib.conf.auth.internal \
     && sed -i 's/LogFormat "/LogFormat "httpd;access_log;%{ENV}e;%{USERTOKEN}e;/g' /etc/httpd/conf/httpd.conf \
     && echo -e "\nErrorLogFormat \"httpd;error_log;%{ENV}e;%{USERTOKEN}e;[%{u}t] [%-m:%l] [pid %P:tid %T] %7F: %E: [client\ %a] %M% ,\ referer\ %{Referer}i\"" >> /etc/httpd/conf/httpd.conf \
     && sed -i 's/CustomLog "logs\/access_log"/CustomLog "\/tmp\/loghttpd"/g' /etc/httpd/conf/httpd.conf \
@@ -60,13 +65,15 @@ VOLUME ${MP_DIR}/var
 
 # Repository parameters
 
-ENV REPO_HOST midpoint-data
-ENV REPO_PORT default
-ENV REPO_USER root
-ENV REPO_DATABASE midpoint
-ENV REPO_JDBC_URL default
-ENV REPO_PASSWORD_FILE /run/secrets/m_database_password.txt
 ENV REPO_DATABASE_TYPE mariadb
+ENV REPO_JDBC_URL default
+ENV REPO_HOST midpoint_data
+ENV REPO_PORT default
+ENV REPO_DATABASE registry
+ENV REPO_USER registry_user
+ENV REPO_PASSWORD_FILE /run/secrets/mp_database_password.txt
+ENV REPO_MISSING_SCHEMA_ACTION create
+ENV REPO_UPGRADEABLE_SCHEMA_ACTION stop
 
 # Logging parameters
 
@@ -83,10 +90,11 @@ ENV LOGOUT_URL https://localhost:8443/Shibboleth.sso/Logout
 
 # Other parameters
 
-ENV KEYSTORE_PASSWORD_FILE /run/secrets/m_keystore_password.txt
-ENV MEM 2048m
-ENV TIER_RELEASE=not-released-yet
-ENV TIER_MAINTAINER=tier
+ENV MP_KEYSTORE_PASSWORD_FILE /run/secrets/mp_keystore_password.txt
+ENV MP_MEM_MAX 2048m
+ENV MP_MEM_INIT 1024m
+ENV TIER_RELEASE not-released-yet
+ENV TIER_MAINTAINER tier
 
 # TIER Beacon Opt-out
 # Completely uncomment the following ENV line to prevent the containers from sending analytics information to Internet2.
@@ -100,4 +108,6 @@ ENV TIER_MAINTAINER=tier
 
 RUN /opt/tier/setenv.sh
 
+HEALTHCHECK --interval=1m --timeout=30s --start-period=2m CMD /usr/local/bin/healthcheck.sh
+
 CMD ["/usr/local/bin/startup.sh"]

Jenkinsfile

@@ -3,76 +3,88 @@ pipeline {
     environment { 
         maintainer = "t"
         imagename = 'm'
-        imagename_data = 'md'
         tag = 'l'
     }
     stages {
-        stage('Setting build context') {
+        stage ('Setting build context') {
             steps {
                 script {
                     maintainer = maintain()
                     imagename = imagename()
-                    imagename_data = imagename_data()
-                    if(env.BRANCH_NAME == "master") {
+                    if (env.BRANCH_NAME == "master" || env.BRANCH_NAME == "bats") {		// temporary
                        tag = "latest"
                     } else {
                        tag = env.BRANCH_NAME
                     }
-                    if(!imagename || !imagename_data){
-                        echo "You must define imagename and imagename_data in common.bash"
+                    if (!imagename) {
+                        echo "You must define imagename in common.bash"
                         currentBuild.result = 'FAILURE'
-                     }
+                    }
                     sh 'mkdir -p bin'
                     sh 'mkdir -p tmp'
-                    dir('tmp'){
-                      git([ url: "https://github.internet2.edu/docker/util.git", credentialsId: "jenkins-github-access-token" ])
-                      sh 'ls'
-                      sh 'mv bin/* ../bin/.'
+                    dir ('tmp') {
+                        git([ url: "https://github.internet2.edu/docker/util.git", credentialsId: "jenkins-github-access-token" ])
+                        sh 'ls'
+                        sh 'mv bin/* ../bin/.'
                     }
                 }  
-             }
+            }
         }    
-        stage('Clean') {
+        stage ('Build') {
+            steps {
+                script {
+                    try {
+			sh '(ls -l ; docker ps -a ; docker image ls ; echo Destroying ; bin/destroy.sh ; docker image ls) 2>&1 | tee debug'	// temporary
+                        sh './download-midpoint 2>&1 | tee -a debug ; test ${PIPESTATUS[0]} -eq 0'
+                        sh 'bin/rebuild.sh 2>&1 | tee -a debug ; test ${PIPESTATUS[0]} -eq 0'
+                        //sh 'echo Build output ; cat debug'
+                    } catch (error) {
+                        def error_details = readFile('./debug')
+                        def message = "BUILD ERROR: There was a problem building ${imagename}:${tag}. \n\n ${error_details}"
+                        sh "rm -f ./debug"
+                        handleError(message)
+                    }
+                }
+            }
+        }
+        stage ('Test') {
             steps {
                 script {
-                   try{
-                     sh 'bin/destroy.sh >> debug'
-                   } catch(error) {
-                     def error_details = readFile('./debug');
-                     def message = "BUILD ERROR: There was a problem building the Base Image. \n\n ${error_details}"
-                     sh "rm -f ./debug"
-                     handleError(message)
-                   }
+                    try {
+                        sh 'echo Docker containers before root tests ; docker ps -a'		// temporary
+                        sh 'bin/test.sh 2>&1 | tee debug ; test ${PIPESTATUS[0]} -eq 0'
+                        sh 'echo Docker containers before compositions tests ; docker ps -a'		// temporary
+
+                        sh '(cd demo/simple ; bats tests ) 2>&1 | tee -a debug ; test ${PIPESTATUS[0]} -eq 0'
+                        sh '(cd demo/shibboleth ; bats tests ) 2>&1 | tee -a debug ; test ${PIPESTATUS[0]} -eq 0'
+                        sh '(cd demo/postgresql ; bats tests ) 2>&1 | tee -a debug ; test ${PIPESTATUS[0]} -eq 0'
+
+                        // temporarily disabled
+                        //sh '(cd demo/complex ; bats tests ) 2>&1 | tee -a debug ; test ${PIPESTATUS[0]} -eq 0'
+                        // sh 'echo Test output ; cat debug'
+                    } catch (error) {
+                        def error_details = readFile('./debug')
+                        def message = "BUILD ERROR: There was a problem testing ${imagename}:${tag}. \n\n ${error_details}"
+                        sh "rm -f ./debug"
+                        handleError(message)
+                    }
                 }
             }
-        } 
-        stage('Build') {
+        }
+        stage ('Push') {
             steps {
                 script {
-                   sh 'midpoint/download-midpoint'
-                   docker.withRegistry('https://registry.hub.docker.com/',   "dockerhub-$maintainer") {
-                      def baseImg = docker.build("$maintainer/$imagename", "--no-cache midpoint/midpoint-server")
-                      // test the environment 
-                      // sh 'cd test-compose && ./compose.sh'
-                      // bring down after testing
-                      // sh 'cd test-compose && docker-compose down'
-                      baseImg.push("$tag")
-                   }
-                   docker.withRegistry('https://registry.hub.docker.com/',   "dockerhub-$maintainer") {
-                      def baseImg = docker.build("$maintainer/$imagename_data", "--no-cache midpoint/midpoint-data")
-                      // test the environment 
-                      // sh 'cd test-compose && ./compose.sh'
-                      // bring down after testing
-                      // sh 'cd test-compose && docker-compose down'
-                      baseImg.push("$tag")
-                   }
-               }
+                    docker.withRegistry('https://registry.hub.docker.com/', "dockerhub-$maintainer") {
+                        def baseImg = docker.build("$maintainer/$imagename")
+                        baseImg.push("$tag")
+                    }
+                }
             }
         }
-        stage('Notify') {
+        stage ('Notify') {
             steps {
                 echo "$maintainer"
-                slackSend color: 'good', message: "$maintainer/$imagename:$tag and $maintainer/$imagename_data:$tag pushed to DockerHub"
+                slackSend color: 'good', message: "$maintainer/$imagename:$tag pushed to DockerHub"
             }
         }
     }
@@ -82,31 +94,25 @@ pipeline {
         }
         failure {
             // slackSend color: 'good', message: "Build failed"
-            handleError("BUILD ERROR: There was a problem building ${maintainer}/${imagename}:${tag} or ${maintainer}/${imagename_data}:${tag}.")
+            handleError("BUILD ERROR: There was a problem building ${maintainer}/${imagename}:${tag}.")
         }
     }
 }
 
 
 def maintain() {
-  def matcher = readFile('common.bash') =~ 'maintainer="(.+)"'
-  matcher ? matcher[0][1] : 'tier'
+    def matcher = readFile('common.bash') =~ 'maintainer="(.+)"'
+    matcher ? matcher[0][1] : 'tier'
 }
 
 def imagename() {
-  def matcher = readFile('common.bash') =~ 'imagename="(.+)"'
-  matcher ? matcher[0][1] : null
-}
-
-def imagename_data() {
-  def matcher = readFile('common.bash') =~ 'imagename_data="(.+)"'
-  matcher ? matcher[0][1] : null
+    def matcher = readFile('common.bash') =~ 'imagename="(.+)"'
+    matcher ? matcher[0][1] : null
 }
 
-def handleError(String message){
-  echo "${message}"
-  currentBuild.setResult("FAILED")
-  slackSend color: 'danger', message: "${message}"
-  //step([$class: 'Mailer', notifyEveryUnstableBuild: true, recipients: 'chubing@internet2.edu', sendToIndividuals: true])
-  sh 'exit 1'
+def handleError(String message) {
+    echo "${message}"
+    currentBuild.setResult("FAILED")
+    slackSend color: 'danger', message: "${message}"
+    sh 'exit 1'
 }

README.md

@@ -1,28 +1,29 @@
 [![Build Status](https://jenkins.testbed.tier.internet2.edu/job/docker/job/midPoint_container/job/master/badge/icon)](https://jenkins.testbed.tier.internet2.edu/job/docker/job/midPoint_container/job/master/)
 
-This repository contains sources for TIER-supported images related to [Evolveum midPoint](http://midpoint.evolveum.com).
+This repository contains sources for TIER-supported [midPoint](http://midpoint.evolveum.com) image.
 
-# Images
-- `midpoint` contains the midPoint application along with some TIER-specific components: Apache reverse proxy with optional Shibboleth filter and TIER Beacon.
-- `midpoint-mariadb` hosts the default MariaDB database tailored to meet midPoint needs. It can be exchanged for another repository implementation.
+The image contains the midPoint application along with some TIER-specific components: Apache reverse proxy with optional Shibboleth filter and TIER Beacon.
 
 # Supported tags
-These tags apply to both containers:
 - latest
 - midPoint version-specific tags, e.g. 3.9, 3.9.1, 4.0, etc.
 
 # Content
-- `midpoint` directory contains build instructions for both containers (`midpoint` and `midpoint-mariadb`),
-- `demo` directory contains three demonstration scenarios:
+- the root directory contains build instructions for the `midpoint` image 
+- `demo` directory contains a couple of demonstration scenarios:
+  - `simple` to show simple composition of midPoint with the repository,
   - `shibboleth` to show integration with Shibboleth IdP,
   - `postgresql` to show how to change the repository implementation,
   - `complex` to demonstrate more complex deployment of midPoint in a sample university environment, featuring midPoint along with Grouper, LDAP directory, RabbitMQ, Shibboleth IdP, source and target systems.
 
 # Build instructions
-Please see specific subdirectories: [midpoint](midpoint) and [demo/complex](demo/complex).
+```
+$ ./build.sh
+```
+You can then continue with one of demo composition, e.g. simple or complex one.
 
 # Documentation
-- For the `midpoint` and `midpoint-mariadb` containers themselves please see [Dockerized midPoint](https://spaces.at.internet2.edu/display/MID/Dockerized+midPoint) wiki page.
+- For the `midpoint` image and container themselves please see [Dockerized midPoint](https://spaces.at.internet2.edu/display/MID/Dockerized+midPoint) wiki page.
 - For the complex demo please see [midPoint - Grouper integration demo](https://spaces.at.internet2.edu/display/MID/midPoint+-+Grouper+integration+demo) wiki page.
 
 This is a work in progress. For its current status please see [Status of the work](https://spaces.at.internet2.edu/display/MID/Status+of+the+work).

build.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+cd "$(dirname "$0")"
+SKIP_DOWNLOAD=0
+while getopts "nh?" opt; do
+    case $opt in
+    n) SKIP_DOWNLOAD=1 ;;
+    h | ?) echo "Options: -n skip download" ; exit 0 ;;
+    *) echo "Unknown option: $opt" ; exit 1 ;;
+    esac
+done
+if [ "$SKIP_DOWNLOAD" = "0" ]; then ./download-midpoint; fi
+docker build --tag tier/midpoint:latest .
+echo "---------------------------------------------------------------------------------------"
+echo "The midPoint containers were successfully built. To start them, execute the following:"
+echo ""
+echo "(for simple demo)"
+echo ""
+echo "$ cd" $(pwd)/demo/simple
+echo "$ docker-compose up"
+echo ""
+echo "(for complex demo)"
+echo ""
+echo "$ cd" $(pwd)/demo/complex
+echo "$ docker-compose up --build"

common.bash

@@ -1,3 +1,2 @@
 maintainer="tier"
 imagename="midpoint"
-imagename_data="midpoint-mariadb"

container_files/httpd/conf/midpoint.conf.auth.internal

@@ -0,0 +1,6 @@
+
+Timeout 2400
+ProxyTimeout 2400
+ProxyBadHeader Ignore
+
+ProxyPass /midpoint ajp://localhost:9090/midpoint  timeout=2400 retry=0

midpoint/midpoint-server/container_files/httpd/conf/midpoint.conf.auth.shibboleth → container_files/httpd/conf/midpoint.conf.auth.shibboleth

@@ -3,7 +3,7 @@ Timeout 2400
 ProxyTimeout 2400
 ProxyBadHeader Ignore
 
-ProxyPass /midpoint ajp://localhost:9090/midpoint  timeout=2400
+ProxyPass /midpoint ajp://localhost:9090/midpoint  timeout=2400 retry=0
 
 <Location /midpoint>
   AuthType shibboleth

midpoint/midpoint-server/container_files/mp-dir/repository-url → container_files/mp-dir/repository-url

@@ -1,7 +1,7 @@
 #!/bin/bash
 
 function resolvePort() {
-    if [ $REPO_PORT == "default" ]; then
+    if [[ $REPO_PORT == "default" ]]; then
         case $REPO_DATABASE_TYPE in
             mariadb)
               echo 3306
@@ -26,7 +26,7 @@ function resolvePort() {
     fi
 }
 
-if [ $REPO_JDBC_URL == "default" ]; then
+if [[ $REPO_JDBC_URL == "default" ]]; then
   REPO_PORT=$( resolvePort )
   case $REPO_DATABASE_TYPE in
 	mariadb)

midpoint/midpoint-server/container_files/supervisor/supervisord.conf → container_files/supervisor/supervisord.conf

@@ -16,10 +16,10 @@ command=/bin/bash -c "/usr/local/bin/start-midpoint.sh"
 stdout_logfile=/dev/fd/2
 stdout_logfile_maxbytes=0
 redirect_stderr=true
+autorestart=false
 
 [program:crond]
 command=/usr/sbin/crond -n -i -m off
 stdout_logfile=/tmp/logcrond
 stdout_logfile_maxbytes=0
 redirect_stderr=true
-autorestart=false

container_files/usr-local-bin/healthcheck.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+(set -o pipefail ; curl -k -f https://localhost:443/midpoint/actuator/health | tr -d '[:space:]' | grep -q "\"status\":\"UP\"") || exit 1

container_files/usr-local-bin/start-httpd.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+echo "Linking secrets and config files; using authentication: $AUTHENTICATION"
+ln -sf /run/secrets/mp_sp-key.pem /etc/shibboleth/sp-key.pem
+ln -sf /run/secrets/mp_host-key.pem /etc/pki/tls/private/host-key.pem
+ln -sf /etc/httpd/conf.d/midpoint.conf.auth.$AUTHENTICATION /etc/httpd/conf.d/midpoint.conf
+ln -sf /etc/httpd/conf.d/shib.conf.auth.$AUTHENTICATION /etc/httpd/conf.d/shib.conf
+ln -sf /etc/httpd/conf.modules.d/00-shib.conf.auth.$AUTHENTICATION /etc/httpd/conf.modules.d/00-shib.conf
+
+case $AUTHENTICATION in
+  shibboleth)
+    echo "*** Starting httpd WITH Shibboleth support"
+    httpd-shib-foreground
+    ;;
+  internal)
+    echo "*** Starting httpd WITHOUT Shibboleth support"
+    rm -f /etc/httpd/logs/httpd.pid /run/httpd/httpd.pid
+    httpd -DFOREGROUND
+    ;;
+  *)
+    echo "*** Couldn't start httpd: unsupported AUTHENTICATION variable value: '$AUTHENTICATION'"
+    sleep infinity
+    ;;
+esac