Rework the architecture of the solution

The architecture of the solution was reworked to a great
extent: now there's a single LDAP directory instead of multiple
ones. Sources are MySQL-only. Container names have been changed.

Bidirectional midPoint <-> Grouper integration works, although
in "full" mode only (no relative changes).

grouper-midpoint/README.md

@@ -8,24 +8,20 @@ In contrary to the original idea, the midPoint -> Grouper connection is realized
 
 There are the following containers:
 
-- `s-data`: source data (LDAP & MySQL), simulating systems of record
-- `m-server`: midPoint application (GUI, REST, tasks, etc); it reads from `s-data`, updates its own repository and pushes data to Grouper via `i-data`
-- `m-data`: midPoint repository (MySQL)
-- `i-data`: intermediate repository for communication from midPoint to Grouper (LDAP)
-- `g-ui`, `g-daemon`, `g-ws`, `gsh`: Grouper containers
-- `g-data`: the Grouper repository (MySQL)
-- `idp`: Shibboleth identity provider; it uses `i-data` as the auhentication source
-- `t-data`: target(s) where identities should be provisioned (currently LDAP)
+- `sources`: source data (MySQL), simulating systems of record
+- `midpoint-server`: midPoint application (GUI, REST, tasks, etc); it reads from `sources`, updates its own repository and `directory`
+- `midpoint-data`: midPoint repository (MySQL)
+- `directory`: central LDAP directory; used also by Grouper and Shibboleth IdP
+- `grouper-ui`, `grouper-daemon`, `grouper-ws`, `gsh`: Grouper containers
+- `grouper-data`: the Grouper repository (MySQL)
+- `idp`: Shibboleth identity provider; it uses `directory` as the auhentication source
+- `targets`: target(s) where identities should be provisioned (currently MySQL)
 
 All files needed to build and compose these containers are in `mp-gr` directory.
 
 TODO ...
 
 TODO:
- - grouper loader jobs
- - grouper -> midPoint connection
- - add banderson to sysadmin group (via midPoint)
- - user passwords in i-data (via midPoint)
+ - Grouper -> midPoint via MQ
  - groups for courses are not created automatically on first import (why?)
- - grouper loader jobs should be created at initialization
  - fix hardcoded password for grouper loader LDAP

grouper-midpoint/mp-gr/configs-and-secrets/grouper/grouper-loader.properties

@@ -8,8 +8,8 @@
 #note the URL should start with ldap: or ldaps: if it is SSL.  
 #It should contain the server and port (optional if not default), and baseDn,
 #e.g. ldaps://ldapserver.school.edu:636/dc=school,dc=edu
-#ldap.demo.url = ldap://i-data:389/dc=internet2,dc=edu
-ldap.demo.url = ldap://i-data:389
+#ldap.demo.url = ldap://directory:389/dc=internet2,dc=edu
+ldap.demo.url = ldap://directory:389
 
 #optional, if authenticated
 ldap.demo.user = cn=admin,dc=internet2,dc=edu

grouper-midpoint/mp-gr/configs-and-secrets/grouper/grouper.client.properties

@@ -78,7 +78,7 @@ grouper.messaging.system.rabbitmqSystem.name = rabbitmqSystem
 grouper.messaging.system.rabbitmqSystem.class = edu.internet2.middleware.grouperMessagingRabbitmq.GrouperMessagingRabbitmqSystem
 
 # host address of rabbitmq queue
-grouper.messaging.system.rabbitmqSystem.host = rabbitmq
+grouper.messaging.system.rabbitmqSystem.host = mq
 
 # virtual host of rabbitmq queue
 grouper.messaging.system.rabbitmqSystem.virtualhost =

grouper-midpoint/mp-gr/configs-and-secrets/grouper/grouper.hibernate.properties

@@ -20,7 +20,7 @@
 # e.g. hsqldb (b):      jdbc:hsqldb:hsql://localhost:9001/grouper
 # e.g. postgres:        jdbc:postgresql://localhost:5432/database
 # e.g. mssql:           jdbc:sqlserver://localhost:3280;databaseName=grouper
-hibernate.connection.url = jdbc:mysql://g-data:3306/grouper?CharSet=utf8&useUnicode=true&characterEncoding=utf8
+hibernate.connection.url = jdbc:mysql://grouper-data:3306/grouper?CharSet=utf8&useUnicode=true&characterEncoding=utf8
 
 hibernate.connection.username         = root
 # If you are using an empty password, depending upon your version of

grouper-midpoint/mp-gr/configs-and-secrets/grouper/subject.properties

@@ -5,7 +5,7 @@ subjectApi.source.ldap.name = EDU Ldap
 subjectApi.source.ldap.types = person
 subjectApi.source.ldap.adapterClass = edu.internet2.middleware.grouper.subj.GrouperJndiSourceAdapter
 subjectApi.source.ldap.param.INITIAL_CONTEXT_FACTORY.value = com.sun.jndi.ldap.LdapCtxFactory
-subjectApi.source.ldap.param.PROVIDER_URL.value = ldap://i-data:389
+subjectApi.source.ldap.param.PROVIDER_URL.value = ldap://directory:389
 subjectApi.source.ldap.param.SECURITY_AUTHENTICATION.value = simple
 subjectApi.source.ldap.param.SECURITY_PRINCIPAL.value = cn=admin,dc=internet2,dc=edu
 subjectApi.source.ldap.param.SECURITY_CREDENTIALS.value.elConfig = ${java.lang.System.getenv().get('SUBJECT_SOURCE_LDAP_PASSWORD_FILE') != null ? org.apache.commons.io.FileUtils.readFileToString(java.lang.System.getenv().get('SUBJECT_SOURCE_LDAP_PASSWORD_FILE'), "utf-8") : java.lang.System.getenv().get('SUBJECT_SOURCE_LDAP_PASSWORD')}

grouper-midpoint/mp-gr/docker-compose.yml

@@ -1,12 +1,12 @@
 version: "3.3"
 
 services:
-  g-daemon:
-    build: ./g-daemon/
-    command: bash -c "while ! curl -s g-data:3306 > /dev/null; do echo waiting for mysql on g-data to start; sleep 3; done; while ! curl -s ldap://i-data:389 > /dev/null; do echo waiting for ldap on i-data to start; sleep 3; done; exec daemon"
+  grouper-daemon:
+    build: ./grouper-daemon/
+    command: bash -c "while ! curl -s grouper-data:3306 > /dev/null; do echo waiting for mysql on grouper-data to start; sleep 3; done; while ! curl -s ldap://directory:389 > /dev/null; do echo waiting for ldap on directory to start; sleep 3; done; exec daemon"
     depends_on:
-     - g-data
-     - i-data
+     - grouper-data
+     - directory
     environment:
      - ENV=dev
      - GROUPER_CLIENT_WEBSERVICE_PASSWORD_FILE=password
@@ -34,12 +34,12 @@ services:
        target: /opt/grouper/conf/grouper.client.properties
 
 
-  g-ui:
-    build: ./g-ui/
-    command: bash -c "while ! curl -s g-data:3306 > /dev/null; do echo waiting for mysql on g-data to start; sleep 3; done; while ! curl -s ldap://i-data:389 > /dev/null; do echo waiting for ldap on i-data to start; sleep 3; done; exec ui"
+  grouper-ui:
+    build: ./grouper-ui/
+    command: bash -c "while ! curl -s grouper-data:3306 > /dev/null; do echo waiting for mysql on grouper-data to start; sleep 3; done; while ! curl -s ldap://directory:389 > /dev/null; do echo waiting for ldap on directory to start; sleep 3; done; exec ui"
     depends_on:
-     - g-data
-     - i-data
+     - grouper-data
+     - directory
     environment:
      - ENV=dev
      - GROUPER_DATABASE_PASSWORD_FILE=/run/secrets/g_database_password.txt
@@ -86,12 +86,12 @@ services:
        target: /etc/pki/tls/certs/cachain.pem
 
 
-#  g-ws:
-#    build: ./g-ws/
-#    command: bash -c "while ! curl -s g-data:3306 > /dev/null; do echo waiting for mysql on g-data to start; sleep 3; done; while ! curl -s ldap://i-data:389 > /dev/null; do echo waiting for ldap on i-data to start; sleep 3; done; exec ws"
+#  grouper-ws:
+#    build: ./grouper-ws/
+#    command: bash -c "while ! curl -s grouper-data:3306 > /dev/null; do echo waiting for mysql on grouper-data to start; sleep 3; done; while ! curl -s ldap://directory:389 > /dev/null; do echo waiting for ldap on directory to start; sleep 3; done; exec ws"
 #    depends_on:
-#     - g-data
-#     - i-data
+#     - grouper-data
+#     - directory
 #    environment:
 #     - ENV=dev
 #     - GROUPER_DATABASE_PASSWORD_FILE=/run/secrets/g_database_password.txt
@@ -130,8 +130,8 @@ services:
 #  gsh:
 #    build: ./gsh/
 #    depends_on:
-#     - g-data
-#     - i-data
+#     - grouper-data
+#     - directory
 #    environment:
 #     - ENV=dev
 #     - GROUPER_DATABASE_PASSWORD_FILE=/run/secrets/g_database_password.txt
@@ -155,8 +155,8 @@ services:
 #       source: ./configs-and-secrets/grouper/grouper.client.properties
 #       target: /opt/grouper/conf/grouper.client.properties
 
-  g-data:
-    build: ./g-data/
+  grouper-data:
+    build: ./grouper-data/
     expose:
      - "3306"
     networks:
@@ -166,57 +166,54 @@ services:
     volumes:
      - grouper_mysql:/var/lib/mysql  
 
-  i-data:
-    build: ./i-data/
+  directory:
+    build: ./directory/
     expose:
      - "389"
     ports:
      - "389:389"
     networks:
      - back
     volumes:
-     - intermediate_ldap:/var/lib/dirsrv
+     - ldap:/var/lib/dirsrv
 
-  s-data:
-    build: ./s-data/
+  sources:
+    build: ./sources/
     expose:
-     - "389"
      - "3306"
     ports:
-     - "1389:389"
      - "13306:3306"
     networks:
      - back
     volumes:
      - source_mysql:/var/lib/mysql
-     - source_ldap:/var/lib/dirsrv
 
-  t-data:
-    build: ./t-data/
+  targets:
+    build: ./targets/
     expose:
-     - "389"
+     - "3306"
     ports:
-     - "2389:389"
+     - "23306:389"
     networks:
      - back
     volumes:
-     - target_ldap:/var/lib/dirsrv
+     - target_mysql:/var/lib/mysql
 
-  m-data:
-    build: ./m-data/
+  midpoint-data:
+    build: ./midpoint-data/
     expose:
      - "3306"
     ports:
-     - "23306:3306"
+     - "33306:3306"
     networks:
      - back
     volumes:
      - midpoint_mysql:/var/lib/mysql
 
-  m-server:
-    build: ./m-server/
+  midpoint-server:
+    build: ./midpoint-server/
     depends_on:
-      - m-data
+      - midpoint-data
     expose:
       - 8080
     ports:
@@ -232,7 +229,7 @@ services:
   idp:
     build: ./idp/
     depends_on:
-     - i-data
+     - directory
     environment:
      - JETTY_MAX_HEAP=64m
      - JETTY_BROWSER_SSL_KEYSTORE_PASSWORD=password
@@ -243,8 +240,7 @@ services:
     ports:
      - "4443:4443"
 
-
-  rabbitmq:
+  mq:
     image: rabbitmq:management
     environment:
      - RABBITMQ_NODENAME=docker-rabbit
@@ -285,8 +281,7 @@ secrets:
 volumes:
   grouper_mysql:
   source_mysql:
-  source_ldap:
-  target_ldap:
-  intermediate_ldap:
+  target_mysql:
+  ldap:
   midpoint_mysql:
   midpoint_home:

grouper-midpoint/mp-gr/g-data/Dockerfile → grouper-midpoint/mp-gr/grouper-data/Dockerfile

@@ -8,7 +8,7 @@ RUN yum install -y epel-release \
     && yum clean all \
     && rm -rf /var/cache/yum
 
-#COPY container_files/seed-data/ /seed-data/
+COPY container_files/seed-data/ /seed-data/
 COPY container_files/conf/ /opt/grouper/grouper.apiBinary/conf/
 
 RUN mysql_install_db \
@@ -28,7 +28,12 @@ RUN mysql_install_db \
 
 RUN (mysqld_safe & ) \
     && while ! curl -s localhost:3306 > /dev/null; do echo waiting for mysqld to start; sleep 1; done; \
-    bin/gsh -registry -check -runscript -noprompt
+    bin/gsh -registry -check -runscript -noprompt && \
+    echo "Running demo.gsh" && \
+    sleep 10 && \
+    bin/gsh /seed-data/demo.gsh && \
+    echo "demo.gsh DONE" && \
+    rm /seed-data/demo.gsh
 
 EXPOSE 3306
 

grouper-midpoint/mp-gr/grouper-data/container_files/seed-data/demo.gsh

@@ -0,0 +1,65 @@
+System.out.println("************** demo.gsh starting...");
+
+gs = GrouperSession.startRootSession();
+addRootStem("ref", "ref");
+addStem("ref", "course", "course")
+addStem("ref", "affiliation", "affiliation")
+
+group = new GroupSave(gs).assignName("etc:affiliationLoader").assignCreateParentStemsIfNotExist(true).save();
+group.getAttributeDelegate().assignAttribute(LoaderLdapUtils.grouperLoaderLdapAttributeDefName()).getAttributeAssign();
+attributeAssign = group.getAttributeDelegate().retrieveAssignment(null, LoaderLdapUtils.grouperLoaderLdapAttributeDefName(), false, true);
+attributeAssign.getAttributeValueDelegate().assignValue(LoaderLdapUtils.grouperLoaderLdapQuartzCronName(), "0 * * * * ?");
+attributeAssign.getAttributeValueDelegate().assignValue(LoaderLdapUtils.grouperLoaderLdapTypeName(), "LDAP_GROUPS_FROM_ATTRIBUTES");
+attributeAssign.getAttributeValueDelegate().assignValue(LoaderLdapUtils.grouperLoaderLdapServerIdName(), "demo");
+attributeAssign.getAttributeValueDelegate().assignValue(LoaderLdapUtils.grouperLoaderLdapFilterName(), "(eduPersonAffiliation=*)");
+attributeAssign.getAttributeValueDelegate().assignValue(LoaderLdapUtils.grouperLoaderLdapSearchDnName(), "ou=People,dc=internet2,dc=edu");
+attributeAssign.getAttributeValueDelegate().assignValue(LoaderLdapUtils.grouperLoaderLdapSubjectAttributeName(), "uid");
+attributeAssign.getAttributeValueDelegate().assignValue(LoaderLdapUtils.grouperLoaderLdapSourceIdName(), "ldap");
+attributeAssign.getAttributeValueDelegate().assignValue(LoaderLdapUtils.grouperLoaderLdapGroupAttributeName(), "eduPersonAffiliation");
+attributeAssign.getAttributeValueDelegate().assignValue(LoaderLdapUtils.grouperLoaderLdapSubjectIdTypeName(), "subjectId");
+attributeAssign.getAttributeValueDelegate().assignValue(LoaderLdapUtils.grouperLoaderLdapGroupNameExpressionName(), 'ref:affiliation:${groupAttribute}_systemOfRecord');
+attributeAssign.getAttributeValueDelegate().assignValue(LoaderLdapUtils.grouperLoaderLdapGroupDisplayNameExpressionName(), '${groupAttribute} system of record');
+attributeAssign.getAttributeValueDelegate().assignValue(LoaderLdapUtils.grouperLoaderLdapGroupTypesName(), "addIncludeExclude");
+
+group = new GroupSave(gs).assignName("etc:deptLoader").assignCreateParentStemsIfNotExist(true).save();
+group.getAttributeDelegate().assignAttribute(LoaderLdapUtils.grouperLoaderLdapAttributeDefName()).getAttributeAssign();
+attributeAssign = group.getAttributeDelegate().retrieveAssignment(null, LoaderLdapUtils.grouperLoaderLdapAttributeDefName(), false, true);
+attributeAssign.getAttributeValueDelegate().assignValue(LoaderLdapUtils.grouperLoaderLdapQuartzCronName(), "0 * * * * ?");
+attributeAssign.getAttributeValueDelegate().assignValue(LoaderLdapUtils.grouperLoaderLdapTypeName(), "LDAP_GROUPS_FROM_ATTRIBUTES");
+attributeAssign.getAttributeValueDelegate().assignValue(LoaderLdapUtils.grouperLoaderLdapServerIdName(), "demo");
+attributeAssign.getAttributeValueDelegate().assignValue(LoaderLdapUtils.grouperLoaderLdapFilterName(), "(businessCategory=*)");
+attributeAssign.getAttributeValueDelegate().assignValue(LoaderLdapUtils.grouperLoaderLdapSearchDnName(), "ou=People,dc=internet2,dc=edu");
+attributeAssign.getAttributeValueDelegate().assignValue(LoaderLdapUtils.grouperLoaderLdapSubjectAttributeName(), "uid");
+attributeAssign.getAttributeValueDelegate().assignValue(LoaderLdapUtils.grouperLoaderLdapSourceIdName(), "ldap");
+attributeAssign.getAttributeValueDelegate().assignValue(LoaderLdapUtils.grouperLoaderLdapGroupAttributeName(), "businessCategory");
+attributeAssign.getAttributeValueDelegate().assignValue(LoaderLdapUtils.grouperLoaderLdapSubjectIdTypeName(), "subjectId");
+attributeAssign.getAttributeValueDelegate().assignValue(LoaderLdapUtils.grouperLoaderLdapGroupNameExpressionName(), 'ref:dept:${groupAttribute}');
+attributeAssign.getAttributeValueDelegate().assignValue(LoaderLdapUtils.grouperLoaderLdapGroupDisplayNameExpressionName(), '${groupAttribute}');
+
+group = new GroupSave(gs).assignName("etc:coursesLoader").assignCreateParentStemsIfNotExist(true).save();
+group.getAttributeDelegate().assignAttribute(LoaderLdapUtils.grouperLoaderLdapAttributeDefName()).getAttributeAssign();
+attributeAssign = group.getAttributeDelegate().retrieveAssignment(null, LoaderLdapUtils.grouperLoaderLdapAttributeDefName(), false, true);
+attributeAssign.getAttributeValueDelegate().assignValue(LoaderLdapUtils.grouperLoaderLdapQuartzCronName(), "0 * * * * ?");
+attributeAssign.getAttributeValueDelegate().assignValue(LoaderLdapUtils.grouperLoaderLdapTypeName(), "LDAP_GROUP_LIST");
+attributeAssign.getAttributeValueDelegate().assignValue(LoaderLdapUtils.grouperLoaderLdapFilterName(), "(cn=*)");
+attributeAssign.getAttributeValueDelegate().assignValue(LoaderLdapUtils.grouperLoaderLdapSearchDnName(), "ou=Courses,ou=Groups,dc=internet2,dc=edu");
+attributeAssign.getAttributeValueDelegate().assignValue(LoaderLdapUtils.grouperLoaderLdapServerIdName(), "demo");
+attributeAssign.getAttributeValueDelegate().assignValue(LoaderLdapUtils.grouperLoaderLdapSourceIdName(), "ldap");
+attributeAssign.getAttributeValueDelegate().assignValue(LoaderLdapUtils.grouperLoaderLdapSubjectExpressionName(), '${loaderLdapElUtils.convertDnToSpecificValue(subjectId)}');
+attributeAssign.getAttributeValueDelegate().assignValue(LoaderLdapUtils.grouperLoaderLdapSubjectAttributeName(), "uniqueMember");
+attributeAssign.getAttributeValueDelegate().assignValue(LoaderLdapUtils.grouperLoaderLdapSubjectIdTypeName(), "subjectId");
+attributeAssign.getAttributeValueDelegate().assignValue(LoaderLdapUtils.grouperLoaderLdapExtraAttributesName(), "cn");
+attributeAssign.getAttributeValueDelegate().assignValue(LoaderLdapUtils.grouperLoaderLdapGroupNameExpressionName(), 'ref:course:${groupAttributes["cn"]}');
+
+group = GroupFinder.findByName(gs, "etc:sysadmingroup", true);
+group.getAttributeDelegate().assignAttribute(LoaderLdapUtils.grouperLoaderLdapAttributeDefName()).getAttributeAssign();
+attributeAssign = group.getAttributeDelegate().retrieveAssignment(null, LoaderLdapUtils.grouperLoaderLdapAttributeDefName(), false, true);
+attributeAssign.getAttributeValueDelegate().assignValue(LoaderLdapUtils.grouperLoaderLdapQuartzCronName(), "0 * * * * ?");
+attributeAssign.getAttributeValueDelegate().assignValue(LoaderLdapUtils.grouperLoaderLdapTypeName(), "LDAP_SIMPLE");
+attributeAssign.getAttributeValueDelegate().assignValue(LoaderLdapUtils.grouperLoaderLdapFilterName(), "(cn=sysadmingroup)");
+attributeAssign.getAttributeValueDelegate().assignValue(LoaderLdapUtils.grouperLoaderLdapSearchDnName(), "ou=Groups,dc=internet2,dc=edu");
+attributeAssign.getAttributeValueDelegate().assignValue(LoaderLdapUtils.grouperLoaderLdapServerIdName(), "demo");
+attributeAssign.getAttributeValueDelegate().assignValue(LoaderLdapUtils.grouperLoaderLdapSourceIdName(), "ldap");
+attributeAssign.getAttributeValueDelegate().assignValue(LoaderLdapUtils.grouperLoaderLdapSubjectExpressionName(), '${loaderLdapElUtils.convertDnToSpecificValue(subjectId)}');
+attributeAssign.getAttributeValueDelegate().assignValue(LoaderLdapUtils.grouperLoaderLdapSubjectAttributeName(), "uniqueMember");
+attributeAssign.getAttributeValueDelegate().assignValue(LoaderLdapUtils.grouperLoaderLdapSubjectIdTypeName(), "subjectId");

grouper-midpoint/mp-gr/idp/shibboleth-idp/conf/ldap.properties

@@ -4,7 +4,7 @@
 #idp.authn.LDAP.authenticator                   = anonSearchAuthenticator
 
 ## Connection properties ##
-idp.authn.LDAP.ldapURL                          = ldap://i-data:389
+idp.authn.LDAP.ldapURL                          = ldap://directory:389
 idp.authn.LDAP.useStartTLS                     = false
 idp.authn.LDAP.useSSL                          = false
 #idp.authn.LDAP.connectTimeout                  = 3000

grouper-midpoint/mp-gr/m-server/Dockerfile → grouper-midpoint/mp-gr/midpoint-server/Dockerfile

@@ -36,7 +36,7 @@ RUN echo 'Extracting midPoint archive...' \
 
 VOLUME ${MP_DIR}/var
 
-ENV REPO_HOST m-data
+ENV REPO_HOST midpoint-data
 ENV REPO_PORT 3306
 ENV REPO_USER root
 ENV REPO_DATABASE midpoint