Add grouper loaders and t-data container

Also added server-side files: ScriptedSQL connector and scripts
for the connections (SIS->midPoint, Grouper->midPoint).

grouper-midpoint/README.md

@@ -12,12 +12,20 @@ There are the following containers:
 - `m-server`: midPoint application (GUI, REST, tasks, etc); it reads from `s-data`, updates its own repository and pushes data to Grouper via `i-data`
 - `m-data`: midPoint repository (MySQL)
 - `i-data`: intermediate repository for communication from midPoint to Grouper (LDAP)
-- `g-ui`, `g-daemon`, `g-ws`: Grouper containers
+- `g-ui`, `g-daemon`, `g-ws`, `gsh`: Grouper containers
 - `g-data`: the Grouper repository (MySQL)
 - `idp`: Shibboleth identity provider; it uses `i-data` as the auhentication source
+- `t-data`: target(s) where identities should be provisioned (currently LDAP)
 
 All files needed to build and compose these containers are in `mp-gr` directory.
 
 TODO ...
 
-TODO: how to initialize things
+TODO:
+ - grouper loader jobs
+ - grouper -> midPoint connection
+ - add banderson to sysadmin group (via midPoint)
+ - user passwords in i-data (via midPoint)
+ - groups for courses are not created automatically on first import (why?)
+ - grouper loader jobs should be created at initialization
+ - fix hardcoded password for grouper loader LDAP

grouper-midpoint/mp-gr/configs-and-secrets/grouper/grouper-loader.properties

@@ -8,13 +8,15 @@
 #note the URL should start with ldap: or ldaps: if it is SSL.  
 #It should contain the server and port (optional if not default), and baseDn,
 #e.g. ldaps://ldapserver.school.edu:636/dc=school,dc=edu
-ldap.demo.url = ldap://i-data:389/dc=example,dc=edu
+#ldap.demo.url = ldap://i-data:389/dc=internet2,dc=edu
+ldap.demo.url = ldap://i-data:389
 
 #optional, if authenticated
 ldap.demo.user = cn=admin,dc=internet2,dc=edu
 
 #optional, if authenticated note the password can be stored encrypted in an external file
-ldap.demo.pass = ${java.lang.System.getenv().get('SUBJECT_SOURCE_LDAP_PASSWORD_FILE') != null ? org.apache.commons.io.FileUtils.readFileToString(java.lang.System.getenv().get('SUBJECT_SOURCE_LDAP_PASSWORD_FILE'), "utf-8") : java.lang.System.getenv().get('SUBJECT_SOURCE_LDAP_PASSWORD')}
+#ldap.demo.pass = ${java.lang.System.getenv().get('SUBJECT_SOURCE_LDAP_PASSWORD_FILE') != null ? org.apache.commons.io.FileUtils.readFileToString(java.lang.System.getenv().get('SUBJECT_SOURCE_LDAP_PASSWORD_FILE'), "utf-8") : java.lang.System.getenv().get('SUBJECT_SOURCE_LDAP_PASSWORD')}
+ldap.demo.pass=password
 
 #optional, if you are using tls, set this to true.  Generally you will not be using an SSL URL to use TLS...
 ldap.demo.tls = false

grouper-midpoint/mp-gr/docker-compose.yml

@@ -191,6 +191,17 @@ services:
      - source_mysql:/var/lib/mysql
      - source_ldap:/var/lib/dirsrv
 
+  t-data:
+    build: ./t-data/
+    expose:
+     - "389"
+    ports:
+     - "2389:389"
+    networks:
+     - back
+    volumes:
+     - target_ldap:/var/lib/dirsrv
+
   m-data:
     build: ./m-data/
     expose:
@@ -275,6 +286,7 @@ volumes:
   grouper_mysql:
   source_mysql:
   source_ldap:
+  target_ldap:
   intermediate_ldap:
   midpoint_mysql:
   midpoint_home:

grouper-midpoint/mp-gr/i-data/Dockerfile

@@ -1,7 +1,7 @@
 FROM centos:centos7
 
 LABEL author="tier-packaging@internet2.edu <tier-packaging@internet2.edu>"
- 
+
 RUN yum install -y epel-release \
     && yum update -y \
     && yum install -y 389-ds-base 389-admin 389-adminutil \
@@ -25,4 +25,6 @@ RUN useradd ldapadmin \
 
 EXPOSE 389
 
-CMD /usr/sbin/ns-slapd -D /etc/dirsrv/slapd-dir && sleep 100000000
+# temporary!
+
+CMD rm -rf /var/lock/dirsrv/slapd-dir/server/* && /usr/sbin/ns-slapd -D /etc/dirsrv/slapd-dir && sleep 100000000

grouper-midpoint/mp-gr/i-data/container_files/seed-data/ds-setup.inf

@@ -16,7 +16,7 @@ ServerIpAddress = 0.0.0.0
 SysUser = nobody
 
 [slapd]
-AddOrgEntries = Yes
+AddOrgEntries = No
 AddSampleEntries = No
 InstallLdifFile = suggest
 RootDN = cn=Directory Manager

grouper-midpoint/mp-gr/i-data/container_files/seed-data/users.ldif

@@ -5,20 +5,16 @@ cn: admin
 userPassword: password
 description: LDAP administrator
 
-dn: uid=banderson,ou=People,dc=internet2,dc=edu
-objectClass: organizationalPerson
-objectClass: person
-objectClass: top
-objectClass: inetOrgPerson
-givenName: Bob
-uid: banderson
-sn: Anderson
-cn: Bob Anderson
-userPassword: password
-
 dn: cn=users,ou=Groups,dc=internet2,dc=edu
 objectClass: groupOfUniqueNames
 objectClass: top
 uniqueMember: uid=banderson,ou=People,dc=internet2,dc=edu
 uniqueMember: uid=jsmith,ou=People,dc=internet2,dc=edu
 cn: users
+
+dn: ou=Courses,ou=Groups,dc=internet2,dc=edu
+objectClass: top
+objectClass: organizationalUnit
+ou: Courses
+
+

grouper-midpoint/mp-gr/m-server/Dockerfile

@@ -29,6 +29,7 @@ ENV MP_DIR /opt/midpoint
 RUN mkdir -p ${MP_DIR}/var
 
 COPY ${MP_DIST_FILE} ${MP_DIR}
+COPY container_files/ ${MP_DIR}/var/
 
 RUN echo 'Extracting midPoint archive...' \
  && tar xzf ${MP_DIR}/midpoint-${MP_VERSION}-dist.tar.gz -C ${MP_DIR} --strip-components=1

grouper-midpoint/mp-gr/m-server/container_files/res/grouper/SchemaScript.groovy

@@ -0,0 +1,91 @@
+/* 
+ * ====================
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
+ * 
+ * Copyright 2013 ForgeRock. All rights reserved.
+ * 
+ * The contents of this file are subject to the terms of the Common Development
+ * and Distribution License("CDDL") (the "License").  You may not use this file
+ * except in compliance with the License.
+ * 
+ * You can obtain a copy of the License at
+ * http://opensource.org/licenses/cddl1.php
+ * See the License for the specific language governing permissions and limitations
+ * under the License.
+ * 
+ * When distributing the Covered Code, include this CDDL Header Notice in each file
+ * and include the License file at http://opensource.org/licenses/cddl1.php.
+ * If applicable, add the following below this CDDL Header, with the fields
+ * enclosed by brackets [] replaced by your own identifying information:
+ * "Portions Copyrighted [year] [name of copyright owner]"
+ * ====================
+ * Portions Copyrighted 2013 ConnId.
+ */
+import org.identityconnectors.framework.common.objects.AttributeInfo;
+import org.identityconnectors.framework.common.objects.AttributeInfo.Flags;
+import org.identityconnectors.framework.common.objects.AttributeInfoBuilder;
+import org.identityconnectors.framework.common.objects.ObjectClassInfo;
+import org.identityconnectors.framework.common.objects.ObjectClassInfoBuilder;
+
+// Parameters:
+// The connector sends the following:
+// action: a string describing the action ("SCHEMA" here)
+// log: a handler to the Log facility
+// builder: SchemaBuilder instance for the connector
+//
+// The connector will make the final call to builder.build()
+// so the scipt just need to declare the different object types.
+
+// This sample shows how to create 3 basic ObjectTypes: __ACCOUNT__, __GROUP__ and organization.
+// Each of them contains one required attribute and normal String attributes
+
+
+log.info("Entering "+action+" Script");
+
+// Declare the __ACCOUNT__ attributes
+// Make the uid required
+uidAIB = new AttributeInfoBuilder("uid",String.class);
+uidAIB.setRequired(true);
+
+accAttrsInfo = new HashSet<AttributeInfo>();
+accAttrsInfo.add(uidAIB.build());
+accAttrsInfo.add(AttributeInfoBuilder.build("subject_id", String.class));
+accAttrsInfo.add(AttributeInfoBuilder.build("subject_identifier0", String.class));
+accAttrsInfo.add(AttributeInfoBuilder.build("sort_string0", String.class));
+accAttrsInfo.add(AttributeInfoBuilder.build("search_string0", String.class));
+accAttrsInfo.add(AttributeInfoBuilder.build("name", String.class));
+accAttrsInfo.add(AttributeInfoBuilder.build("description", String.class));
+accAttrsInfo.add(AttributeInfoBuilder.build("group", String.class, [Flags.MULTIVALUED] as Set));
+// Create the __ACCOUNT__ Object class
+final ObjectClassInfo ociAccount = new ObjectClassInfoBuilder().setType("__ACCOUNT__").addAllAttributeInfo(accAttrsInfo).build();
+builder.defineObjectClass(ociAccount);
+
+/*
+// Declare the __GROUP__ attributes
+// Make the gid required
+gidAIB = new AttributeInfoBuilder("gid",String.class);
+gidAIB.setRequired(true);
+
+grpAttrsInfo = new HashSet<AttributeInfo>();
+grpAttrsInfo.add(gidAIB.build());
+grpAttrsInfo.add(AttributeInfoBuilder.build("name", String.class));
+grpAttrsInfo.add(AttributeInfoBuilder.build("description", String.class));
+// Create the __GROUP__ Object class
+final ObjectClassInfo ociGroup = new ObjectClassInfoBuilder().setType("__GROUP__").addAllAttributeInfo(grpAttrsInfo).build();
+builder.defineObjectClass(ociGroup);
+
+
+// Declare the organization attributes
+// Make the name required
+nAIB = new AttributeInfoBuilder("name",String.class);
+nAIB.setRequired(true);
+
+orgAttrsInfo = new HashSet<AttributeInfo>();
+orgAttrsInfo.add(nAIB.build());
+orgAttrsInfo.add(AttributeInfoBuilder.build("description", String.class));
+// Create the organization Object class
+final ObjectClassInfo ociOrg = new ObjectClassInfoBuilder().setType("organization").addAllAttributeInfo(orgAttrsInfo).build();
+builder.defineObjectClass(ociOrg);
+*/
+
+log.info("Schema script done");

grouper-midpoint/mp-gr/m-server/container_files/res/grouper/SearchScript.groovy

@@ -0,0 +1,101 @@
+/* 
+ * ====================
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
+ * 
+ * Copyright 2013 ForgeRock. All rights reserved.
+ * 
+ * The contents of this file are subject to the terms of the Common Development
+ * and Distribution License("CDDL") (the "License").  You may not use this file
+ * except in compliance with the License.
+ * 
+ * You can obtain a copy of the License at
+ * http://opensource.org/licenses/cddl1.php
+ * See the License for the specific language governing permissions and limitations
+ * under the License.
+ * 
+ * When distributing the Covered Code, include this CDDL Header Notice in each file
+ * and include the License file at http://opensource.org/licenses/cddl1.php.
+ * If applicable, add the following below this CDDL Header, with the fields
+ * enclosed by brackets [] replaced by your own identifying information:
+ * "Portions Copyrighted [year] [name of copyright owner]"
+ * ====================
+ * Portions Copyrighted 2013 ConnId.
+ */
+import groovy.sql.Sql;
+import groovy.sql.DataSet;
+
+// Parameters:
+// The connector sends the following:
+// connection: handler to the SQL connection
+// objectClass: a String describing the Object class (__ACCOUNT__ / __GROUP__ / other)
+// action: a string describing the action ("SEARCH" here)
+// log: a handler to the Log facility
+// options: a handler to the OperationOptions Map
+// query: a handler to the Query Map
+//
+// The Query map describes the filter used.
+//
+// query = [ operation: "CONTAINS", left: attribute, right: "value", not: true/false ]
+// query = [ operation: "ENDSWITH", left: attribute, right: "value", not: true/false ]
+// query = [ operation: "STARTSWITH", left: attribute, right: "value", not: true/false ]
+// query = [ operation: "EQUALS", left: attribute, right: "value", not: true/false ]
+// query = [ operation: "GREATERTHAN", left: attribute, right: "value", not: true/false ]
+// query = [ operation: "GREATERTHANOREQUAL", left: attribute, right: "value", not: true/false ]
+// query = [ operation: "LESSTHAN", left: attribute, right: "value", not: true/false ]
+// query = [ operation: "LESSTHANOREQUAL", left: attribute, right: "value", not: true/false ]
+// query = null : then we assume we fetch everything
+//
+// AND and OR filter just embed a left/right couple of queries.
+// query = [ operation: "AND", left: query1, right: query2 ]
+// query = [ operation: "OR", left: query1, right: query2 ]
+//
+// Returns: A list of Maps. Each map describing one row.
+// !!!! Each Map must contain a '__UID__' and '__NAME__' attribute.
+// This is required to build a ConnectorObject.
+
+log.info("Entering "+action+" Script");
+
+def sql = new Sql(connection);
+def result = []
+def where = "";
+
+switch ( objectClass ) {
+    case "__ACCOUNT__":
+    sql.eachRow("\
+select m.id, m.name, m.subject_id, m.subject_identifier0, m.sort_string0, m.search_string0, m.description, m.subject_source, group_concat(distinct g.name) as groups \
+from \
+    grouper_members m \
+        left join grouper_memberships_all_v gm on m.id=gm.member_id and gm.owner_id in \
+            (select m.subject_id \
+                from grouper_memberships gm join grouper_members m on gm.member_id=m.id \
+                where gm.owner_id = (select subject_id from grouper_members where name='etc:exportedGroups' and subject_type='group')) \
+        left join grouper_groups g on gm.owner_id=g.id \
+group by m.id \
+having \
+        subject_source = 'ldap';", 
+    {result.add(
+      [__UID__:it.id, 
+      __NAME__:it.subject_id, 
+      uid:it.id, 
+      subject_id:it.subject_id, 
+      subject_identifier0:it.subject_identifier0, 
+      sort_string0:it.sort_string0, 
+      search_string0:it.search_string0, 
+      name:it.name, 
+      description:it.description, 
+      group:it.groups?.tokenize(',')])} );
+    break
+
+/*    case "__GROUP__":
+    sql.eachRow("SELECT * FROM Groups" + where, {result.add([__UID__:it.name, __NAME__:it.name, gid:it.gid, ,description:it.description])} );
+    break
+
+    case "organization":
+    sql.eachRow("SELECT * FROM Organizations" + where, {result.add([__UID__:it.name, __NAME__:it.name, description:it.description])} );
+    break	*/
+
+    default:
+    result;
+}
+
+return result;

grouper-midpoint/mp-gr/m-server/container_files/res/grouper/TestScript.groovy

@@ -0,0 +1,38 @@
+/* 
+ * ====================
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
+ * 
+ * Copyright 2013 ForgeRock. All rights reserved.
+ * 
+ * The contents of this file are subject to the terms of the Common Development
+ * and Distribution License("CDDL") (the "License").  You may not use this file
+ * except in compliance with the License.
+ * 
+ * You can obtain a copy of the License at
+ * http://opensource.org/licenses/cddl1.php
+ * See the License for the specific language governing permissions and limitations
+ * under the License.
+ * 
+ * When distributing the Covered Code, include this CDDL Header Notice in each file
+ * and include the License file at http://opensource.org/licenses/cddl1.php.
+ * If applicable, add the following below this CDDL Header, with the fields
+ * enclosed by brackets [] replaced by your own identifying information:
+ * "Portions Copyrighted [year] [name of copyright owner]"
+ * ====================
+ * Portions Copyrighted 2013 ConnId.
+ */
+import groovy.sql.Sql;
+import groovy.sql.DataSet;
+
+// Parameters:
+// The connector sends the following:
+// connection: handler to the SQL connection
+// action: a string describing the action ("TEST" here)
+// log: a handler to the Log facility
+
+log.info("Entering "+action+" Script");
+def sql = new Sql(connection);
+
+sql.eachRow("select * from grouper_members limit 10", { println it.subject_id } );
+
+