Add Grouper->midPoint membership LiveSync

Changes are retrieved from Grouper using
RabbitMQ, then enhanced with MySQL data and passed to midPoint.

Almost working. (Not finished yet.)

grouper-midpoint/mp-gr/configs-and-secrets/grouper/grouper.client.properties

@@ -48,7 +48,7 @@ grouperClient.ldap.password =
 # url of web service, should include everything up to the first resource to access
 # e.g. http://groups.school.edu:8090/grouper-ws/servicesRest
 # e.g. https://groups.school.edu/grouper-ws/servicesRest
-grouperClient.webService.url = https://g-ws/grouper-ws/servicesRest
+grouperClient.webService.url = https://grouper-ws/grouper-ws/servicesRest
 
 # kerberos principal used to connect to web service
 grouperClient.webService.login = banderson

grouper-midpoint/mp-gr/midpoint-server/container_files/res/grouper2/SchemaScript.groovy

@@ -0,0 +1,84 @@
+/* 
+ * ====================
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
+ * 
+ * Copyright 2013 ForgeRock. All rights reserved.
+ * 
+ * The contents of this file are subject to the terms of the Common Development
+ * and Distribution License("CDDL") (the "License").  You may not use this file
+ * except in compliance with the License.
+ * 
+ * You can obtain a copy of the License at
+ * http://opensource.org/licenses/cddl1.php
+ * See the License for the specific language governing permissions and limitations
+ * under the License.
+ * 
+ * When distributing the Covered Code, include this CDDL Header Notice in each file
+ * and include the License file at http://opensource.org/licenses/cddl1.php.
+ * If applicable, add the following below this CDDL Header, with the fields
+ * enclosed by brackets [] replaced by your own identifying information:
+ * "Portions Copyrighted [year] [name of copyright owner]"
+ * ====================
+ * Portions Copyrighted 2013 ConnId.
+ */
+import org.identityconnectors.framework.common.objects.AttributeInfo;
+import org.identityconnectors.framework.common.objects.AttributeInfo.Flags;
+import org.identityconnectors.framework.common.objects.AttributeInfoBuilder;
+import org.identityconnectors.framework.common.objects.ObjectClassInfo;
+import org.identityconnectors.framework.common.objects.ObjectClassInfoBuilder;
+
+// Parameters:
+// The connector sends the following:
+// action: a string describing the action ("SCHEMA" here)
+// log: a handler to the Log facility
+// builder: SchemaBuilder instance for the connector
+//
+// The connector will make the final call to builder.build()
+// so the scipt just need to declare the different object types.
+
+// This sample shows how to create 3 basic ObjectTypes: __ACCOUNT__, __GROUP__ and organization.
+// Each of them contains one required attribute and normal String attributes
+
+
+log.info("Entering "+action+" Script");
+
+// __UID__ = grouper_members.id
+// __NAME__ = grouper_members.subject_id
+accAttrsInfo = new HashSet<AttributeInfo>();
+accAttrsInfo.add(AttributeInfoBuilder.build("subject_id", String.class));
+accAttrsInfo.add(AttributeInfoBuilder.build("subject_identifier0", String.class));
+accAttrsInfo.add(AttributeInfoBuilder.build("sort_string0", String.class));
+accAttrsInfo.add(AttributeInfoBuilder.build("search_string0", String.class));
+accAttrsInfo.add(AttributeInfoBuilder.build("name", String.class));
+accAttrsInfo.add(AttributeInfoBuilder.build("description", String.class));
+accAttrsInfo.add(AttributeInfoBuilder.build("group", String.class, [Flags.MULTIVALUED] as Set));
+ociAccount = new ObjectClassInfoBuilder().setType("__ACCOUNT__").addAllAttributeInfo(accAttrsInfo).build();
+builder.defineObjectClass(ociAccount);
+
+// __UID__ = grouper_groups.id
+// __NAME__ = grouper_groups.name
+grpAttrsInfo = new HashSet<AttributeInfo>();
+grpAttrsInfo.add(AttributeInfoBuilder.build("displayName", String.class));
+grpAttrsInfo.add(AttributeInfoBuilder.build("extension", String.class));
+grpAttrsInfo.add(AttributeInfoBuilder.build("displayExtension", String.class));
+grpAttrsInfo.add(AttributeInfoBuilder.build("description", String.class));
+grpAttrsInfo.add(AttributeInfoBuilder.build("type", String.class));
+ociGroup = new ObjectClassInfoBuilder().setType("__GROUP__").addAllAttributeInfo(grpAttrsInfo).build();
+builder.defineObjectClass(ociGroup);
+
+
+/*
+// Declare the organization attributes
+// Make the name required
+nAIB = new AttributeInfoBuilder("name",String.class);
+nAIB.setRequired(true);
+
+orgAttrsInfo = new HashSet<AttributeInfo>();
+orgAttrsInfo.add(nAIB.build());
+orgAttrsInfo.add(AttributeInfoBuilder.build("description", String.class));
+// Create the organization Object class
+final ObjectClassInfo ociOrg = new ObjectClassInfoBuilder().setType("organization").addAllAttributeInfo(orgAttrsInfo).build();
+builder.defineObjectClass(ociOrg);
+*/
+
+log.info("Schema script done");

grouper-midpoint/mp-gr/midpoint-server/container_files/res/grouper2/SearchScript.groovy

@@ -0,0 +1,112 @@
+/* 
+ * ====================
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
+ * 
+ * Copyright 2013 ForgeRock. All rights reserved.
+ * 
+ * The contents of this file are subject to the terms of the Common Development
+ * and Distribution License("CDDL") (the "License").  You may not use this file
+ * except in compliance with the License.
+ * 
+ * You can obtain a copy of the License at
+ * http://opensource.org/licenses/cddl1.php
+ * See the License for the specific language governing permissions and limitations
+ * under the License.
+ * 
+ * When distributing the Covered Code, include this CDDL Header Notice in each file
+ * and include the License file at http://opensource.org/licenses/cddl1.php.
+ * If applicable, add the following below this CDDL Header, with the fields
+ * enclosed by brackets [] replaced by your own identifying information:
+ * "Portions Copyrighted [year] [name of copyright owner]"
+ * ====================
+ * Portions Copyrighted 2013 ConnId.
+ */
+import groovy.sql.Sql;
+import groovy.sql.DataSet;
+
+// Parameters:
+// The connector sends the following:
+// connection: handler to the SQL connection
+// objectClass: a String describing the Object class (__ACCOUNT__ / __GROUP__ / other)
+// action: a string describing the action ("SEARCH" here)
+// log: a handler to the Log facility
+// options: a handler to the OperationOptions Map
+// query: a handler to the Query Map
+//
+// The Query map describes the filter used.
+//
+// query = [ operation: "CONTAINS", left: attribute, right: "value", not: true/false ]
+// query = [ operation: "ENDSWITH", left: attribute, right: "value", not: true/false ]
+// query = [ operation: "STARTSWITH", left: attribute, right: "value", not: true/false ]
+// query = [ operation: "EQUALS", left: attribute, right: "value", not: true/false ]
+// query = [ operation: "GREATERTHAN", left: attribute, right: "value", not: true/false ]
+// query = [ operation: "GREATERTHANOREQUAL", left: attribute, right: "value", not: true/false ]
+// query = [ operation: "LESSTHAN", left: attribute, right: "value", not: true/false ]
+// query = [ operation: "LESSTHANOREQUAL", left: attribute, right: "value", not: true/false ]
+// query = null : then we assume we fetch everything
+//
+// AND and OR filter just embed a left/right couple of queries.
+// query = [ operation: "AND", left: query1, right: query2 ]
+// query = [ operation: "OR", left: query1, right: query2 ]
+//
+// Returns: A list of Maps. Each map describing one row.
+// !!!! Each Map must contain a '__UID__' and '__NAME__' attribute.
+// This is required to build a ConnectorObject.
+
+log.info("Entering "+action+" Script");
+
+def sql = new Sql(connection);
+def result = []
+def where = "";
+
+switch ( objectClass ) {
+    case "__ACCOUNT__":
+    sql.eachRow("\
+select m.id, m.name, m.subject_id, m.subject_identifier0, m.sort_string0, m.search_string0, m.description, m.subject_source, m.subject_type, group_concat(distinct g.name) as groups \
+from \
+    grouper_members m \
+        left join grouper_memberships_all_v gm on m.id=gm.member_id and gm.owner_id in \
+            (select m.subject_id \
+                from grouper_memberships gm join grouper_members m on gm.member_id=m.id \
+                where gm.owner_id = (select subject_id from grouper_members where name='etc:exportedGroups' and subject_type='group')) \
+        left join grouper_groups g on gm.owner_id=g.id \
+group by m.id \
+having \
+        subject_source = 'ldap' and subject_type = 'person'",
+    {result.add(
+      [__UID__:it.id, 
+      __NAME__:it.subject_id, 
+      subject_id:it.subject_id, 
+      subject_identifier0:it.subject_identifier0, 
+      sort_string0:it.sort_string0, 
+      search_string0:it.search_string0, 
+      name:it.name, 
+      description:it.description, 
+      group:it.groups?.tokenize(',')])} );
+    break
+
+    case "__GROUP__":
+    sql.eachRow("SELECT id, name, display_name, extension, display_extension, description, type_of_group FROM grouper_groups WHERE id in \
+		    (select m.subject_id \
+		        from grouper_memberships gm join grouper_members m on gm.member_id=m.id \
+                            where gm.owner_id = (select subject_id from grouper_members where name='etc:exportedGroups' and subject_type='group'))", 
+      {result.add([
+	__UID__:it.id, 
+	__NAME__:it.name, 
+	displayName:it.display_name, 
+        extension:it.extension,
+	displayExtension:it.display_extension,
+	description:it.description,
+	type:it.type_of_group])} );
+    break
+
+/*
+    case "organization":
+    sql.eachRow("SELECT * FROM Organizations" + where, {result.add([__UID__:it.name, __NAME__:it.name, description:it.description])} );
+    break	*/
+
+    default:
+    result;
+}
+
+return result;

grouper-midpoint/mp-gr/midpoint-server/container_files/res/grouper2/SyncScript.groovy

@@ -0,0 +1,185 @@
+/* 
+ * ====================
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
+ * 
+ * Copyright 2013 ForgeRock. All rights reserved.
+ * 
+ * The contents of this file are subject to the terms of the Common Development
+ * and Distribution License("CDDL") (the "License").  You may not use this file
+ * except in compliance with the License.
+ * 
+ * You can obtain a copy of the License at
+ * http://opensource.org/licenses/cddl1.php
+ * See the License for the specific language governing permissions and limitations
+ * under the License.
+ * 
+ * When distributing the Covered Code, include this CDDL Header Notice in each file
+ * and include the License file at http://opensource.org/licenses/cddl1.php.
+ * If applicable, add the following below this CDDL Header, with the fields
+ * enclosed by brackets [] replaced by your own identifying information:
+ * "Portions Copyrighted [year] [name of copyright owner]"
+ * ====================
+ * Portions Copyrighted 2013 ConnId.
+ */
+import groovy.sql.Sql
+import groovy.sql.DataSet
+import com.rabbitmq.client.*
+
+// Parameters:
+// The connector sends the following:
+// connection: handler to the SQL connection
+// objectClass: a String describing the Object class (__ACCOUNT__ / __GROUP__ / other)
+// action: a string describing the action ("SYNC" or "GET_LATEST_SYNC_TOKEN" here)
+// log: a handler to the Log facility
+// options: a handler to the OperationOptions Map (null if action = "GET_LATEST_SYNC_TOKEN")
+// token: a handler to an Object representing the sync token (null if action = "GET_LATEST_SYNC_TOKEN")
+//
+//
+// Returns:
+// if action = "GET_LATEST_SYNC_TOKEN", it must return an object representing the last known
+// sync token for the corresponding ObjectClass
+// 
+// if action = "SYNC":
+// A list of Maps . Each map describing one update:
+// Map should look like the following:
+//
+// [
+// "token": <Object> token object (could be Integer, Date, String) , [!! could be null]
+// "operation":<String> ("CREATE_OR_UPDATE"|"DELETE")  will always default to CREATE_OR_DELETE ,
+// "uid":<String> uid  (uid of the entry) ,
+// "previousUid":<String> prevuid (This is for rename ops) ,
+// "password":<String> password (optional... allows to pass clear text password if needed),
+// "attributes":Map<String,List> of attributes name/values
+// ]
+
+log.info("Entering "+action+" Script");
+def sql = new Sql(connection);
+
+if (action.equalsIgnoreCase("GET_LATEST_SYNC_TOKEN")) {
+  return 0
+} else if (action.equalsIgnoreCase("SYNC")) {
+
+
+  factory = new ConnectionFactory()
+  factory.host = 'mq'
+  factory.port = 5672
+  connection = factory.newConnection()
+  channel = connection.createChannel()
+  println 'RabbitMQ: conn=' + connection + ', channel=' + channel
+
+  result = []
+
+  for (;;) {
+    response = channel.basicGet('sampleQueue', false)
+    println 'got response: ' + response
+    if (response == null) {
+      break
+    }
+    body = response.body
+    if (body == null) {
+      log.warn('null body in {}', response)
+      continue
+    }
+    text = new String(body)
+    println 'Got message:\n' + text
+
+    jsonSlurper = new groovy.json.JsonSlurper()
+    msg = jsonSlurper.parseText(text)
+
+    events = msg?.esbEvent
+    println 'events = ' + events
+    if (events == null || events.isEmpty()) {
+      println 'esbEvent is null or empty, getting next message; events = ' + events
+      continue
+    }
+
+    for (event in events) {
+
+      type = event.eventType
+      if (type != 'MEMBERSHIP_ADD' && type != 'MEMBERSHIP_DELETE') {
+        println 'event type does not match, getting next message; type = ' + type
+        continue
+      }
+      if (event.sourceId != 'ldap') {
+        println 'sourceId does not match, getting next message; sourceId = ' + event.sourceId
+        continue
+      }
+
+      // the user membership has changed: let's fetch the current status of the user (ConnId requires full 'new state' anyway)
+      subjectId = event.subjectId
+      if (subjectId == null) {
+        println 'subjectId is null, getting next message'
+        continue
+      }
+      println 'subject membership changed: ' + subjectId
+
+      sql.eachRow("\
+select m.id, m.name, m.subject_id, m.subject_identifier0, m.sort_string0, m.search_string0, m.description, m.subject_source, m.subject_type, group_concat(distinct g.name) as groups \
+from \
+    grouper_members m \
+        left join grouper_memberships_all_v gm on m.id=gm.member_id and gm.owner_id in \
+            (select m.subject_id \
+                from grouper_memberships gm join grouper_members m on gm.member_id=m.id \
+                where gm.owner_id = (select subject_id from grouper_members where name='etc:exportedGroups' and subject_type='group')) \
+        left join grouper_groups g on gm.owner_id=g.id \
+group by m.id \
+having \
+        subject_source = 'ldap' and subject_type = 'person' and subject_id = '" + subjectId + "'",
+      {result.add(
+        [operation:"CREATE_OR_UPDATE",
+        token:System.currentTimeMillis(),
+        __UID__:it.id,
+        __NAME__:it.subject_id,
+        subject_id:it.subject_id,
+        subject_identifier0:it.subject_identifier0,
+        sort_string0:it.sort_string0,
+        search_string0:it.search_string0,
+        name:it.name,
+        description:it.description,
+        group:it.groups?.tokenize(',')])} )
+    }
+  }
+
+  channel.close()
+  connection.close()
+
+  println 'result is\n' + result
+
+  return result
+
+/*
+  def result = [];
+  def tstamp = null;
+  if (token != null){
+    tstamp = new java.sql.Timestamp(token);
+  }
+  else{
+    def today= new Date();
+    tstamp = new java.sql.Timestamp(today.time);
+  }
+
+  switch ( objectClass ) {
+  case "__ACCOUNT__":
+    sql.eachRow("select * from Users where timestamp > ${tstamp}",
+      {result.add([operation:"CREATE_OR_UPDATE", uid:it.uid, token:it.timestamp.getTime(), 
+            attributes:[firstname:it.firstname,fullname:it.fullname, lastname:it.lastname, email:it.email, organization:it.organization]])}
+    )
+    break;
+    
+  case "__GROUP__":
+    sql.eachRow("select * from Groups where timestamp > ${tstamp}",
+      {result.add([operation:"CREATE_OR_UPDATE", uid:it.gid,token:it.timestamp.getTime(), 
+            attributes:[gid:it.gid,name:it.name,description:it.description]])}
+    );
+    break;
+  }
+    
+  log.ok("Sync script: found "+result.size()+" events to sync");
+  return result;
+*/
+
+}
+else {
+  log.error("Sync script: action '"+action+"' is not implemented in this script");
+  return null;
+}