Merge pull request #150 from internet2/pc_July23

new grouper connector

Workbench/comanage_midpoint_data/container_files/seed-data/createDBforMP.sql

@@ -0,0 +1,99 @@
+CREATE DATABASE grouper_to_midpoint WITH ENCODING=utf8;
+CREATE USER grouper WITH PASSWORD 'password';
+GRANT ALL PRIVILEGES ON DATABASE grouper_to_midpoint TO grouper;
+
+\connect grouper_to_midpoint;
+set role grouper;
+
+CREATE TABLE gr_mp_groups (
+    group_name varchar(1024) NULL, -- Name of group mapped in some way
+    id_index int8 NOT NULL, -- This is the integer identifier for a group and foreign key to group attributes and memberships
+    display_name varchar(1024) NULL, -- Display name of group mapped in some way
+    description varchar(1024) NULL, -- Description of group mapped in some way
+    last_modified int8 NOT NULL, -- Millis since 1970, will be sequential and unique
+    deleted varchar(1) NOT NULL, -- T or F.  Deleted rows will be removed after they have had time to be processed
+    CONSTRAINT gr_mp_groups_pkey PRIMARY KEY (id_index)
+);
+CREATE INDEX gr_mp_groups_ddx ON gr_mp_groups(display_name);
+CREATE INDEX gr_mp_groups_gdx ON gr_mp_groups(group_name);
+CREATE UNIQUE INDEX gr_mp_groups_idx ON gr_mp_groups(id_index);
+CREATE UNIQUE INDEX gr_mp_groups_ldx ON gr_mp_groups(last_modified);
+COMMENT ON TABLE gr_mp_groups IS 'This table holds groups';
+
+COMMENT ON COLUMN gr_mp_groups.group_name IS 'Name of group mapped in some way';
+COMMENT ON COLUMN gr_mp_groups.id_index IS 'This is the integer identifier for a group and foreign key to group attributes and memberships';
+COMMENT ON COLUMN gr_mp_groups.display_name IS 'Display name of group mapped in some way';
+COMMENT ON COLUMN gr_mp_groups.description IS 'Description of group mapped in some way';
+COMMENT ON COLUMN gr_mp_groups.last_modified IS 'Millis since 1970, will be sequential and unique';
+COMMENT ON COLUMN gr_mp_groups.deleted IS 'T or F.  Deleted rows will be removed after they have had time to be processed';
+
+CREATE TABLE gr_mp_subjects (
+    subject_id_index int8 NOT NULL, -- This is the integer identifier for a subject and foreign key to subject attributes and memberships
+    subject_id varchar(1024) NULL, -- Subject ID mapped in some way
+    last_modified int8 NOT NULL, -- Millis since 1970, will be sequential and unique
+    deleted varchar(1) NOT NULL, -- T or F.  Deleted rows will be removed after they have had time to be processed
+    CONSTRAINT gr_mp_subjects_pkey PRIMARY KEY (subject_id_index)
+);
+CREATE UNIQUE INDEX gr_mp_subjects_idx ON gr_mp_subjects(subject_id_index);
+CREATE UNIQUE INDEX gr_mp_subjects_ldx ON gr_mp_subjects(last_modified);
+CREATE INDEX gr_mp_subjects_sdx ON gr_mp_subjects(subject_id);
+COMMENT ON TABLE gr_mp_subjects IS 'This table holds subjects';
+
+COMMENT ON COLUMN gr_mp_subjects.subject_id_index IS 'This is the integer identifier for a subject and foreign key to subject attributes and memberships';
+COMMENT ON COLUMN gr_mp_subjects.subject_id IS 'Subject ID mapped in some way';
+COMMENT ON COLUMN gr_mp_subjects.last_modified IS 'Millis since 1970, will be sequential and unique';
+COMMENT ON COLUMN gr_mp_subjects.deleted IS 'T or F.  Deleted rows will be removed after they have had time to be processed';
+
+CREATE TABLE gr_mp_group_attributes (
+    group_id_index int8 NOT NULL, -- This is the integer identifier for a group and foreign key to groups and memberships
+    attribute_name varchar(1000) NOT NULL, -- Attribute name for attributes not in the main group table
+    attribute_value varchar(4000) NULL, -- Attribute value could be null
+    last_modified int8 NOT NULL, -- Millis since 1970, will be sequential and unique
+    deleted varchar(1) NOT NULL, -- T or F.  Deleted rows will be removed after they have had time to be processed
+    CONSTRAINT gr_mp_group_attributes_fk FOREIGN KEY (group_id_index) REFERENCES gr_mp_groups(id_index)
+);
+CREATE UNIQUE INDEX gr_mp_group_attributes_idx ON gr_mp_group_attributes(group_id_index, attribute_name, attribute_value);
+CREATE UNIQUE INDEX gr_mp_group_attributes_ldx ON gr_mp_group_attributes(last_modified);
+COMMENT ON TABLE gr_mp_group_attributes IS 'This table holds group attributes which are one to one or one to many to the groups table';
+
+COMMENT ON COLUMN gr_mp_group_attributes.group_id_index IS 'This is the integer identifier for a group and foreign key to groups and memberships';
+COMMENT ON COLUMN gr_mp_group_attributes.attribute_name IS 'Attribute name for attributes not in the main group table';
+COMMENT ON COLUMN gr_mp_group_attributes.attribute_value IS 'Attribute value could be null';
+COMMENT ON COLUMN gr_mp_group_attributes.last_modified IS 'Millis since 1970, will be sequential and unique';
+COMMENT ON COLUMN gr_mp_group_attributes.deleted IS 'T or F.  Deleted rows will be removed after they have had time to be processed';
+
+CREATE TABLE gr_mp_memberships (
+    group_id_index int8 NOT NULL, -- This is the foreign key to groups
+    subject_id_index int8 NOT NULL, -- This is the foreign key to subjects
+    last_modified int8 NOT NULL, -- Millis since 1970, will be sequential and unique
+    deleted varchar(1) NOT NULL, -- T or F.  Deleted rows will be removed after they have had time to be processed
+    CONSTRAINT gr_mp_memberships_gfk FOREIGN KEY (group_id_index) REFERENCES gr_mp_groups(id_index),
+    CONSTRAINT gr_mp_memberships_sfk FOREIGN KEY (subject_id_index) REFERENCES gr_mp_subjects(subject_id_index)
+);
+CREATE UNIQUE INDEX gr_mp_memberships_idx ON gr_mp_memberships(group_id_index, subject_id_index);
+CREATE UNIQUE INDEX gr_mp_memberships_ldx ON gr_mp_memberships(last_modified);
+COMMENT ON TABLE gr_mp_memberships IS 'This table holds memberships.  The primary key is group_id_index and subject_id_index';
+
+COMMENT ON COLUMN gr_mp_memberships.group_id_index IS 'This is the foreign key to groups';
+COMMENT ON COLUMN gr_mp_memberships.subject_id_index IS 'This is the foreign key to subjects';
+COMMENT ON COLUMN gr_mp_memberships.last_modified IS 'Millis since 1970, will be sequential and unique';
+COMMENT ON COLUMN gr_mp_memberships.deleted IS 'T or F.  Deleted rows will be removed after they have had time to be processed';
+
+CREATE TABLE gr_mp_subject_attributes (
+    subject_id_index int8 NOT NULL, -- This is the integer identifier and foreign key to subjects
+    attribute_name varchar(1000) NOT NULL, -- Attribute name for attributes not in the main subject table
+    attribute_value varchar(4000) NULL, -- Attribute value could be null
+    last_modified int8 NOT NULL, -- Millis since 1970, will be sequential and unique
+    deleted varchar(1) NOT NULL, -- T or F.  Deleted rows will be removed after they have had time to be processed
+    CONSTRAINT gr_mp_subject_attributes_fk FOREIGN KEY (subject_id_index) REFERENCES gr_mp_subjects(subject_id_index)
+);
+CREATE UNIQUE INDEX gr_mp_subject_attributes_idx ON gr_mp_subject_attributes(subject_id_index, attribute_name, attribute_value);
+CREATE UNIQUE INDEX gr_mp_subject_attributes_ldx ON gr_mp_subject_attributes(last_modified);
+COMMENT ON TABLE gr_mp_subject_attributes IS 'This table holds subject attributes which are one to one or one to many to the subjects table';
+
+COMMENT ON COLUMN gr_mp_subject_attributes.subject_id_index IS 'This is the integer identifier and foreign key to subjects';
+COMMENT ON COLUMN gr_mp_subject_attributes.attribute_name IS 'Attribute name for attributes not in the main subject table';
+COMMENT ON COLUMN gr_mp_subject_attributes.attribute_value IS 'Attribute value could be null';
+COMMENT ON COLUMN gr_mp_subject_attributes.last_modified IS 'Millis since 1970, will be sequential and unique';
+COMMENT ON COLUMN gr_mp_subject_attributes.deleted IS 'T or F.  Deleted rows will be removed after they have had time to be processed';
+

Workbench/configs-and-secrets/grouper/application/grouper-loader.properties

@@ -51,13 +51,11 @@ db.sis.pass = 49321420423
 db.sis.url = jdbc:mysql://sources:3306/sis
 db.sis.driver = com.mysql.jdbc.Driver
 
-
 # midpoint External System
-#db.midPoint.driver = com.mysql.jdbc.Driver
-db.midPoint.driver = com.mysql.cj.jdbc.Driver
+db.midPoint.driver = org.postgresql.Driver
 #db.midPoint.pass = ${java.lang.System.getenv().get('GROUPER_DATABASE_PASSWORD_FILE') != null ? org.apache.commons.io.FileUtils.readFileToString(java.lang.System.getenv().get('GROUPER_DATABASE_PASSWORD_FILE'), "utf-8") : java.lang.System.getenv().get('GROUPER_DATABASE_PASSWORD') }
 db.midPoint.pass = password
-db.midPoint.url = jdbc:mysql://grouper_data:3306/grouper_to_midpoint?CharSet=utf8&useUnicode=true&characterEncoding=utf8
+db.midPoint.url = jdbc:postgresql://comanage_midpoint_data:5432/grouper_to_midpoint?CharSet=utf8
 db.midPoint.user = grouper
 
 # provisioner midpoint
@@ -66,9 +64,14 @@ provisioner.midPoint.customizeEntityCrud = true
 provisioner.midPoint.customizeGroupCrud = true
 provisioner.midPoint.customizeMembershipCrud = true
 provisioner.midPoint.dbExternalSystemConfigId = midPoint
-provisioner.midPoint.deleteEntities = false
-provisioner.midPoint.deleteGroups = false
-provisioner.midPoint.deleteMemberships = false
+provisioner.midPoint.deleteEntities = true
+provisioner.midPoint.deleteEntitiesIfNotExistInGrouper = false
+provisioner.midPoint.deleteEntitiesIfGrouperDeleted = true
+provisioner.midPoint.deleteGroups = true
+provisioner.midPoint.deleteGroupsIfNotExistInGrouper = true
+provisioner.midPoint.deleteMemberships = true
+provisioner.midPoint.deleteMembershipsIfNotExistInGrouper = false
+provisioner.midPoint.deleteMembershipsIfGrouperDeleted = true
 provisioner.midPoint.makeChangesToEntities = true
 provisioner.midPoint.midPointDeletedColumnName = deleted
 provisioner.midPoint.midPointLastModifiedColumnName = last_modified

Workbench/grouper_data/Dockerfile

@@ -10,8 +10,7 @@ RUN yum install -y epel-release \
 
 COPY container_files/conf/ /opt/grouper/grouperWebapp/WEB-INF/classes/
 COPY container_files/bootstrap/ /tmp/
-COPY container_files/mysql/createDBforMP.sql /
-COPY container_files/mysql/setupDBforMP.sql /
+COPY container_files/mysql/createSQLuser.sql /
 
 RUN ln -s /usr/bin/resolveip /usr/libexec/resolveip
 
@@ -27,15 +26,15 @@ RUN mysql_install_db \
     && echo "mysqladmin --silent --wait=30 ping || exit 1" >> /tmp/config \
     && echo "mysql -e 'GRANT ALL PRIVILEGES ON *.* TO \"root\"@\"%\" WITH GRANT OPTION;'" >> /tmp/config \
     && echo "mysql -e 'CREATE DATABASE grouper CHARACTER SET utf8 COLLATE utf8_bin;'" >> /tmp/config \
-    && echo "mysql < /createDBforMP.sql" >> /tmp/config \
-    && echo "mysql -u grouper -p'password' grouper_to_midpoint < /setupDBforMP.sql" >> /tmp/config \
+    && echo "mysql < /createSQLuser.sql" >> /tmp/config \
     && bash /tmp/config \
     && rm -f /tmp/config
 
 RUN (mysqld_safe & ) \
     && while ! curl -s localhost:3306 > /dev/null; do echo waiting for mysqld to start; sleep 1; done; \
     /opt/grouper/grouperWebapp/WEB-INF/bin/gsh.sh -registry -check -runscript -noprompt && \
-    /opt/grouper/grouperWebapp/WEB-INF/bin/gsh.sh /tmp/initialize.gsh
+    /opt/grouper/grouperWebapp/WEB-INF/bin/gsh.sh /tmp/initialize.gsh && \
+    /opt/grouper/grouperWebapp/WEB-INF/bin/gsh.sh /tmp/set-prov.gsh
 
 EXPOSE 3306
 

Workbench/grouper_data/container_files/bootstrap/initialize.gsh

@@ -9,8 +9,16 @@ addStem("", "org", "org")
 addStem("", "test", "test")
 
 addRootStem("ref", "ref")
-addStem("ref", "course", "course")
-addStem("ref", "affiliation", "affiliation")
+addStem("ref", "course", "Course")
+addStem("ref", "dept", "Department")
+addStem("ref", "affiliation", "Affiliation")
+
+new GroupSave().assignName("ref:affiliation:alum").assignDisplayName("Alumni").assignCreateParentStemsIfNotExist(true).save();
+new GroupSave().assignName("ref:affiliation:community").assignDisplayName("Community").assignCreateParentStemsIfNotExist(true).save();
+new GroupSave().assignName("ref:affiliation:faculty").assignDisplayName("Faculty").assignCreateParentStemsIfNotExist(true).save();
+new GroupSave().assignName("ref:affiliation:member").assignDisplayName("Member").assignCreateParentStemsIfNotExist(true).save();
+new GroupSave().assignName("ref:affiliation:staff").assignDisplayName("Staff").assignCreateParentStemsIfNotExist(true).save();
+new GroupSave().assignName("ref:affiliation:student").assignDisplayName("Student").assignCreateParentStemsIfNotExist(true).save();
 
 group = GroupFinder.findByName(gs, "etc:sysadmingroup", true)
 group.getAttributeDelegate().assignAttribute(LoaderLdapUtils.grouperLoaderLdapAttributeDefName()).getAttributeAssign()

Workbench/grouper_data/container_files/bootstrap/set-prov.gsh

@@ -0,0 +1,37 @@
+
+provisioner_name="midPoint";
+GrouperSession grouperSession = GrouperSession.startRootSession();
+
+def setProvOnStem(grouperSession,provisioner_name,folder_name) {
+  AttributeAssign attributeAssignMarker = null;
+  attributeAssignMarker = new AttributeAssignSave(grouperSession).assignOwnerStemName(folder_name).assignNameOfAttributeDefName("etc:provisioning:provisioningMarker").save();
+  new AttributeAssignSave(grouperSession).assignOwnerAttributeAssign(attributeAssignMarker).assignNameOfAttributeDefName("etc:provisioning:provisioningDirectAssign").addValue("true").save();
+  new AttributeAssignSave(grouperSession).assignOwnerAttributeAssign(attributeAssignMarker).assignNameOfAttributeDefName("etc:provisioning:provisioningDoProvision").addValue(provisioner_name).save();
+  new AttributeAssignSave(grouperSession).assignOwnerAttributeAssign(attributeAssignMarker).assignNameOfAttributeDefName("etc:provisioning:provisioningStemScope").addValue("sub").save();
+  new AttributeAssignSave(grouperSession).assignOwnerAttributeAssign(attributeAssignMarker).assignNameOfAttributeDefName("etc:provisioning:provisioningTarget").addValue(provisioner_name).save();
+  new AttributeAssignSave(grouperSession).assignOwnerAttributeAssign(attributeAssignMarker).assignNameOfAttributeDefName("etc:provisioning:provisioningMetadataJson").addValue("{\"destination\":\"midpoint\",\"actor\":\"initial load\"}").save();
+
+}
+
+def setProvOnGroup(grouperSession,provisioner_name,group_name) {
+  AttributeAssign attributeAssignMarker = null;
+  attributeAssignMarker = new AttributeAssignSave(grouperSession).assignOwnerGroupName(group_name).assignNameOfAttributeDefName("etc:provisioning:provisioningMarker").save();
+  new AttributeAssignSave(grouperSession).assignOwnerAttributeAssign(attributeAssignMarker).assignNameOfAttributeDefName("etc:provisioning:provisioningDirectAssign").addValue("true").save();
+  new AttributeAssignSave(grouperSession).assignOwnerAttributeAssign(attributeAssignMarker).assignNameOfAttributeDefName("etc:provisioning:provisioningDoProvision").addValue(provisioner_name).save();
+  new AttributeAssignSave(grouperSession).assignOwnerAttributeAssign(attributeAssignMarker).assignNameOfAttributeDefName("etc:provisioning:provisioningStemScope").addValue("sub").save();
+  new AttributeAssignSave(grouperSession).assignOwnerAttributeAssign(attributeAssignMarker).assignNameOfAttributeDefName("etc:provisioning:provisioningTarget").addValue(provisioner_name).save();
+  new AttributeAssignSave(grouperSession).assignOwnerAttributeAssign(attributeAssignMarker).assignNameOfAttributeDefName("etc:provisioning:provisioningMetadataJson").addValue("{\"destination\":\"midpoint\",\"actor\":\"initial load\"}").save();
+
+}
+
+setProvOnStem(grouperSession,provisioner_name,"app")
+setProvOnStem(grouperSession,provisioner_name,"test")
+setProvOnStem(grouperSession,provisioner_name,"ref:dept")
+setProvOnStem(grouperSession,provisioner_name,"ref:course")
+setProvOnGroup(grouperSession,provisioner_name,"ref:affiliation:alum")
+setProvOnGroup(grouperSession,provisioner_name,"ref:affiliation:community")
+setProvOnGroup(grouperSession,provisioner_name,"ref:affiliation:faculty")
+setProvOnGroup(grouperSession,provisioner_name,"ref:affiliation:member")
+setProvOnGroup(grouperSession,provisioner_name,"ref:affiliation:staff")
+setProvOnGroup(grouperSession,provisioner_name,"ref:affiliation:student")
+

Workbench/grouper_data/container_files/mysql/createDBforMP.sql → Workbench/grouper_data/container_files/mysql/createSQLuser.sql

@@ -1,7 +1,5 @@
-CREATE DATABASE grouper_to_midpoint CHARACTER SET utf8 COLLATE utf8_bin;
 CREATE USER 'grouper'@'%' IDENTIFIED BY 'password';
 CREATE USER 'grouper'@'localhost' IDENTIFIED BY 'password';
 GRANT ALL PRIVILEGES ON * . * TO 'grouper'@'%';
 GRANT ALL PRIVILEGES ON * . * TO 'grouper'@'localhost';
 FLUSH PRIVILEGES;
-

Workbench/idp_ui/Dockerfile

@@ -1,4 +1,4 @@
-FROM i2incommon/shib-idp-ui:1.17.4
+FROM i2incommon/shib-idp-ui:1.18.0
 
 ARG CSPHOSTNAME=localhost
 ENV CSPHOSTNAME=$CSPHOSTNAME

Workbench/idp_ui_api/Dockerfile

@@ -1,4 +1,4 @@
-FROM i2incommon/shib-idp-ui:1.17.4
+FROM i2incommon/shib-idp-ui:1.18.0
 
 ARG CSPHOSTNAME=localhost
 ENV CSPHOSTNAME=$CSPHOSTNAME