Skip to content

Commit

Permalink
SqlSource,FileSource. Mapper and cache engine improvements.
Browse files Browse the repository at this point in the history
  • Loading branch information
@Ioannis
Ioannis committed Nov 28, 2025
1 parent 21f29c3 commit 3a28c0d
Show file tree
Hide file tree
Showing 5 changed files with 228 additions and 74 deletions.
153 changes: 102 additions & 51 deletions app/plugins/Transmogrify/config/schema/tables.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,11 @@
"__COMMENT__": "Template for adding new table configurations to the Transmogrify mapping. Keys starting with double underscores (__) are ignored by processors and exist only for documentation.",
"__EXAMPLE_TABLE_TEMPLATE__": {
"__INSTRUCTIONS__": "Copy this object, rename the key to your logical table name (eg, 'my_items'), and adjust values. Do NOT leave this example enabled in production.",
"__INSTRUCTIONS__": "When mapping legacy type fields, call the function-based mapper (eg, &map...Type) before configuring null for the old column name. The mapper still needs the original source column value, and performNoMapping() will unset that column once it sees a null mapping. In other words, always place the null mapping for the old column after the new field’s function mapping so the mapper can read the legacy value before it is removed.",
"source": "cm_my_items",
"displayField": "name",
"addChangelog": true,
"plugin": "MyItems",
"booleans": ["is_active", "is_primary"],
"cache": [
"co_id",
Expand Down Expand Up @@ -196,6 +198,11 @@
"cache": ["co_id"],
"fieldMap": {
"name": "description",
"sync_affiliation_type_id": "&mapAffiliationType",
"sync_identifier_type_id": "&mapIdentifierType",
"match_identifier_type_id": "&mapIdentifierType",
"match_email_address_type_id": "&mapEmailType",
"co_pipeline_id": "pipeline_id",
"sync_on_add": null,
"sync_on_update": null,
"sync_on_delete": null,
Expand All @@ -205,13 +212,8 @@
"match_type": null,
"sync_affiliation": null,
"sync_identifier_type": null,
"sync_affiliation_type_id": "&mapAffiliationType",
"sync_identifier_type_id": "&mapIdentifierType",
"match_identifier_type_id": "&mapIdentifierType",
"match_email_address_type_id": "&mapEmailType",
"establish_clusters": null,
"co_enrollment_flow_id": null,
"co_pipeline_id": "pipeline_id"
"co_enrollment_flow_id": null
}
},
"external_identity_sources": {
Expand All @@ -222,13 +224,13 @@
"fieldMap": {
"plugin": "&mapExternalIdentitySourcePlugin",
"co_pipeline_id": "pipeline_id",
"org_identity_source_id": "external_identity_source_id",
"sync_mode": null,
"sync_query_mismatch_mode": null,
"sync_query_skip_known": null,
"sync_on_user_login": null,
"eppn_identifier_type": null,
"eppn_suffix": null,
"org_identity_source_id": "external_identity_source_id"
"eppn_suffix": null
}
},
"orcid_sources": {
Expand Down Expand Up @@ -263,17 +265,16 @@
"addChangelog": true,
"fieldMap": {
"org_identity_source_id": "external_identity_source_id",
"duplicate_mode": null,
"redirect_on_duplicate": "redirect_on_duplicate",
"sp_type": "sp_mode",
"default_affiliation_type_id": "&mapAffiliationType",
"default_affiliation": null,
"address_type_id": "&mapToDefaultAddressTypeId",
"email_address_type_id": "&mapToDefaultEmailAddressTypeId",
"name_type_id": "&mapToDefaultNameTypeId",
"telephone_number_type_id": "&mapToDefaultTelephoneNumberTypeId",
"env_o": "env_organization",
"env_ou": "env_department",
"env_identifier_sorid": "env_identifier_sourcekey",
"env_identifier_eppn_login": null,
"env_identifier_eptid_login": null,
"env_identifier_epuid_login": null,
Expand All @@ -282,9 +283,10 @@
"env_identifier_orcid_login": null,
"env_identifier_samlpairwiseid_login": null,
"env_identifier_samlsubjectid_login": null,
"env_identifier_sorid": "env_identifier_sourcekey",
"env_identifier_sorid_login": null,
"env_identifier_network_login": null
"env_identifier_network_login": null,
"duplicate_mode": null,
"default_affiliation": null
}
},
"api_sources": {
Expand Down Expand Up @@ -313,6 +315,47 @@
},
"addChangelog": true
},
"file_sources": {
"source": "cm_file_sources",
"plugin": "FileConnector",
"displayField": "id",
"booleans": [
"threshold_override"
],
"fieldMap": {
"org_identity_source_id": "external_identity_source_id",
"filepath": "filename",
"format": "=C3",
"threshold_warn": "threshold_check"
},
"addChangelog": true
},
"sql_sources": {
"source": "cm_sql_sources",
"plugin": "SqlConnector",
"displayField": "id",
"booleans": [
"threshold_override"
],
"fieldMap": {
"org_identity_source_id": "external_identity_source_id",
"address_type_id": "&mapAddressType",
"email_address_type_id": "&mapEmailType",
"identifier_type_id": "&mapIdentifierType",
"name_type_id": "&mapNameType",
"pronouns_type_id": "&mapPronounsTypeDefault",
"telephone_number_type_id": "&mapTelephoneType",
"url_type_id": "&mapUrlType",
"__NOTE__": "Old columns removal should go after the function mappers",
"address_type": null,
"email_address_type": null,
"identifier_type": null,
"name_type": null,
"telephone_number_type": null,
"url_type": null
},
"addChangelog": true
},
"__NOTES__": "DATA MIGRATIONS",
"authentication_events": {
"source": "cm_authentication_events",
Expand All @@ -337,12 +380,12 @@
"co_person_id": "person_id",
"co_person_role_id": "person_role_id",
"affiliation_type_id": "&mapAffiliationType",
"affiliation": null,
"manager_co_person_id": "manager_person_id",
"sponsor_co_person_id": "sponsor_person_id",
"o": "organization",
"ou": "department",
"source_org_identity_id": null
"source_org_identity_id": null,
"affiliation": null
}
},
"external_identities": {
Expand All @@ -352,7 +395,6 @@
"sqlSelect": "orgidentitiesSqlSelect",
"postRow": "mapExternalIdentityToExternalIdentityRole",
"fieldMap": {
"co_id": null,
"person_id": "&mapOrgIdentityCoPersonId",
"org_identity_id": "external_identity_id",
"title": null,
Expand All @@ -362,7 +404,8 @@
"manager_identifier": null,
"sponsor_identifier": null,
"valid_from": null,
"valid_through": null
"valid_through": null,
"co_id": null
}
},
"groups": {
Expand All @@ -372,10 +415,10 @@
"booleans": ["nesting_mode_all", "open"],
"preRow": "applyCheckGroupNameARRule",
"fieldMap": {
"auto": null,
"co_group_id": "group_id",
"group_type": "?S",
"introduction": null
"introduction": null,
"auto": null
},
"postTable": "createOwnersGroups"
},
Expand All @@ -397,11 +440,11 @@
"fieldMap": {
"co_group_id": "group_id",
"co_person_id": "person_id",
"member": null,
"owner": null,
"co_group_nesting_id": "group_nesting_id",
"co_group_member_id": "group_member_id",
"source_org_identity_id": null
"source_org_identity_id": null,
"member": null,
"owner": null
}
},
"names": {
Expand Down Expand Up @@ -514,12 +557,12 @@
"resolver_co_person_id": "resolver_person_id",
"action": "&mapNotificationAction",
"source_url": "source",
"email_body": "email_body_text",
"source_controller": null,
"source_action": null,
"source_id": null,
"source_arg0": null,
"source_val0": null,
"email_body": "email_body_text"
"source_val0": null
}
},
"history_records": {
Expand All @@ -542,14 +585,14 @@
"displayField": "id",
"fieldMap": {
"job_type": "plugin",
"job_mode": null,
"queue_time": "register_time",
"complete_time": "finish_time",
"job_type_fk": null,
"job_params": "parameters",
"requeued_from_co_job_id": "requeued_from_job_id",
"max_retry": null,
"max_retry_count": null
"max_retry_count": null,
"job_type_fk": null,
"job_mode": null
},
"preRow": "validateJobIsTransmogrifiable"
},
Expand All @@ -575,6 +618,12 @@
"fieldMap": {
"authz_level": "authz_type",
"authz_co_group_id": "authz_group_id",
"notification_co_group_id": "notification_group_id",
"redirect_on_finalize": "redirect_on_finalize",
"status": "=S",
"approval_template_id": "notification_message_template_id",
"finalization_template_id": "finalization_message_template_id",
"co_enrollment_flow_id": "enrollment_flow_id",
"my_identity_shortcut": null,
"match_policy": null,
"match_server_id": null,
Expand All @@ -588,8 +637,6 @@
"invitation_validity": null,
"regenerate_expired_verification": null,
"require_authn": null,
"notification_co_group_id": "notification_group_id",
"status": "=S",
"notify_from": null,
"verification_subject": null,
"verification_body": null,
Expand All @@ -598,26 +645,22 @@
"notify_on_approval": null,
"approval_subject": null,
"approval_body": null,
"approval_template_id": "notification_message_template_id",
"approver_template_id": null,
"denial_template_id": null,
"notify_on_finalize": null,
"finalization_template_id": "finalization_message_template_id",
"introduction_text": null,
"conclusion_text": null,
"introduction_text_pa": null,
"t_and_c_mode": null,
"redirect_on_submit": null,
"redirect_on_confirm": null,
"redirect_on_finalize": "redirect_on_finalize",
"return_url_allowlist": null,
"ignore_authoritative": null,
"duplicate_mode": null,
"co_theme_id": null,
"theme_stacking": null,
"establish_authenticators": null,
"establish_cluster_accounts": null,
"co_enrollment_flow_id": "enrollment_flow_id"
"establish_cluster_accounts": null
}
},
"petitions": {
Expand All @@ -626,47 +669,47 @@
"canSkip": "true",
"fieldMap": {
"co_enrollment_flow_id": "enrollment_flow_id",
"co_id": null,
"cou_id": "cou_id",
"status": "&mapPetitionStatus",
"token": null,
"enrollee_org_identity_id": null,
"archived_org_identity_id": null,
"enrollee_co_person_id": "enrollee_person_id",
"enrollee_co_person_role_id": null,
"petitioner_co_person_id": "petitioner_person_id",
"sponsor_co_person_id": null,
"approver_co_person_id": null,
"co_invite_id": null,
"vetting_request_id": null,
"authenticated_identifier": "petitioner_identifier",
"reference_identifier": "enrollee_identifier",
"co_petition_id": "petition_id",
"petitioner_token": null,
"enrollee_token": null,
"return_url": null,
"approver_comment": null,
"co_petition_id": "petition_id"
"sponsor_co_person_id": null,
"approver_co_person_id": null,
"co_invite_id": null,
"vetting_request_id": null,
"enrollee_org_identity_id": null,
"archived_org_identity_id": null,
"enrollee_co_person_role_id": null,
"token": null,
"co_id": null
}
},
"petition_meta_hist_recs": {
"source": "cm_co_petitions",
"displayField": "id",
"fieldMap": {
"co_enrollment_flow_id": "enrollment_flow_id",
"co_id": null,
"cou_id": null,
"historic_petition_viewer_id" :"&mapHistoricPetitionViewerId",
"enrollee_org_identity_id": "enrollee_org_identity_id",
"archived_org_identity_id": "archived_org_identity_id",
"enrollee_co_person_id": null,
"enrollee_co_person_role_id": "enrollee_person_role_id",
"petitioner_co_person_id": null,
"sponsor_co_person_id": "sponsor_person_id",
"approver_co_person_id": "approver_person_id",
"co_petition_id": "petition_id",
"co_id": null,
"cou_id": null,
"status": null,
"authenticated_identifier": null,
"reference_identifier": null,
"status": null,
"co_petition_id": "petition_id"
"petitioner_co_person_id": null,
"enrollee_co_person_id": null
}
},
"petition_hist_attrs": {
Expand All @@ -690,10 +733,18 @@
"org_identity_source_id": "external_identity_source_id",
"sorid": "source_key",
"org_identity_id": "external_identity_id",
"co_petition_id": null,
"co_person_id": "adopted_person_id",
"reference_identifier": "reference_identifier",
"org_identity_source_record_id": "ext_identity_source_record_id"
"org_identity_source_record_id": "ext_identity_source_record_id",
"co_petition_id": null
}
},
"api_source_records": {
"source": "cm_api_source_records",
"displayField": "id",
"plugin": "ApiConnector",
"fieldMap": {
"sorid": "source_key"
}
}
}
Loading

0 comments on commit 3a28c0d

Please sign in to comment.