Skip to content

Commit

Permalink
ApiSources
Browse files Browse the repository at this point in the history
  • Loading branch information
@Ioannis
Ioannis committed Nov 27, 2025
1 parent 66e466f commit 21f29c3
Show file tree
Hide file tree
Showing 6 changed files with 463 additions and 143 deletions.
9 changes: 8 additions & 1 deletion app/plugins/Transmogrify/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,12 +72,19 @@ These options come directly from TransmogrifyCommand::buildOptionParser.
- --groups-colon-replacement STRING
- Optional: replace ":" with STRING in Standard group names during migration (opt‑in). Use with care; the name "CO" remains invalid and will not be auto‑renamed.

- --plugin-bootstrap
- Initialize plugin registry and activate plugins referenced in tables.json.
- Runs only the plugin bootstrap step and exits with success; it does not process or migrate any tables.
- Transmogrify scans tables.json for tables that declare a "plugin" and activates those plugins if present but suspended (eg, table "servers" with "plugin": "CoreServer" implies model "CoreServer.Servers"). Useful when preparing the deployment for a later migration that depends on plugin-provided models/schemas.

## Typical usage

- Migrate everything using the default mapping
- bin/cake transmogrify

- Run only plugin bootstrap (sync plugin registry and activate plugins referenced by tables.json), then exit (no migration)
- bin/cake transmogrify --plugin-bootstrap

- Preview environment information
- bin/cake transmogrify --info
- bin/cake transmogrify --info --info-json
Expand Down Expand Up @@ -114,7 +121,7 @@ Hints:
- You’ll see warnings for rows skipped due to unresolved foreign keys or missing type mappings.
- Progress UI:
- A single‑line progress bar updates in place.
- Warnings and errors appear under the bar as they happen.
- Warnings and errors appear over the bar as they happen.


## Per-table skip prompt
Expand Down
224 changes: 130 additions & 94 deletions app/plugins/Transmogrify/config/schema/tables.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,7 @@
"source": "cm_api_users",
"displayField": "username",
"booleans": ["privileged"],
"cache": ["co_id"],
"fieldMap": {
"password": "api_key"
}
Expand All @@ -100,6 +101,7 @@
},
"servers": {
"source": "cm_servers",
"plugin": "CoreServer",
"displayField": "description",
"addChangelog": true,
"cache": ["co_id", "status"],
Expand All @@ -110,6 +112,7 @@
},
"http_servers": {
"source": "cm_http_servers",
"plugin": "CoreServer",
"displayField": "serverurl",
"addChangelog": true,
"booleans": [
Expand All @@ -125,6 +128,7 @@
},
"oauth2_servers": {
"source": "cm_oauth2_servers",
"plugin": "CoreServer",
"displayField": "serverurl",
"addChangelog": true,
"cache": ["server_id"],
Expand All @@ -136,6 +140,7 @@
},
"sql_servers": {
"source": "cm_sql_servers",
"plugin": "CoreServer",
"displayField": "hostname",
"cache": ["server_id"],
"addChangelog": true,
Expand All @@ -145,6 +150,7 @@
},
"match_servers": {
"source": "cm_match_servers",
"plugin": "CoreServer",
"displayField": "username",
"addChangelog": true,
"booleans": [
Expand All @@ -162,6 +168,7 @@
"match_server_attributes": {
"source": "cm_match_server_attributes",
"displayField": "attribute",
"plugin": "CoreServer",
"addChangelog": true,
"booleans": [
"required"
Expand All @@ -183,6 +190,129 @@
"co_message_template_id": "message_template_id"
}
},
"pipelines": {
"source": "cm_co_pipelines",
"displayField": "description",
"cache": ["co_id"],
"fieldMap": {
"name": "description",
"sync_on_add": null,
"sync_on_update": null,
"sync_on_delete": null,
"sync_coperson_status": null,
"sync_coperson_attributes": null,
"create_role": null,
"match_type": null,
"sync_affiliation": null,
"sync_identifier_type": null,
"sync_affiliation_type_id": "&mapAffiliationType",
"sync_identifier_type_id": "&mapIdentifierType",
"match_identifier_type_id": "&mapIdentifierType",
"match_email_address_type_id": "&mapEmailType",
"establish_clusters": null,
"co_enrollment_flow_id": null,
"co_pipeline_id": "pipeline_id"
}
},
"external_identity_sources": {
"source": "cm_org_identity_sources",
"displayField": "description",
"booleans": ["hash_source_record"],
"cache": ["co_id"],
"fieldMap": {
"plugin": "&mapExternalIdentitySourcePlugin",
"co_pipeline_id": "pipeline_id",
"sync_mode": null,
"sync_query_mismatch_mode": null,
"sync_query_skip_known": null,
"sync_on_user_login": null,
"eppn_identifier_type": null,
"eppn_suffix": null,
"org_identity_source_id": "external_identity_source_id"
}
},
"orcid_sources": {
"source": "cm_orcid_sources",
"displayField": "id",
"plugin": "OrcidSource",
"booleans": [
"scope_inherit"
],
"cache": ["external_identity_source_id", "server_id"],
"fieldMap": {
"org_identity_source_id": "external_identity_source_id",
"default_affiliation_type_id": "&mapToDefaultAffiliationTypeId",
"address_type_id": "&mapToDefaultAddressTypeId",
"email_address_type_id": "&mapToDefaultEmailAddressTypeId",
"name_type_id": "&mapToDefaultNameTypeId",
"telephone_number_type_id": "&mapToDefaultTelephoneNumberTypeId"
},
"addChangelog": true
},
"orcid_tokens": {
"source": "cm_orcid_tokens",
"displayField": "id",
"plugin": "OrcidSource",
"cache": ["orcid_identifier", "orcid_source_id"],
"addChangelog": true
},
"env_sources": {
"source": "cm_env_sources",
"plugin": "EnvSource",
"displayField": "id",
"addChangelog": true,
"fieldMap": {
"org_identity_source_id": "external_identity_source_id",
"duplicate_mode": null,
"redirect_on_duplicate": "redirect_on_duplicate",
"sp_type": "sp_mode",
"default_affiliation_type_id": "&mapAffiliationType",
"default_affiliation": null,
"address_type_id": "&mapToDefaultAddressTypeId",
"email_address_type_id": "&mapToDefaultEmailAddressTypeId",
"name_type_id": "&mapToDefaultNameTypeId",
"telephone_number_type_id": "&mapToDefaultTelephoneNumberTypeId",
"env_o": "env_organization",
"env_ou": "env_department",
"env_identifier_eppn_login": null,
"env_identifier_eptid_login": null,
"env_identifier_epuid_login": null,
"env_identifier_oidcsub_login": null,
"env_identifier_orcid": null,
"env_identifier_orcid_login": null,
"env_identifier_samlpairwiseid_login": null,
"env_identifier_samlsubjectid_login": null,
"env_identifier_sorid": "env_identifier_sourcekey",
"env_identifier_sorid_login": null,
"env_identifier_network_login": null
}
},
"api_sources": {
"source": "cm_api_sources",
"plugin": "ApiConnector",
"displayField": "id",
"postRow": "createApiForApiSource",
"fieldMap": {
"org_identity_source_id": "external_identity_source_id",
"api_user_id": null,
"poll_mode": null,
"kafka_server_id": null
},
"addChangelog": true
},
"api_source_endpoints": {
"source": "cm_api_sources",
"plugin": "ApiConnector",
"displayField": "id",
"fieldMap": {
"org_identity_source_id": "external_identity_source_id",
"api_id": "&mapApiIdFromCache",
"api_user_id": null,
"poll_mode": null,
"kafka_server_id": null
},
"addChangelog": true
},
"__NOTES__": "DATA MIGRATIONS",
"authentication_events": {
"source": "cm_authentication_events",
Expand Down Expand Up @@ -551,47 +681,6 @@
},
"addChangelog": true
},
"pipelines": {
"source": "cm_co_pipelines",
"displayField": "description",
"cache": ["co_id"],
"fieldMap": {
"name": "description",
"sync_on_add": null,
"sync_on_update": null,
"sync_on_delete": null,
"sync_coperson_status": null,
"sync_coperson_attributes": null,
"create_role": null,
"match_type": null,
"sync_affiliation": null,
"sync_identifier_type": null,
"sync_affiliation_type_id": "&mapAffiliationType",
"sync_identifier_type_id": "&mapIdentifierType",
"match_identifier_type_id": "&mapIdentifierType",
"match_email_address_type_id": "&mapEmailType",
"establish_clusters": null,
"co_enrollment_flow_id": null,
"co_pipeline_id": "pipeline_id"
}
},
"external_identity_sources": {
"source": "cm_org_identity_sources",
"displayField": "description",
"booleans": ["hash_source_record"],
"cache": ["co_id"],
"fieldMap": {
"plugin": "&mapExternalIdentitySourcePlugin",
"co_pipeline_id": "pipeline_id",
"sync_mode": null,
"sync_query_mismatch_mode": null,
"sync_query_skip_known": null,
"sync_on_user_login": null,
"eppn_identifier_type": null,
"eppn_suffix": null,
"org_identity_source_id": "external_identity_source_id"
}
},
"ext_identity_source_records": {
"source": "cm_org_identity_source_records",
"displayField": "id",
Expand All @@ -606,58 +695,5 @@
"reference_identifier": "reference_identifier",
"org_identity_source_record_id": "ext_identity_source_record_id"
}
},
"orcid_sources": {
"source": "cm_orcid_sources",
"displayField": "id",
"booleans": [
"scope_inherit"
],
"cache": ["external_identity_source_id", "server_id"],
"fieldMap": {
"org_identity_source_id": "external_identity_source_id",
"default_affiliation_type_id": "&mapToDefaultAffiliationTypeId",
"address_type_id": "&mapToDefaultAddressTypeId",
"email_address_type_id": "&mapToDefaultEmailAddressTypeId",
"name_type_id": "&mapToDefaultNameTypeId",
"telephone_number_type_id": "&mapToDefaultTelephoneNumberTypeId"
},
"addChangelog": true
},
"orcid_tokens": {
"source": "cm_orcid_tokens",
"displayField": "id",
"cache": ["orcid_identifier", "orcid_source_id"],
"addChangelog": true
},
"env_sources": {
"source": "cm_env_sources",
"displayField": "id",
"addChangelog": true,
"fieldMap": {
"org_identity_source_id": "external_identity_source_id",
"duplicate_mode": null,
"redirect_on_duplicate": "redirect_on_duplicate",
"sp_type": "sp_mode",
"default_affiliation_type_id": "&mapAffiliationType",
"default_affiliation": null,
"address_type_id": "&mapToDefaultAddressTypeId",
"email_address_type_id": "&mapToDefaultEmailAddressTypeId",
"name_type_id": "&mapToDefaultNameTypeId",
"telephone_number_type_id": "&mapToDefaultTelephoneNumberTypeId",
"env_o": "env_organization",
"env_ou": "env_department",
"env_identifier_eppn_login": null,
"env_identifier_eptid_login": null,
"env_identifier_epuid_login": null,
"env_identifier_oidcsub_login": null,
"env_identifier_orcid": null,
"env_identifier_orcid_login": null,
"env_identifier_samlpairwiseid_login": null,
"env_identifier_samlsubjectid_login": null,
"env_identifier_sorid": "env_identifier_sourcekey",
"env_identifier_sorid_login": null,
"env_identifier_network_login": null
}
}
}
Loading

0 comments on commit 21f29c3

Please sign in to comment.