diff --git a/app/composer.json b/app/composer.json index b5a21792a..d3872f6e9 100644 --- a/app/composer.json +++ b/app/composer.json @@ -47,7 +47,8 @@ "PipelineToolkit\\": "availableplugins/PipelineToolkit/src/", "SqlConnector\\": "availableplugins/SqlConnector/src/", "SshKeyAuthenticator\\": "plugins/SshKeyAuthenticator/src/", - "CoreJob\\": "plugins/CoreJob/src/" + "CoreJob\\": "plugins/CoreJob/src/", + "Transmogrify\\": "plugins/Transmogrify/src/" } }, "autoload-dev": { @@ -66,7 +67,8 @@ "PipelineToolkit\\Test\\": "availableplugins/PipelineToolkit/tests/", "SqlConnector\\Test\\": "availableplugins/SqlConnector/tests/", "SshKeyAuthenticator\\Test\\": "plugins/SshKeyAuthenticator/tests/", - "CoreJob\\Test\\": "plugins/CoreJob/tests/" + "CoreJob\\Test\\": "plugins/CoreJob/tests/", + "Transmogrify\\": "plugins/Transmogrify/src/" } }, "scripts": { diff --git a/app/plugins/Transmogrify/README.md b/app/plugins/Transmogrify/README.md new file mode 100644 index 000000000..2cceaee86 --- /dev/null +++ b/app/plugins/Transmogrify/README.md @@ -0,0 +1,222 @@ +# Transmogrify (COmanage Registry Plugin) + +Transmogrify is a command‑line migration tool bundled as a CakePHP plugin for COmanage Registry PE. It copies and transforms data from a legacy/source Registry schema (cm_… tables) into the PE target schema using a configurable mapping (config/schema/tables.json) plus code hooks for special cases. + +It is designed to be: +- Safe by default: target tables that already contain data are skipped and not overwritten. +- Transparent: shows a progress bar with in‑line warnings and errors. +- Configurable: table selection, custom tables.json path, and optional helper modes (info, schema, etc). + + +## Prerequisites +- Configure database connections in local/config/database.php + - Source DB (used by Transmogrify) and target DB must both be reachable. Transmogrify initializes two Doctrine DBAL connections internally. +- The default tables mapping file is at: + - app/plugins/Transmogrify/config/schema/tables.json +- Actions + - Finalize any pending Jobs. Jobs in Queued or Progress state will be skipped. + - Restore extended type defaults + - Run the health checks. + +## Command +Invoke from your app root: + +- bin/cake transmogrify [options] + +Cake’s standard flags also apply (eg: --verbose, --quiet, -h). + + +## Options +These options come directly from TransmogrifyCommand::buildOptionParser. + +- --tables-config PATH + - Path to the transmogrify tables JSON config (defaults to the plugin’s tables.json). + +- --dump-tables-config + - Print the effective tables configuration (after schema extension) and exit. + +- --table NAME (repeatable) + - Migrate only the specified table(s). Repeat --table to select multiple. + - If omitted, Transmogrify processes all known tables in the configured order. + +- --list-tables + - List available target tables from the transmogrify config and exit. + +- --info + - Print source/target database configuration and exit. + +- --info-json + - Output info in JSON (use with --info). + +- --info-ping + - Ping connections and include connectivity + server version details (use with --info or --info-schema). + +- --info-schema + - Print schema information and whether the database is empty (defaults to inspecting the target DB). + +- --info-schema-role ROLE + - When used with --info-schema, select which database to inspect: source or target (default: target). + +- --login-identifier-copy + - Enable helper logic to copy/set up login identifiers during migration (see your deployment’s identifier policy). Use together with --login-identifier-type to choose the identifier type. + +- --login-identifier-type TYPE + - Identifier type value to use for login identifiers when --login-identifier-copy is set. + +- --orgidentities-health + - Run Org Identities health check (eligibility/exclusion breakdown based on non-historical links and person existence) and print a transmogrification readiness report, then exit. + +- --groups-health + - Run Groups health check (AR‑Group‑9: invalid Standard group names) and print a transmogrification readiness report, then exit. + +- --groups-colon-replacement STRING + - Optional: replace ":" with STRING in Standard group names during migration (opt‑in). Use with care; the name "CO" remains invalid and will not be auto‑renamed. + + +## Typical usage + +- Migrate everything using the default mapping + - bin/cake transmogrify + +- Preview environment information + - bin/cake transmogrify --info + - bin/cake transmogrify --info --info-json + - bin/cake transmogrify --info --info-ping + +- Inspect schema state (target by default, or choose source) + - bin/cake transmogrify --info-schema + - bin/cake transmogrify --info-schema --info-schema-role source + +- List the tables Transmogrify knows how to process + - bin/cake transmogrify --list-tables + +- Dump the effective tables configuration + - bin/cake transmogrify --dump-tables-config + +- Migrate a subset of tables (in safe order) + - bin/cake transmogrify --table types --table people --table person_roles + +- Use a custom tables.json mapping + - bin/cake transmogrify --tables-config /path/to/your/tables.json + +- Migrate with login identifier help + - bin/cake transmogrify --table identifiers --login-identifier-copy --login-identifier-type eppn + +Hints: +- Combine Cake’s verbosity flags for extra diagnostics: add --verbose to see per‑row details emitted by some hooks; add --quiet to minimize output (progress still shows). + + +## Behavior notes +- Target table already has data? Transmogrify will skip that table and warn (no overwrite). +- Primary key sequences are aligned automatically to preserve/accept explicit IDs where possible. +- Ordering and foreign keys: + - The tool emits rows in a dependency‑friendly order and may retry deferred rows to satisfy FK dependencies. + - You’ll see warnings for rows skipped due to unresolved foreign keys or missing type mappings. +- Progress UI: + - A single‑line progress bar updates in place. + - Warnings and errors appear under the bar as they happen. + + +## Mapping and hooks +- Mapping is defined in config/schema/tables.json. + - You can specify per‑table field maps, boolean normalization, caches, and pre/post hooks. +- Some tables use custom SELECTs to ensure parent‑before‑child ordering (eg recursive chains). +- Type lookups depend on the ‘types’ cache; migrate ‘types’ early or include it in the same run if you migrate data that requires type IDs. + + +## Exit codes +- 0: Success +- Non‑zero: Error (check the emitted [ERROR] lines) + + +## Getting help +- bin/cake transmogrify -h +- Consult this plugin’s code for advanced behavior: + - src/Command/TransmogrifyCommand.php + - config/schema/tables.json + - src/Lib/Traits/* (type mapping, caching, row transformations, hooks) + +## Org Identities → External Identities: Readiness and Migration Behavior + +This plugin migrates Organizational Identities to the new model introduced in COmanage Registry v5: + +- OrgIdentity was split into ExternalIdentity and ExternalIdentityRole. +- ExternalIdentity directly relates to Person (a Person may have multiple External Identities, but each External Identity belongs to a single Person). +- The CoOrgIdentityLink crosswalk was eliminated and Organizational Identity Pooling was dropped. +- affiliation was replaced by affiliation_type_id. +- o was renamed to organization. +- ou was renamed to department. +- External Identities no longer have Primary Names. +- External Identities do not carry the login flag on Identifiers. As of v5.1.0, the Identifier Mapper Pipeline Plugin can be used to set this flag. + +Eligibility (reasoning) is based on “non‑historical” links in cm_co_org_identity_links (co_org_identity_link_id IS NULL): +- A) No non‑historical link: excluded (x) +- B) Has non‑historical link(s) but all co_person_id are NULL: excluded (x) +- C) Has at least one non‑historical link with a non‑NULL co_person_id: included (✓) + +Included (✓) Org Identities are migrated as: +- One ExternalIdentity linked directly to the Person (person_id from the link) +- One ExternalIdentityRole for role‑like attributes with v5 field changes applied: + - affiliation → affiliation_type_id (type‑mapped) + - o → organization + - ou → department +- External Identities do not have Primary Names; identifier “login” flags are not set by this migration (use the Identifier Mapper Pipeline if needed) + +### Recommended preflight: Org Identities Health command + +Run this before migrating to verify which Org Identities will be included vs excluded, using the same non‑historical link reasoning: + +```bash +bin/cake transmogrify --orgidentities-health +``` + + +You’ll see a fixed‑width table with Reason, Included/Excluded counts, and an Indicator (✓ included, x excluded). Reasons are: +- A) No non‑historical link (excluded) +- B) Has non‑historical link(s) but all co_person_id are NULL (excluded) +- C) Has at least one non‑historical link with a non‑NULL co_person_id (included) + +Totals summarize overall readiness. Use this report to address data conditions (eg, missing person links) so that important Org Identities are eligible for migration. + +### Recommended preflight: Groups Health command (Standard naming rule) + +Transmogrify enforces a naming rule for Standard groups: a Standard group is considered invalid if its name contains a colon (:) or equals “CO” (case‑insensitive, trimmed). Invalid Standard groups will not be migrated by default and require admin action (eg, rename) before proceeding. + +Run this health check to see how many groups are affected and where action is needed: + +```bash +bin/cake transmogrify --groups-health +``` + +You’ll see a fixed‑width table with Reason, Included/Excluded counts, and an Indicator (✓ valid/eligible, x invalid/action required). Reasons are: +- Invalid: Standard group name contains “:” (excluded) + - Standard groups (type S) whose name includes a colon are considered invalid by default and require renaming before migration. + +- Invalid: Standard group name equals “CO” (excluded) + - Standard groups (type S) named exactly “CO” (case‑insensitive, trimmed) are considered invalid and require renaming. + +- Valid: Does not violate the naming rule (included) + - All other groups: either not Standard type, or Standard whose name does not contain “:” and is not exactly “CO”. + +Totals summarize overall readiness: +- Invalid (total): total number of invalid Standard groups (sum of the invalid reasons). +- Valid (total): total number of groups eligible to migrate without renaming. +- Total Groups: grand total of groups evaluated (valid + invalid). + +Use this report to identify Standard groups that must be renamed to comply with the rule. After remediation, re‑run the health check and verify that Invalid (total) is 0 and all groups you intend to migrate appear under Valid. + +Optional remediation helper (opt‑in): colon replacement +- By default, Transmogrify does not change group names and will error on invalid Standard names. +- You can opt in to replace “:” in Standard group names with a safer character or string during migration. The special name “CO” remains invalid and is not auto‑renamed. + +Example (replace ":" with "-"), shorthand when passing a lone "-" is problematic: + +```bash +bin/cake transmogrify --groups-colon-replacement-dash +``` + +For every other character, use the full option: + +```bash +bin/cake transmogrify --groups-colon-replacement '@' +``` diff --git a/app/plugins/Transmogrify/config/plugin.json b/app/plugins/Transmogrify/config/plugin.json new file mode 100644 index 000000000..92d32ce1e --- /dev/null +++ b/app/plugins/Transmogrify/config/plugin.json @@ -0,0 +1,7 @@ +{ + "name": "Transmogrify", + "version": "1.0.0", + "description": "Data migration command to transmogrify data from a source database into the target schema.", + "types": {}, + "schema": null +} diff --git a/app/plugins/Transmogrify/config/schema/tables.json b/app/plugins/Transmogrify/config/schema/tables.json new file mode 100644 index 000000000..74a234bc0 --- /dev/null +++ b/app/plugins/Transmogrify/config/schema/tables.json @@ -0,0 +1,400 @@ +{ + "__COMMENT__": "Template for adding new table configurations to the Transmogrify mapping. Keys starting with double underscores (__) are ignored by processors and exist only for documentation.", + "__EXAMPLE_TABLE_TEMPLATE__": { + "__INSTRUCTIONS__": "Copy this object, rename the key to your logical table name (eg, 'my_items'), and adjust values. Do NOT leave this example enabled in production.", + "source": "cm_my_items", + "displayField": "name", + "addChangelog": true, + "booleans": ["is_active", "is_primary"], + "cache": [ + "co_id", + ["co_id", "status"], + ["co_id", "name", "value"] + ], + "sqlSelect": "myItemsSqlSelect", + "preTable": "beforeInsertMyItems", + "postTable": "afterInsertMyItems", + "preRow": "beforeInsertMyItemRow", + "postRow": "afterInsertMyItemRow", + "fieldMap": { + "id": null, + "co_id": "co_id", + "name": "name", + "description": "details", + "status": "status", + "value": "amount", + "created": "&mapNow", + "modified": "&mapNow", + "type_id": "&mapExtendedType", + "owner_identifier": "&mapIdentifierToPersonId", + "valid_from": "valid_from", + "valid_through": "valid_through" + }, + "__NOTES__": "Common keys: source (DB table), displayField (for printing), addChangelog (add created/modified history), booleans (type coercion), cache (key(s) to cache rows by), sqlSelect (custom SELECT provider function), pre/postTable and pre/postRow (hook function names), fieldMap (left=new table column, right=source column name or &function). Use &mapNow to inject current timestamp, &mapExtendedType to resolve cm_co_extended_types, and mapping helpers found in Transmogrify traits. Note: There is no generic config-driven enforcement for required/defaults/unique; enforce uniqueness via database constraints." + }, + "__NOTES__": "CONFIGURATION MIGRATIONS", + "cos": { + "source": "cm_cos", + "displayField": "name", + "addChangelog": true, + "cache": ["status"] + }, + "types": { + "source": "cm_co_extended_types", + "displayField": "display_name", + "postTable": "insertPronounTypes", + "fieldMap": { + "attribute": "&mapExtendedType", + "name": "value", + "created": "&mapNow", + "modified": "&mapNow" + }, + "cache": [["co_id", "attribute", "value"]] + }, + "co_settings": { + "source": "cm_co_settings", + "displayField": "co_id", + "addChangelog": true, + "booleans": [], + "postTable": "insertDefaultSettings", + "cache": ["co_id"], + "fieldMap": { + "global_search_limit": "search_global_limit", + "required_fields_addr": "required_fields_address", + "permitted_fields_telephone_number": "&populateCoSettingsPhone", + "enable_nsf_demo": null, + "disable_expiration": null, + "disable_ois_sync": null, + "group_validity_sync_window": null, + "garbage_collection_interval": null, + "enable_normalization": null, + "enable_empty_cou": null, + "invitation_validity": null, + "t_and_c_login_mode": null, + "sponsor_eligibility": null, + "sponsor_co_group_id": null, + "theme_stacking": null, + "default_co_pipeline_id": null, + "elect_strategy_primary_name": null, + "co_dashboard_id": null, + "co_theme_id": null, + "person_picker_email_type": null, + "person_picker_identifier_type": null, + "person_picker_display_types": null, + "group_create_admin_only": null, + "t_and_c_return_url_allowlist": null + } + }, + "api_users": { + "source": "cm_api_users", + "displayField": "username", + "booleans": ["privileged"], + "fieldMap": { + "password": "api_key" + } + }, + "cous": { + "source": "cm_cous", + "displayField": "name", + "sqlSelect": "couSqlSelect" + }, + "servers": { + "source": "cm_servers", + "displayField": "description", + "addChangelog": true, + "cache": ["co_id", "status"], + "fieldMap": { + "plugin": "&mapServerTypeToPlugin", + "server_type": null + } + }, + "http_servers": { + "source": "cm_http_servers", + "displayField": "serverurl", + "addChangelog": true, + "booleans": [ + "ssl_verify_host", + "ssl_verify_peer" + ], + "cache": ["server_id"], + "fieldMap": { + "serverurl": "url", + "ssl_verify_peer": "skip_ssl_verification", + "ssl_verify_host": null + } + }, + "oauth2_servers": { + "source": "cm_oauth2_servers", + "displayField": "serverurl", + "addChangelog": true, + "cache": ["server_id"], + "booleans": ["access_token_exp"], + "fieldMap": { + "serverurl": "url", + "proxy": null + } + }, + "sql_servers": { + "source": "cm_sql_servers", + "displayField": "hostname", + "cache": ["server_id"], + "addChangelog": true, + "fieldMap": { + "dbport": "port" + } + }, + "match_servers": { + "source": "cm_match_servers", + "displayField": "username", + "addChangelog": true, + "booleans": [ + "ssl_verify_peer", + "ssl_verify_host" + ], + "cache": ["server_id"], + "fieldMap": { + "serverurl": "url", + "auth_type": "?BA", + "ssl_verify_peer": "skip_ssl_verification", + "ssl_verify_host": null + } + }, + "match_server_attributes": { + "source": "cm_match_server_attributes", + "displayField": "attribute", + "addChangelog": true, + "booleans": [ + "required" + ], + "cache": ["match_server_id"], + "fieldMap": { + "type_id": "&mapMatchAttributeTypeId", + "type": null + } + }, + "__NOTES__": "DATA MIGRATIONS", + "authentication_events": { + "source": "cm_authentication_events", + "displayField": "authenticated_identifier" + }, + "people": { + "source": "cm_co_people", + "displayField": "id", + "cache": ["co_id"], + "fieldMap": { + "co_person_id": "person_id" + } + }, + "person_roles": { + "source": "cm_co_person_roles", + "sqlSelect": "roleSqlSelect", + "displayField": "id", + "cache": ["status", "person_id", "manager_person_id", "sponsor_person_id"], + "fieldMap": { + "co_person_id": "person_id", + "co_person_role_id": "person_role_id", + "affiliation_type_id": "&mapAffiliationType", + "affiliation": null, + "manager_co_person_id": "manager_person_id", + "sponsor_co_person_id": "sponsor_person_id", + "o": "organization", + "ou": "department", + "source_org_identity_id": null + } + }, + "external_identities": { + "source": "cm_org_identities", + "displayField": "id", + "cache": ["person_id"], + "sqlSelect": "orgidentitiesSqlSelect", + "postRow": "mapExternalIdentityToExternalIdentityRole", + "fieldMap": { + "co_id": null, + "person_id": "&mapOrgIdentitycoPersonId", + "org_identity_id": "external_identity_id", + "title": null, + "o": null, + "ou": null, + "affiliation": null, + "manager_identifier": null, + "sponsor_identifier": null, + "valid_from": null, + "valid_through": null + } + }, + "groups": { + "source": "cm_co_groups", + "displayField": "name", + "cache": ["co_id", "owners_group_id"], + "booleans": ["nesting_mode_all", "open"], + "preRow": "applyCheckGroupNameARRule", + "fieldMap": { + "auto": null, + "co_group_id": "group_id", + "group_type": "?S", + "introduction": null + }, + "postTable": "createOwnersGroups" + }, + "group_nestings": { + "source": "cm_co_group_nestings", + "displayField": "id", + "booleans": ["negate"], + "fieldMap": { + "co_group_id": "group_id", + "target_co_group_id": "target_group_id", + "co_group_nesting_id": "group_nesting_id" + } + }, + "group_members": { + "source": "cm_co_group_members", + "displayField": "id", + "booleans": ["member", "owner"], + "preRow": "reconcileGroupMembershipOwnership", + "fieldMap": { + "co_group_id": "group_id", + "co_person_id": "person_id", + "member": null, + "owner": null, + "co_group_nesting_id": "group_nesting_id", + "co_group_member_id": "group_member_id", + "source_org_identity_id": null + } + }, + "names": { + "source": "cm_names", + "displayField": "id", + "booleans": ["primary_name"], + "sqlSelect": "mveaSqlSelect", + "fieldMap": { + "type_id": "&mapNameType", + "co_person_id": "person_id", + "org_identity_id": "external_identity_id", + "type": null + } + }, + "email_addresses": { + "source": "cm_email_addresses", + "displayField": "id", + "booleans": ["verified"], + "sqlSelect": "mveaSqlSelect", + "fieldMap": { + "type_id": "&mapEmailType", + "co_person_id": "person_id", + "org_identity_id": "external_identity_id", + "co_department_id": null, + "organization_id": null, + "type": null + } + }, + "identifiers": { + "source": "cm_identifiers", + "displayField": "id", + "booleans": ["login"], + "sqlSelect": "mveaSqlSelect", + "preRow": "mapLoginIdentifiers", + "fieldMap": { + "type_id": "&mapIdentifierType", + "co_group_id": "group_id", + "co_person_id": "person_id", + "org_identity_id": "external_identity_id", + "co_department_id": null, + "co_provisioning_target_id": null, + "organization_id": null, + "type": null, + "language": null + } + }, + "urls": { + "source": "cm_urls", + "displayField": "id", + "sqlSelect": "mveaSqlSelect", + "fieldMap": { + "type_id": "&mapUrlType", + "co_person_id": "person_id", + "org_identity_id": "external_identity_id", + "co_department_id": null, + "organization_id": null, + "type": null, + "language": null + } + }, + "addresses": { + "source": "cm_addresses", + "displayField": "id", + "sqlSelect": "mveaSqlSelect", + "fieldMap": { + "type_id": "&mapAddressType", + "co_person_role_id": "person_role_id", + "org_identity_id": "external_identity_id", + "co_department_id": null, + "organization_id": null, + "type": null + } + }, + "telephone_numbers": { + "source": "cm_telephone_numbers", + "displayField": "id", + "sqlSelect": "mveaSqlSelect", + "fieldMap": { + "type_id": "&mapTelephoneType", + "co_person_role_id": "person_role_id", + "org_identity_id": "external_identity_id", + "co_department_id": null, + "organization_id": null, + "type": null + } + }, + "ad_hoc_attributes": { + "source": "cm_ad_hoc_attributes", + "displayField": "id", + "sqlSelect": "mveaSqlSelect", + "fieldMap": { + "co_person_role_id": "person_role_id", + "org_identity_id": "external_identity_id", + "co_department_id": null, + "organization_id": null + }, + "postTable": "migrateExtendedAttributesToAdHocAttributes" + }, + "history_records": { + "source": "cm_history_records", + "displayField": "id", + "sqlSelect": "historyRecordsSqlSelect", + "fieldMap": { + "actor_co_person_id": "actor_person_id", + "co_person_id": "person_id", + "co_person_role_id": "person_role_id", + "co_group_id": "group_id", + "org_identity_id": "external_identity_id", + "co_email_list_id": null, + "co_service_id": null + } + }, + "jobs": { + "source": "cm_co_jobs", + "displayField": "id", + "fieldMap": { + "job_type": "plugin", + "job_mode": null, + "queue_time": "register_time", + "complete_time": "finish_time", + "job_type_fk": null, + "job_params": "parameters", + "requeued_from_co_job_id": "requeued_from_job_id", + "max_retry": null, + "max_retry_count": null + }, + "preRow": "validateJobIsTransmogrifiable" + }, + "job_history_records": { + "source": "cm_co_job_history_records", + "displayField": "id", + "sqlSelect": "jobHistoryRecordsSqlSelect", + "fieldMap": { + "co_job_id": "job_id", + "co_person_id": "person_id", + "org_identity_id": "external_identity_id" + } + } +} diff --git a/app/plugins/Transmogrify/src/Command/TransmogrifyCommand.php b/app/plugins/Transmogrify/src/Command/TransmogrifyCommand.php new file mode 100644 index 000000000..73a325a1e --- /dev/null +++ b/app/plugins/Transmogrify/src/Command/TransmogrifyCommand.php @@ -0,0 +1,724 @@ +pluginRoot = dirname(__DIR__, 2); + parent::__construct(); + } + + /** + * Override run command + * + * @param array $argv + * @param ConsoleIo $io + * @return int + */ + public function run(array $argv, ConsoleIo $io): int + { + $this->inconn = DBALConnection::factory($io, 'transmogrify'); + $this->outconn = DBALConnection::factory($io); + return parent::run($argv, $io); + } + + /** + * Build an Option Parser. + * + * @since COmanage Registry v5.0.0 + * @param ConsoleOptionParser $parser ConsoleOptionParser + * @return ConsoleOptionParser ConsoleOptionParser + */ + + protected function buildOptionParser(ConsoleOptionParser $parser): ConsoleOptionParser + { + // Allow overriding the tables config path + $parser->addOption('tables-config', [ + 'help' => 'Path to transmogrify tables JSON config', + 'default' => TransmogrifyEnum::TABLES_JSON_PATH + ]); + $parser->addOption('dump-tables-config', [ + 'help' => 'Output the effective tables configuration (after schema extension) and exit', + 'boolean' => true + ]); + // Specify a table (or repeat option) to migrate only a subset + $parser->addOption('table', [ + 'help' => 'Migrate only the specified table. Repeat the option to migrate multiple tables', + 'multiple' => true + ]); + // List available target tables and exit + $parser->addOption('list-tables', [ + 'help' => 'List available target tables from the transmogrify config and exit', + 'boolean' => true + ]); + // Info options integrated into TransmogrifyCommand + $parser->addOption('info', [ + 'help' => 'Print source and target database configuration and exit', + 'boolean' => true + ]); + $parser->addOption('info-json', [ + 'help' => 'Output info in JSON (use with --info)', + 'boolean' => true + ]); + $parser->addOption('info-ping', [ + 'help' => 'Ping connections and include connectivity + server version (use with --info or --info-schema)', + 'boolean' => true + ]); + $parser->addOption('info-schema', [ + 'help' => 'Print schema information and whether the database is empty (defaults to target). Use --info-schema-role to select source/target', + 'boolean' => true + ]); + $parser->addOption('info-schema-role', [ + 'help' => 'When using --info-schema, which database to inspect: source or target (default: target)' + ]); + $parser->addOption('login-identifier-copy', [ + 'help' => __d('command', 'tm.login-identifier-copy'), + 'boolean' => true + ]); + + $parser->addOption('login-identifier-type', [ + 'help' => __d('command', 'tm.login-identifier-type') + ]); + + // Health report option (Org Identities readiness) + $parser->addOption('orgidentities-health', [ + 'help' => 'Run Org Identities health check (eligibility/exclusion breakdown) and exit', + 'boolean' => true + ]); + // Health report option (Groups naming rule readiness) + $parser->addOption('groups-health', [ + 'help' => 'Run Groups health check (AR-Group-9: invalid Standard names) and exit', + 'boolean' => true + ]); + // Optional: replace colons in Standard group names during migration (opt-in, off by default) + $parser->addOption('groups-colon-replacement', [ + 'help' => 'If set, replace ":" with this value in Standard group names during migration. WARNING: name "CO" remains invalid and is not auto-renamed.' + ]); + // Convenience flag for using a literal dash as replacement + $parser->addOption('groups-colon-replacement-dash', [ + 'help' => 'Use "-" as the replacement for ":" in Standard group names (shorthand when passing a lone "-" is problematic)', + 'boolean' => true + ]); + + $parser->setEpilog(__d('command', 'tm.epilog')); + + return $parser; + } + + /** + * Execute the Transmogrify Command. + * + * @param Arguments $args Command Arguments + * @param ConsoleIo $io Console IO + * @throws Exception + * @since COmanage Registry v5.0.0 + */ + + public function execute(Arguments $args, ConsoleIo $io): int + { + $this->args = $args; + $this->io = $io; + + // Now that BaseCommand set verbosity, construct the printer so it can detect it correctly + $this->cmdPrinter = new CommandLinePrinter($io, 'green', 50, true); + + // Validate "info" option combinations and handle errors + $code = $this->validateInfoOptions($io); + if ($code !== null) { + return $code; + } + + // Handle info modes early (no tables config needed unless ping) + $code = $this->maybeHandleInfo(); + if ($code !== null) { + return $code; + } + + // Health report: run and exit + if ($this->args->getOption('orgidentities-health')) { + OrgIdentitiesHealth::run($this->inconn, $this->io); + return BaseCommand::CODE_SUCCESS; + } + if ($this->args->getOption('groups-health')) { + GroupsHealth::run($this->inconn, $this->io); + return BaseCommand::CODE_SUCCESS; + } + + // Load tables configuration (from JSON) and extend it with schema data + $this->loadTablesConfig(); + + // Dump tables config if requested + $code = $this->maybeDumpTablesConfig($io); + if ($code !== null) { + return $code; + } + + // List tables and exit + $code = $this->maybeListTables($io); + if ($code !== null) { + return $code; + } + + // Build list of tables to migrate from --table option and positional args + $selected = $this->buildSelectedTables($args); + + // Validate and warn for subset selection + $code = $this->maybeValidateSelectedTables($selected, $io); + if ($code !== null) { + return $code; + } + + // Register the current version for future upgrade purposes + $this->metaTable = TableRegistry::getTableLocator()->get('Meta'); + $this->metaTable->setUpgradeVersion(); + + // Track remaining selected tables (if any) so we can exit early when done + $pendingSelected = []; + if (!empty($selected)) { + $pendingSelected = array_fill_keys($selected, true); + } + + foreach(array_keys($this->tables) as $t) { + // Initializations per table migration + $outboundTableEmpty = true; + $skipTableTransmogrification = false; + $inboundQualifiedTableName = $this->inconn->qualifyTableName($this->tables[$t]['source']); + $outboundQualifiedTableName = $this->outconn->qualifyTableName($t); + $Model = TableRegistry::getTableLocator()->get($t); + + $this->cmdPrinter->out(message: sprintf("Transmogrifying table: %s(%s)", Inflector::classify($t), $t)); + + + /* + * Run checks before processing the table + **/ + + // Check if source table exists and warn if not present + if(!empty($this->tables[$t]['source'])) { + $src = $this->tables[$t]['source']; + if (!$this->tableExists($src)) { + $this->io->warning("Source table '$src' does not exist in source database, skipping table '$t'"); + continue; + } + } + + // Skip a table if already contains data + if($Model->find()->count() > 0) { + $outboundTableEmpty = false; + $this->cmdPrinter->warning("Table (" . $t . ") is not empty. We will not overwrite existing data."); + } + + // Skip tables not in the selected subset if specified + // We will print a warning, and we will parse all the tables because we need to construct the cache. + // Nevertheless, we will not allow any database processing + if ( + !empty($selected) + && !in_array($t, $selected) + ) { + $skipTableTransmogrification = true; + $this->cmdPrinter->warning("Skipping Transmogrification. Table ($t) is not in the selected subset."); + } + + // Mark the table as skipped if it is not empty and not selected + $this->cache['skipInsert'][$outboundQualifiedTableName] = !$outboundTableEmpty || $skipTableTransmogrification; + + /* + * End of checks + */ + + + // Configure sequence ID for the target table + if(!RawSqlQueries::setSequenceId( + $this->inconn, + $this->outconn, + $this->tables[$t]['source'], + $t, + $this->cmdPrinter + )) { + $this->cmdPrinter->warning("Skipping Transmogrification. Can not properly configure the Sequence for the primary key for the Table (\"$t\""); + return BaseCommand::CODE_ERROR; + } + + // Execute any pre-processing hooks for the current table + $this->runPreTableHook($t); + + // Step 8: Build and execute query to fetch all source records + $insql = match(true) { + !empty($this->tables[$t]['sqlSelect']) => $this->runSqlSelectHook($t, $inboundQualifiedTableName), + default => RawSqlQueries::buildSelectAllOrderedById($inboundQualifiedTableName) + }; + + // Verbose message to show the SQL query being executed + $this->cmdPrinter->verbose(sprintf('[Inbound SQL] Table=%s | %s', $t, $insql)); + // Fetch the inbound data. + $stmt = $this->inconn->executeQuery($insql); + + /* + * PROGRESS STARTING + **/ + // If a custom SELECT is used, count the exact result set; otherwise count the whole table + if (!empty($this->tables[$t]['sqlSelect'])) { + $countSql = RawSqlQueries::buildCountFromSelect($insql); + $count = (int)$this->inconn->fetchOne($countSql); + } else { + $count = (int)$this->inconn->fetchOne(RawSqlQueries::buildCountAll($inboundQualifiedTableName)); + } + $this->cmdPrinter->start($count); + $tally = 0; + $this->cache['error'] = 0; + $this->cache['warns'] = 0; + + while ($row = $stmt->fetchAssociative()) { + if(!empty($row[ $this->tables[$t]['displayField'] ])) { + $this->cmdPrinter->verbose("$t " . $row[ $this->tables[$t]['displayField'] ]); + } + + try { + // Create a copy of the original row data to preserve it for post-processing + $origRow = $row; + + // Execute any pre-processing hooks to transform or validate the row data + $this->runPreRowHook($t, $origRow, $row); + + // Set changelog defaults (created/modified timestamps, user IDs) + // Must be done before boolean normalization as it adds new fields + $this->populateChangelogDefaults( + $t, + $row, + isset($this->tables[$t]['addChangelog']) && $this->tables[$t]['addChangelog'] + ); + + // Convert boolean values to database-compatible format + $this->normalizeBooleanFieldsForDb($t, $row); + + // Map old field names to new schema field names + $this->mapLegacyFieldNames($t, $row); + + // Insert the transformed row into the target database + if($this->cache['skipInsert'][$outboundQualifiedTableName] === false) { + // Check if a parent record for this row was previously rejected; if so, skip this insert + if ($this->skipIfRejectedParent(currentTable: $t, row: $row)) { + continue; + } + + $this->outconn->insert($outboundQualifiedTableName, $row); + // Execute any post-processing hooks after successful insertion + $this->runPostRowHook($t, $origRow, $row); + } + + // Store row data in cache for potential later use + $this->cacheResults($t, $row); + } catch(ForeignKeyConstraintViolationException $e) { + // A foreign key associated with this record did not load, so we can't + // load this record. This can happen, eg, because the source_field_id + // did not load, perhaps because it was associated with an Org Identity + // not linked to a CO Person that was not migrated. + $this->cache['warns'] += 1; + $this->cache['rejected'][$outboundQualifiedTableName][$row['id']] = $row; + $this->cmdPrinter->warning("Skipping $t record " . $row['id'] . " due to invalid foreign key: " . $e->getMessage()); + $this->cmdPrinter->pause(); + } + catch(\InvalidArgumentException $e) { + // If we can't find a value for mapping we skip the record + // (ie: mapLegacyFieldNames basically requires a successful mapping) + $this->cache['warns'] += 1; + $this->cache['rejected'][$outboundQualifiedTableName][$row['id']] = $row; + $this->cmdPrinter->warning("Skipping $t record " . $row['id'] . ": " . $e->getMessage()); + $this->cmdPrinter->pause(); + } + catch(\Exception $e) { + $this->cache['error'] += 1; + $this->cmdPrinter->error("$t record " . $row['id'] . ": " . $e->getMessage()); + $this->cmdPrinter->pause(); + } + + $tally++; + // Always delegate progress updates to the printer; it will decide what to draw + $this->cmdPrinter->update($tally); + } + + $this->cmdPrinter->finish(); + /** + * FINISH PROGRESS + */ + + // Output final warning and error counts for the table + $this->cmdPrinter->warning(sprintf('Warnings: %d', $this->cache['warns'])); + $this->cmdPrinter->error(sprintf('Errors: %d', $this->cache['error'])); + + // Execute any post-processing hooks for the table + if($this->cache['skipInsert'][$outboundQualifiedTableName] === false) { + $this->cmdPrinter->out('Running post-table hook for ' . $t); + $this->runPostTableHook($t); + } + + // If user selected a subset, exit as soon as all selected tables are processed + if (!empty($pendingSelected) && isset($pendingSelected[$t])) { + unset($pendingSelected[$t]); + if (empty($pendingSelected)) { + $this->cmdPrinter->out('All selected tables have been processed. Exiting.'); + return BaseCommand::CODE_SUCCESS; + } + } + + // Prompt for confirmation before processing table + $tables = array_keys($this->tables); + $currentIndex = array_search($t, $tables); + if (isset($tables[$currentIndex + 1])) { + $this->cmdPrinter->info("Next table to process: " . $tables[$currentIndex + 1]); + } else { + $this->cmdPrinter->out(PHP_EOL. "Table import complete. Exiting."); + } + + $this->cmdPrinter->pause(); + } + + return BaseCommand::CODE_SUCCESS; + } + + + /** + * Validate incompatible/invalid "info" related options. + * Returns an exit code when invalid, or null to continue. + * + * @param ConsoleIo $io Console IO object for output + * @return int|null Command exit code or null to continue + * @since COmanage Registry v5.2.0 + */ + private function validateInfoOptions(ConsoleIo $io): ?int + { + if ( + $this->args->getOption('info-ping') + && !$this->args->getOption('info') + && !$this->args->getOption('info-schema') + ) { + $io->err('Option --info-ping must be used together with --info or --info-schema.'); + $io->err('Examples:'); + $io->err(' bin/cake transmogrify --info --info-ping'); + $io->err(' bin/cake transmogrify --info-schema --info-ping [--info-schema-role source|target]'); + return BaseCommand::CODE_ERROR; + } + + if ( + $this->args->getOption('info-schema-role') !== null + && !$this->args->getOption('info-schema') + ) { + $io->err('Option --info-schema-role must be used together with --info-schema.'); + $io->err('Examples:'); + $io->err(' bin/cake transmogrify --info-schema --info-schema-role target'); + $io->err(' bin/cake transmogrify --info-schema --info-ping --info-schema-role source'); + $io->err(' bin/cake transmogrify --info-schema --info-json --info-schema-role source'); + return BaseCommand::CODE_ERROR; + } + + return null; + } + + /** + * Handle --info / --info-schema early-exit modes. + * Returns exit code if handled, or null to continue normal execution. + * + * @return int|null Command exit code if handled, null to continue execution + * @since COmanage Registry v5.2.0 + */ + private function maybeHandleInfo(): ?int + { + if ($this->args->getOption('info')) { + (DbInfoPrinter::initialize($this->io, $this->dbInfoService))->print( + (bool)$this->args->getOption('info-json'), + (bool)$this->args->getOption('info-ping') + ); + return BaseCommand::CODE_SUCCESS; + } + + if ($this->args->getOption('info-schema')) { + $role = $this->args->getOption('info-schema-role') ?: 'target'; + if (!in_array($role, ['source', 'target'], true)) { $role = 'target'; } + (DbInfoPrinter::initialize($this->io, $this->dbInfoService))->print( + (bool)$this->args->getOption('info-json'), + (bool)$this->args->getOption('info-ping'), + true, + $role + ); + return BaseCommand::CODE_SUCCESS; + } + + return null; + } + + /** + * Load tables configuration from JSON and attach to $this->tables. + * + * @return void + * @since COmanage Registry v5.2.0 + */ + private function loadTablesConfig(): void + { + $path = $this->args->getOption('tables-config') ?? Transmogrify::TABLES_JSON_PATH; + if (!str_starts_with($path, $this->pluginRoot . DS)) { + $path = $this->pluginRoot . DS . $path; + } + $this->tables = $this->configLoader->load($path); + } + + /** + * If requested, dump effective tables configuration and exit. + * + * @param ConsoleIo $io Console IO object for output + * @return int|null Command exit code if dumped, null to continue + * @since COmanage Registry v5.2.0 + */ + private function maybeDumpTablesConfig(ConsoleIo $io): ?int + { + if ($this->args->getOption('dump-tables-config')) { + $io->out(json_encode($this->tables, JSON_PRETTY_PRINT)); + return BaseCommand::CODE_SUCCESS; + } + return null; + } + + /** + * If requested, list available tables and exit. + * + * @param ConsoleIo $io Console IO object for output + * @return int|null Command exit code if listed, null to continue + * @since COmanage Registry v5.2.0 + */ + private function maybeListTables(ConsoleIo $io): ?int + { + if ($this->args->getOption('list-tables')) { + $io->out(implode("\n", array_keys($this->tables))); + return BaseCommand::CODE_SUCCESS; + } + return null; + } + + /** + * Build list of tables from --table options and positional args. + * + * @param Arguments $args Command arguments + * @return array List of selected table names + * @since COmanage Registry v5.2.0 + */ + private function buildSelectedTables(Arguments $args): array + { + $selected = $args->getArrayOption('table') ?? []; + $positional = $args->getArguments(); + if (!empty($positional)) { + $selected = array_merge($selected, $positional); + } + return array_values(array_unique($selected)); + } + + /** + * Validate selected tables against config and warn about partial migration. + * Returns exit code on error, or null if OK. + * + * @param array $selected List of selected table names + * @param ConsoleIo $io Console IO object for output + * @return int|null Command exit code on error, null if valid + * @since COmanage Registry v5.2.0 + */ + private function maybeValidateSelectedTables(array $selected, ConsoleIo $io): ?int + { + if (!empty($selected)) { + $unknown = array_diff($selected, array_keys($this->tables)); + if (!empty($unknown)) { + $io->err('Unknown table(s): ' . implode(', ', $unknown)); + $io->err('Use --list-tables to see available options.'); + return BaseCommand::CODE_ERROR; + } + $io->warning('Migrating a subset of tables may lead to foreign key or type mapping warnings if dependencies are not loaded (eg, types, people, groups).'); + $io->out('Selected tables: ' . implode(', ', $selected)); + } + return null; + } + + /** + * Check if a table exists in the source database + * + * @param string $tableName Name of table to check + * @return bool True if table exists + * @throws \Exception + */ + protected function tableExists(string $tableName): bool + { + $dbSchemaManager = $this->inconn->createSchemaManager(); + $tableList = $dbSchemaManager->listTableNames(); + return in_array($tableName, $tableList); + } + + /** + * Check whether this row references a rejected record and, if so, mark it rejected and warn. + * This preserves the original self-reference check and adds a generic parent-table check. + * + * Self-reference (original semantics): + * if (cache['rejected'][qualifiedCurrentTable][row[singular(currentTable)_id]]) then skip + * + * Cross-table parent: + * find a *_id in the row that corresponds to a known target table (eg, job_id -> jobs), + * then if (cache['rejected'][qualifiedParentTable][row[parent_fk]]) skip. + * + * @param string $currentTable Logical target table name (eg, 'job_history_records') + * @param array $row Row to insert + * @return bool True if the row should be skipped, false otherwise + */ + private function skipIfRejectedParent(string $currentTable, array $row): bool + { + if (!isset($this->cache['rejected'])) { + return false; + } + + // Compute qualified table names once + $qualifiedCurrent = $this->outconn->qualifyTableName($currentTable); + + // 1) Self-reference check (preserves the original semantics) + // With the original code, fkOutboundQualifiedTableName was derived from the table, + // effectively matching "_id". + $selfFk = StringUtilities::classNameToForeignKey($currentTable); + if ( + isset($row[$selfFk]) && + !empty($this->cache['rejected'][$qualifiedCurrent][$row[$selfFk]]) + ) { + $childId = $row['id'] ?? null; + if ($childId !== null) { + $this->cache['rejected'][$qualifiedCurrent][$childId] = $row; + } + $this->cmdPrinter->warning(sprintf( + 'Skipping record %d in table %s - parent %s(%d) was rejected (self-reference)', + (int)($childId ?? 0), + $currentTable, + $currentTable, + (int)$row[$selfFk] + )); + return true; + } + + // 2) Cross-table parents: check ALL candidate *_id columns + foreach ($row as $col => $val) { + if ($val === null) { continue; } + if (!is_string($col) || !str_ends_with($col, '_id')) { continue; } + if ($col === 'id' || str_ends_with($col, '_type_id')) { continue; } + + $base = substr($col, 0, -3); + $candidate = Inflector::pluralize(Inflector::underscore($base)); + + // Skip self-table here (already handled) + if ($candidate === $currentTable) { continue; } + + // Only check known target tables + if (!isset($this->tables[$candidate])) { continue; } + + $qualifiedParent = $this->outconn->qualifyTableName($candidate); + $parentId = $val; + + if (!empty($this->cache['rejected'][$qualifiedParent][$parentId])) { + $childId = $row['id'] ?? null; + if ($childId !== null) { + $this->cache['rejected'][$qualifiedCurrent][$childId] = $row; + } + $this->cmdPrinter->warning(sprintf( + 'Skipping record %d in table %s - parent %s(%d) was rejected', + (int)($childId ?? 0), + $currentTable, + $candidate, + (int)$parentId + )); + return true; + } + } + + return false; + } +} \ No newline at end of file diff --git a/app/plugins/Transmogrify/src/Command/TransmogrifySourceToTargetCommand.php b/app/plugins/Transmogrify/src/Command/TransmogrifySourceToTargetCommand.php new file mode 100644 index 000000000..c054687f3 --- /dev/null +++ b/app/plugins/Transmogrify/src/Command/TransmogrifySourceToTargetCommand.php @@ -0,0 +1,771 @@ +setDescription('Print a JSON template shaped like __EXAMPLE_TABLE_TEMPLATE__ for mapping a source table to a target table. Target config is used only to read the example template; no values are copied. Paths must be absolute.'); + + // Switch from positional arguments to named options (long and short) + $parser->addOption('source-path', [ + 'short' => 's', + 'help' => 'Absolute path to the SOURCE configuration file (JSON or XML).', + ]); + $parser->addOption('target-path', [ + 'short' => 't', + 'help' => 'Absolute path to the TARGET configuration file (JSON or XML).', + ]); + $parser->addOption('source-table', [ + 'short' => 'S', + 'help' => 'Table key/name in the source configuration to map from.', + ]); + $parser->addOption('target-table', [ + 'short' => 'T', + 'help' => 'Table key/name in the target configuration to map to.', + ]); + $parser->addOption('source-prefix', [ + 'short' => 'p', + 'help' => 'Prefix to prepend to the source table in the output (eg, "cm_"). Default: empty string.', + ]); + + // Association tree printer option + $parser->addOption('assoc-tree', [ + 'short' => 'A', + 'help' => 'Print an ASCII tree of associations starting from the given --source-table based on the source XML. Use with -s and -S. Ignores other options and exits.', + 'boolean' => true, + ]); + + $epilog = []; + $epilog[] = 'Examples:'; + $epilog[] = ' bin/cake transmogrify_source_to_target \\\n --source-path /path/to/registry/app/Config/Schema/schema.xml \\\n --target-path /path/to/registry/app/config/transmogrifytables.json \\\n --source-table cm_co_people \\\n --target-table people'; + $epilog[] = ''; + $epilog[] = ' bin/cake transmogrify_source_to_target -s /abs/source.xml -t /abs/target.json -S cousins -T cous -p cm_'; + $epilog[] = ''; + $epilog[] = 'Notes:'; + $epilog[] = ' - Paths must be absolute.'; + $epilog[] = ' - Files may be JSON (.json) or XML (.xml).'; + $epilog[] = ' - The output is a single JSON object similar to __EXAMPLE_TABLE_TEMPLATE__ in tables.json.'; + $epilog[] = ' - Use --source-prefix/-p to control any prefix (like cm_) applied to the source table name in the output.'; + $parser->setEpilog(implode(PHP_EOL, $epilog)); + + return parent::buildOptionParser($parser); + } + + /** + * Execute the command + * + * @param Arguments $args Command arguments + * @param ConsoleIo $io Console IO instance + * @return int Exit code + */ + public function execute(Arguments $args, ConsoleIo $io): int + { + // capture prefix for helper methods + $this->currentPrefix = (string)($args->getOption('source-prefix') ?? ''); + $sourcePath = (string)($args->getOption('source-path') ?? ''); + $targetPath = (string)($args->getOption('target-path') ?? ''); + $sourceTable = (string)($args->getOption('source-table') ?? ''); + $targetTable = (string)($args->getOption('target-table') ?? ''); + $sourcePrefix = (string)($args->getOption('source-prefix') ?? ''); + $assocTree = (bool)($args->getOption('assoc-tree') ?? false); + + // If only association tree requested, we only need source path and source table + if ($assocTree) { + $missing = []; + if ($sourcePath === '') { $missing[] = '--source-path (-s)'; } + if ($sourceTable === '') { $missing[] = '--source-table (-S)'; } + if (!empty($missing)) { + $io->err('Missing required option(s) for --assoc-tree: ' . implode(', ', $missing)); + $io->err('Run with --help to see usage.'); + return BaseCommand::CODE_ERROR; + } + if ($sourcePath === '' || !str_starts_with($sourcePath, DIRECTORY_SEPARATOR)) { + $io->err('SOURCE path must be absolute: ' . $sourcePath); + return BaseCommand::CODE_ERROR; + } + if (!is_readable($sourcePath)) { + $io->err('SOURCE file not readable: ' . $sourcePath); + return BaseCommand::CODE_ERROR; + } + + try { + $srcCfgRaw = $this->configLoaderService->loadGeneric($sourcePath); + } catch (\Throwable $e) { + $io->err('Failed to load source configuration: ' . $e->getMessage()); + return BaseCommand::CODE_ERROR; + } + $this->printAssociationTreeFromSchema($srcCfgRaw, $sourceTable, $io); + return BaseCommand::CODE_SUCCESS; + } + + // Validate required named options are present + $missing = []; + if ($sourcePath === '') { $missing[] = '--source-path (-s)'; } + if ($targetPath === '') { $missing[] = '--target-path (-t)'; } + if ($sourceTable === '') { $missing[] = '--source-table (-S)'; } + if ($targetTable === '') { $missing[] = '--target-table (-T)'; } + if (!empty($missing)) { + $io->err('Missing required option(s): ' . implode(', ', $missing)); + $io->err('Run with --help to see usage.'); + return BaseCommand::CODE_ERROR; + } + + // Validate absolute paths + foreach (['source' => $sourcePath, 'target' => $targetPath] as $label => $path) { + if ($path === '' || !str_starts_with($path, DIRECTORY_SEPARATOR)) { + $io->err(strtoupper($label) . ' path must be absolute: ' . $path); + return BaseCommand::CODE_ERROR; + } + if (!is_readable($path)) { + $io->err(strtoupper($label) . ' file not readable: ' . $path); + return BaseCommand::CODE_ERROR; + } + } + + try { + // Load target config to read __EXAMPLE_TABLE_TEMPLATE__ shape + $tgtCfg = $this->configLoaderService->loadGeneric($targetPath); + // Load source config to extract old schema (eg, from XML) + $srcCfgRaw = $this->configLoaderService->loadGeneric($sourcePath); + } catch (\Throwable $e) { + $io->err('Failed to load configuration: ' . $e->getMessage()); + return BaseCommand::CODE_ERROR; + } + + // Load the migration config (tables.json) to derive generic co_* FK renames + $tablesJsonPath = dirname(__DIR__, 2) . DIRECTORY_SEPARATOR . 'config' . DIRECTORY_SEPARATOR . 'schema' . DIRECTORY_SEPARATOR . 'tables.json'; + $tablesCfg = []; + if (is_readable($tablesJsonPath)) { + try { + $tablesCfg = $this->configLoaderService->load($tablesJsonPath); + } catch (\Throwable $e) { + $io->warning('Could not load migration config tables.json: ' . $e->getMessage()); + } + } + + // Normalize possible schema.xml structures into transmogrify-like configs for source + $srcCfg = $this->normalizeConfig(is_array($srcCfgRaw) ? $srcCfgRaw : []); + + // Extract example template (shape) from target config without filtering out __ keys first + $example = is_array($tgtCfg) && array_key_exists('__EXAMPLE_TABLE_TEMPLATE__', $tgtCfg) + ? (array)$tgtCfg['__EXAMPLE_TABLE_TEMPLATE__'] + : []; + + // Determine the output keys based on the example (ignoring any __* documentation keys) + $exampleKeys = array_filter(array_keys($example), static function($k) { + return !is_string($k) || !str_starts_with($k, '__'); + }); + + // Determine source fields from old schema configuration (if available) + $sourceFields = []; + [$srcKey, $srcTableCfg] = $this->findTableConfig($srcCfg, $sourceTable); + if (is_array($srcTableCfg)) { + if (!empty($srcTableCfg['fieldMap']) && is_array($srcTableCfg['fieldMap'])) { + $sourceFields = array_keys($srcTableCfg['fieldMap']); + } elseif (!empty($srcTableCfg['fields']) && is_array($srcTableCfg['fields'])) { + $sourceFields = array_values($srcTableCfg['fields']); + } + } + $sourceFieldSet = array_flip($sourceFields); + + // Build legacy FK pattern map (eg, *_co_person_id -> *_person_id, *_co_message_template_id -> *_message_template_id) + $legacyFkPatterns = $this->buildLegacyFkPatterns($tablesCfg, (string)$sourcePrefix); + + // Inspect default database to find target table columns + try { + $conn = DBALConnection::factory($io); + $columns = []; + $candidate = $targetTable; + $alts = [ $candidate ]; + // toggle provided prefix to try alternative table naming + $prefix = (string)($sourcePrefix ?? ''); + if ($prefix !== '') { + $alts[] = str_starts_with($candidate, $prefix) ? substr($candidate, strlen($prefix)) : ($prefix . $candidate); + } + $foundTable = null; + foreach (array_unique($alts) as $tbl) { + if ($conn->isMySQL()) { + $db = $conn->fetchOne('SELECT DATABASE()'); + $cols = $conn->fetchAllAssociative('SELECT column_name, data_type, column_type FROM information_schema.columns WHERE table_schema = ? AND table_name = ? ORDER BY ordinal_position', [$db, $tbl]); + if (!empty($cols)) { $columns = $cols; $foundTable = $tbl; break; } + } else { + // PostgreSQL: search in current schema(s) + $cols = $conn->fetchAllAssociative("SELECT column_name, data_type FROM information_schema.columns WHERE table_name = ? ORDER BY ordinal_position", [$tbl]); + if (!empty($cols)) { $columns = $cols; $foundTable = $tbl; break; } + } + } + if ($foundTable === null) { + $io->err('Target table not found in default database: ' . $targetTable); + return BaseCommand::CODE_ERROR; + } + + // Build target column name lists + $targetCols = array_map(static fn($c) => (string)$c['column_name'], $columns); + $targetColSetAll = array_flip($targetCols); + + $singular = \Cake\Utility\Inflector::singularize($targetTable); + $metaCols = [ + $singular . '_id', + 'revision', + 'deleted', + 'actor_identifier', + 'created', + 'modified', + ]; + + $targetColSet = $targetColSetAll; + foreach ($metaCols as $mc) { + unset($targetColSet[$mc]); + } + + // Determine if target natively has changelog fields (controls addChangelog) + $targetHasChangelog = ( + array_key_exists($singular . '_id', $targetColSetAll) + && array_key_exists('revision', $targetColSetAll) + && array_key_exists('deleted', $targetColSetAll) + && array_key_exists('actor_identifier', $targetColSetAll) + ); + + // Determine if source has changelog (v4 pattern: revision, deleted, actor_identifier, and legacy self-FK) + $legacySelfFkExact = 'co_' . $singular . '_id'; + $legacySelfFkSuffix = '_co_' . $singular . '_id'; + $fieldsLower = array_map('strtolower', $sourceFields); + $sourceFieldSet = array_flip($fieldsLower); + $sourceHasChangelog = ( + isset($sourceFieldSet['revision']) + && isset($sourceFieldSet['deleted']) + && isset($sourceFieldSet['actor_identifier']) + && ( + in_array($legacySelfFkExact, $fieldsLower, true) + || (bool)array_filter($fieldsLower, fn($s) => str_ends_with($s, $legacySelfFkSuffix)) + ) + ); + + // Detect booleans by database type (for non-metadata only) + $booleanCols = []; + foreach ($columns as $col) { + $colName = (string)$col['column_name']; + if (in_array($colName, $metaCols, true)) { continue; } + if ($conn->isMySQL()) { + $dt = strtolower((string)($col['data_type'] ?? '')); + $ct = strtolower((string)($col['column_type'] ?? '')); + $isBool = ($dt === 'tinyint' && str_starts_with($ct, 'tinyint(1)')) || ($dt === 'bit' && str_starts_with($ct, 'bit(1)')) || ($dt === 'boolean'); + if ($isBool) { $booleanCols[] = $colName; } + } else { + $dt = strtolower((string)($col['data_type'] ?? '')); + if ($dt === 'boolean' || $dt === 'bool') { $booleanCols[] = $colName; } + } + } + + // Build fieldMap as source-left => target-right (include null when unknown) + $fieldMap = []; + $mappedTargets = []; + $sourceMeta = ['revision','deleted','actor_identifier','created','modified']; + foreach ($sourceFields as $scol) { + if (in_array($scol, $sourceMeta, true)) { continue; } + + // 1) exact same-name target (non-metadata set) + $tcol = array_key_exists($scol, $targetColSet) ? $scol : null; + + // 2) normalization-based mapping (generic co_* fk rename via patterns + specific one-offs) + if ($tcol === null) { + $candidate = $this->normalizeTargetNameForSourceColumn($scol, $legacyFkPatterns); + if ($candidate !== null) { + $isMetaSelfFk = ($candidate === $singular . '_id'); + // accept candidate if it's a normal column, or it's the allowed self-FK + if (array_key_exists($candidate, $targetColSet) + || ($isMetaSelfFk && array_key_exists($candidate, $targetColSetAll))) { + $tcol = $candidate; + } + } + } + + $fieldMap[$scol] = $tcol; + if ($tcol !== null) { $mappedTargets[$tcol] = true; } + } + + // Collect target-only columns (user can decide later). + $targetUnmapped = []; + foreach (array_keys($targetColSetAll) as $tcol) { + // skip global metadata from the hint list + if (in_array($tcol, $metaCols, true)) { continue; } + if (!isset($mappedTargets[$tcol])) { + $targetUnmapped[] = $tcol; + } + } + } catch (\Throwable $e) { + $io->err('Failed to inspect database schema: ' . $e->getMessage()); + return BaseCommand::CODE_ERROR; + } + + + // Build a template strictly using the example shape. The target config is only used for the example; no values are copied. + $defaults = [ + 'source' => $sourcePrefix . $sourceTable, + 'displayField' => $this->determineDisplayField($sourceFields), + // Note: we will attach/omit addChangelog below based on source/target + //'addChangelog' => '??', + 'booleans' => $booleanCols, + 'cache' => [], + 'sqlSelect' => null, + 'preTable' => null, + 'postTable' => null, + 'preRow' => null, + 'postRow' => null, + 'fieldMap' => $fieldMap ?: new \stdClass(), + ]; + + // If example keys were found, limit output to those keys (plus ensure 'source' exists). + if (!empty($exampleKeys)) { + $template = []; + foreach ($exampleKeys as $k) { + if ($k === 'source') { + $template[$k] = $sourcePrefix . $sourceTable; + continue; + } + $template[$k] = $defaults[$k] ?? null; + } + foreach (['source','fieldMap'] as $must) { + if (!array_key_exists($must, $template)) { + $template[$must] = $defaults[$must]; + } + } + } else { + $template = $defaults; + } + + // Decide addChangelog per rules: + // - both have changelog: omit configuration + // - target has changelog (source doesn't): true + // - target doesn't have changelog (source does): false + $addChangelogOpt = null; + if ($targetHasChangelog && $sourceHasChangelog) { + $addChangelogOpt = null; // omit + } elseif ($targetHasChangelog && !$sourceHasChangelog) { + $addChangelogOpt = true; + } elseif (!$targetHasChangelog && $sourceHasChangelog) { + $addChangelogOpt = false; + } else { + $addChangelogOpt = null; // neither has: omit + } + + // Apply addChangelog decision + if (array_key_exists('addChangelog', $template)) { + unset($template['addChangelog']); + } + if ($addChangelogOpt !== null) { + $template['addChangelog'] = $addChangelogOpt; + } + + + // Attach target-only hints for the user + if (!empty($targetUnmapped)) { + $template['targetUnmapped'] = $targetUnmapped; + } + + // Wrap result under the target-table key for direct insertion into tables.json + $wrapped = [ $targetTable => $template ]; + + // Ensure JSON encoding preserves empty objects properly for fieldMap + $json = json_encode($wrapped, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES); + if ($json === false) { + $io->err('Failed to encode output JSON'); + return BaseCommand::CODE_ERROR; + } + + $io->out($json); + return BaseCommand::CODE_SUCCESS; + } + + /** + * Filter out documentation keys (starting with __) from configuration array + * + * @param array $cfg Configuration array to filter + * @return array Filtered configuration without documentation keys + */ + private function filterDocKeys(array $cfg): array + { + return array_filter($cfg, static function ($value, $key) { + return !(is_string($key) && str_starts_with($key, '__')); + }, ARRAY_FILTER_USE_BOTH); + } + + /** + * Decide if addChangelog should be true based on source configuration fields. + * Requirements: revision, deleted, actor_identifier, and self-referencing FK (eg, cous -> cou_id). + * + * @param string $tableName Name/key of the source table (may be prefixed like cm_) + * @param array $sourceFields List of source column names + */ + private function shouldAddChangelog(string $tableName, array $sourceFields): bool + { + if (empty($sourceFields)) { + return false; + } + $fields = array_map('strtolower', $sourceFields); + $fieldSet = array_flip($fields); + + // Required fixed columns + foreach (['revision','deleted','actor_identifier'] as $req) { + if (!array_key_exists($req, $fieldSet)) { + return false; + } + } + // Determine expected self FK + $name = strtolower($tableName); + $prefix = (string)($this->currentPrefix ?? ''); + if ($prefix !== '' && str_starts_with($name, $prefix)) { + $name = substr($name, strlen($prefix)); + } + // If table name is schema-qualified (eg, public.table), take part after dot + $dot = strrpos($name, '.'); + if ($dot !== false) { + $name = substr($name, $dot + 1); + } + $base = $name; + if (str_ends_with($base, 's')) { + $base = substr($base, 0, -1); + } + $selfFk = $base . '_id'; + + return array_key_exists($selfFk, $fieldSet); + } + + /** + * Normalize a loaded configuration array. + * - If it's a schema-like array (has 'table' nodes), convert to a transmogrify-like map + * keyed by table name with at least 'source' and 'fieldMap'. + */ + private function normalizeConfig(array $cfg): array + { + // Detect schema root + $tablesNode = null; + if (array_key_exists('table', $cfg)) { + $tablesNode = $cfg['table']; + } elseif (isset($cfg['schema']) && is_array($cfg['schema']) && array_key_exists('table', $cfg['schema'])) { + $tablesNode = $cfg['schema']['table']; + } + if ($tablesNode === null) { + // Already config-like + return $cfg; + } + // Ensure list + if (!is_array($tablesNode) || (is_array($tablesNode) && array_keys($tablesNode) !== range(0, count($tablesNode)-1))) { + // Single table object -> wrap + $tables = [$tablesNode]; + } else { + $tables = $tablesNode; + } + $out = []; + foreach ($tables as $t) { + if (!is_array($t)) { continue; } + // Name is typically under '@attributes' => ['name' => ...] + $name = null; + if (isset($t['@attributes']['name'])) { + $name = (string)$t['@attributes']['name']; + } elseif (isset($t['name'])) { + // Fallback if converter placed it differently + $name = is_array($t['name']) && isset($t['name']['@attributes']) ? (string)($t['name']['@attributes']['value'] ?? '') : (string)$t['name']; + } + if ($name === null || $name === '') { continue; } + + // Build simple 1:1 field map using elements, if present + $fieldMap = new \stdClass(); + if (isset($t['field'])) { + $fieldsNode = $t['field']; + // Normalize to list + if (!is_array($fieldsNode) || (is_array($fieldsNode) && array_keys($fieldsNode) !== range(0, count($fieldsNode)-1))) { + $fields = [$fieldsNode]; + } else { + $fields = $fieldsNode; + } + $map = []; + foreach ($fields as $f) { + if (!is_array($f)) { continue; } + $fname = $f['@attributes']['name'] ?? null; + if ($fname) { + $map[$fname] = $fname; + } + } + if (!empty($map)) { + $fieldMap = $map; + } + } + + $out[$name] = [ + 'source' => $name, + 'displayField' => $this->determineDisplayField(is_array($fieldMap) ? array_keys($fieldMap) : []), + 'fieldMap' => $fieldMap + ]; + } + return $out; + } + + /** + * Locate a table configuration by key or by matching the 'source' attribute. + * Also tries matching names with/without a leading provided prefix. + * + * @param array $cfg + * @param string $nameOrSource Either the config key or the source table name + * @return array{0:string|null,1:array|null} [key, config] + */ + private function findTableConfig(array $cfg, string $nameOrSource): array + { + if (array_key_exists($nameOrSource, $cfg) && is_array($cfg[$nameOrSource])) { + return [$nameOrSource, $cfg[$nameOrSource]]; + } + $prefix = (string)($this->currentPrefix ?? ''); + $alt = $nameOrSource; + if ($prefix !== '') { + $alt = str_starts_with($nameOrSource, $prefix) ? substr($nameOrSource, strlen($prefix)) : ($prefix . $nameOrSource); + } + if (array_key_exists($alt, $cfg) && is_array($cfg[$alt])) { + return [$alt, $cfg[$alt]]; + } + foreach ($cfg as $key => $val) { + if (!is_array($val)) { continue; } + $src = $val['source'] ?? null; + if ($src === $nameOrSource || $src === $alt) { + return [$key, $val]; + } + // Also allow key matching to bare table name + if ($key === $nameOrSource || $key === $alt) { + return [$key, $val]; + } + } + return [null, null]; + } + + /** + * Determine displayField from source table fields using provided order: + * name, display_name, username, authenticated_identifier, description, id. + * Returns null if none are present. + */ + private function determineDisplayField(array $sourceFields): ?string + { + if (empty($sourceFields)) { + return null; + } + $fields = array_map('strtolower', $sourceFields); + $set = array_flip($fields); + foreach (['name','display_name','username','authenticated_identifier','description','id'] as $candidate) { + if (array_key_exists($candidate, $set)) { + return $candidate; + } + } + return null; + } + + /** + * Print an ASCII association tree for a given starting table using the parsed XML schema array. + * Supports common schema.xml structures with elements containing @attributes["foreignTable"] + * and nested elements. + */ + private function printAssociationTreeFromSchema(array $schemaArr, string $startTable, ConsoleIo $io): void + { + // Build associative array tree from schema.xml-like array + // We expect tables under ['schema']['table'] or directly under ['table'] + $tablesNode = $schemaArr['schema']['table'] ?? ($schemaArr['table'] ?? []); + // Normalize to list of tables + if (!is_array($tablesNode) || (is_array($tablesNode) && array_keys($tablesNode) !== range(0, count($tablesNode)-1))) { + $tables = [$tablesNode]; + } else { + $tables = $tablesNode; + } + // Build map: tableName => [ children tables ... ] based on column @attributes['constraint'] value 'REFERENCE table(column)' + $children = []; + foreach ($tables as $t) { + if (!is_array($t)) { continue; } + $tbl = $t['@attributes']['name'] ?? ($t['name'] ?? null); + if ($tbl === null) { continue; } + $tbl = (string)$tbl; + if (!isset($children[$tbl])) { $children[$tbl] = []; } + // inspect columns/field definitions; schema.xml may use or + $colsNode = $t['column'] ?? ($t['field'] ?? []); + $cols = []; + if ($colsNode !== [] && $colsNode !== null) { + if (!is_array($colsNode) || (is_array($colsNode) && array_keys($colsNode) !== range(0, count($colsNode)-1))) { + $cols = [$colsNode]; + } else { + $cols = $colsNode; + } + } + foreach ($cols as $idx => $col) { + if (!is_array($col)) { continue; } + $constraint = $col['constraint'] ?? null; + if (!is_string($constraint)) { continue; } + // Expect format: 'REFERENCE table(column)' + if (preg_match('/^REFERENCES\s+([A-Za-z0-9_\.]+)\s*\(([^)]+)\)/', $constraint, $m)) { + $refTable = $m[1]; + // Remove prefix if it was provided and matches + if ($this->currentPrefix !== '' && str_starts_with($refTable, $this->currentPrefix)) { + $refTable = substr($refTable, strlen($this->currentPrefix)); + } + + // current table depends on refTable -> edge from current to parent + // For tree of ancestors starting from $startTable, we build parent map + // but for associative array tree representation, we'll build nested children where parent has child + // i.e., parent -> [ child1, child2 ] based on references found in child + if (!isset($children[$tbl])) { $children[$tbl] = []; } + if (!in_array($refTable, $children[$tbl], true)) { + $children[$tbl][] = $refTable; + } + } + } + } + // Build recursive associative array tree starting at $startTable + $visited = []; + $buildTree = function(string $node) use (&$buildTree, &$children, &$visited) { + if (isset($visited[$node])) { + // cycle: denote specially + return ['__cycle__' => $node]; + } + $visited[$node] = true; + $kids = $children[$node] ?? []; + $tree = []; + foreach ($kids as $k) { + $tree[$k] = $buildTree($k); + } + return $tree; + }; + $tree = [ $startTable => $buildTree($startTable) ]; + // Output the array via Console IO as JSON for readability + $io->out(json_encode($tree, JSON_PRETTY_PRINT)); + } + + /** + * Build legacy co_* foreign key rename patterns based on migration config. + * Returns an array of [suffixFrom => suffixTo], eg: + * '_co_person_id' => '_person_id' + * '_co_group_id' => '_group_id' + * '_co_message_template_id' => '_message_template_id' + * + * The patterns are applied as “ends-with” replacements to source column names. + */ + private function buildLegacyFkPatterns(array $tablesCfg, string $sourcePrefix): array + { + if (!is_array($tablesCfg) || empty($tablesCfg)) { + return [ + '_co_person_id' => '_person_id', + '_co_group_id' => '_group_id', + // Also support exact-form defaults for standalone columns + 'co_person_id' => 'person_id', + 'co_group_id' => 'group_id', + ]; + } + + // Filter out documentation keys and reduce to table entries that have a 'source' + $entries = []; + foreach ($tablesCfg as $k => $v) { + if (!is_array($v)) { continue; } + if (is_string($k) && str_starts_with($k, '__')) { continue; } + if (!isset($v['source']) || !is_string($v['source'])) { continue; } + $entries[$k] = $v['source']; + } + + $patterns = [ + // suffix form (covers prefix + co_* cases) + '_co_person_id' => '_person_id', + '_co_group_id' => '_group_id', + // exact form (covers exact co_* columns with no left prefix) + 'co_person_id' => 'person_id', + 'co_group_id' => 'group_id', + ]; + + foreach ($entries as $targetKey => $src) { + $bareSource = $src; + if ($sourcePrefix !== '' && str_starts_with($bareSource, $sourcePrefix)) { + $bareSource = substr($bareSource, strlen($sourcePrefix)); + } + + // If bare source starts with "co_", derive both forms for this target + // targetKey 'message_templates' -> singular 'message_template' + if (str_starts_with($bareSource, 'co_')) { + $singular = \Cake\Utility\Inflector::singularize($targetKey); + + // suffix form: ..._co__id -> ...__id + $fromSuffix = '_co_' . $singular . '_id'; + $toSuffix = '_' . $singular . '_id'; + $patterns[$fromSuffix] = $toSuffix; + + // exact form: co__id -> _id + $fromExact = 'co_' . $singular . '_id'; + $toExact = $singular . '_id'; + $patterns[$fromExact] = $toExact; + } + } + + return $patterns; + } + + + /** + * Normalize a v4 source column name to a likely v5 target column name based on: + * - generic co_* foreign key patterns derived from migration config + * - specific one-offs + */ + private function normalizeTargetNameForSourceColumn(string $sourceCol, array $legacyFkPatterns): ?string + { + // Try exact-form first (eg, 'co_message_template_id' -> 'message_template_id') + if (isset($legacyFkPatterns[$sourceCol])) { + return $legacyFkPatterns[$sourceCol]; + } + + // Then apply suffix-based rewrites (eg, 'approver_co_group_id' -> 'approver_group_id') + foreach ($legacyFkPatterns as $from => $to) { + if ($from === $sourceCol) { continue; } // already handled exact + // only treat entries starting with '_' as suffix rules + if (str_starts_with($from, '_') && str_ends_with($sourceCol, $from)) { + $prefix = substr($sourceCol, 0, -strlen($from)); + return $prefix . $to; + } + } + + // Specific one-offs + if ($sourceCol === 'source_url') { + return 'source'; + } + if ($sourceCol === 'email_body') { + return 'email_body_text'; + } + + return null; + } +} diff --git a/app/plugins/Transmogrify/src/Controller/AppController.php b/app/plugins/Transmogrify/src/Controller/AppController.php new file mode 100644 index 000000000..dabc4c59b --- /dev/null +++ b/app/plugins/Transmogrify/src/Controller/AppController.php @@ -0,0 +1,10 @@ +tables[$table]['cache'])) { + return; + } + + // Cache the requested fields. For now, at least, we key on row ID only. + foreach($this->tables[$table]['cache'] as $field) { + if(is_array($field)) { + // This is a list of fields, create a composite key that point to the row ID + + $label = ""; + $key = ""; + + foreach($field as $subfield) { + // eg: co_id+attribute+value+ + $label .= $subfield . "+"; + + // eg: 2+Identifier.type+eppn+ + $key .= $row[$subfield] . "+"; + } + + $this->cache[$table][$label][$key] = $row['id']; + } else { + // If the row has the field then map id to the requested field. + if(array_key_exists($field, $row)) { + $this->cache[$table]['id'][ $row['id'] ][$field] = $row[$field]; + } + } + } + } + + /** + * Find CO ID from related record data + * + * @param array $row Row data containing person_id, external_identity_id, group_id etc + * @return int Mapped CO ID + * @throws \InvalidArgumentException When CO not found + * @since COmanage Registry v5.0.0 + */ + protected function findCoId(array $row): int + { + // By the time we're called, we should have transmogrified the Org Identity + // and CO Person data, so we can just walk the caches + + if (isset($row['co_id'])) { + return (int)$row['co_id']; + } + + // Choose the resolution path by precedence using match(true) + $coId = match (true) { + isset($row['person_id']) => $this->getCoIdFromPersonId((int)$row['person_id']), + + // Map External Identity -> Person -> CO + isset($row['external_identity_id']) => $this->getCoIdFromExternalIdentityId((int)$row['external_identity_id']), + + isset($row['group_id']) => $this->getCoIdFromGroupId((int)$row['group_id']), + + isset($row['match_server_id']) => $this->getCoIdFromMatchServer((int)$row['match_server_id']), + + // Legacy/preRow: org_identity_id follows the same External Identity path + isset($row['org_identity_id']) => $this->getCoIdFromExternalIdentityId((int)$row['org_identity_id']), + + isset($row['co_person_id']) => $this->getCoIdFromPersonId((int)$row['co_person_id']), + + isset($row['co_group_id']) => $this->getCoIdFromGroupId((int)$row['co_group_id']), + + isset($row['co_person_role_id']) => $this->getCoIdFromPersonRoleId((int)$row['co_person_role_id']), + + default => null, + }; + + if ($coId !== null) { + return $coId; + } + + // For the multiple value attributes we are going to get a lot of misses + // because we only move one org identity and only the latest one. No revisions. + // Which means that all the values that belong to deleted org identities are going + // to be misses. + throw new \InvalidArgumentException("CO not found for record"); + } + + /** + * Resolve a CO ID from a CO Person ID via cache. + * + * @param int $personId + * @return int|null + */ + private function getCoIdFromPersonId(int $personId): ?int + { + if (isset($this->cache['people']['id'][$personId]['co_id'])) { + return (int)$this->cache['people']['id'][$personId]['co_id']; + } + return null; + } + + /** + * Resolve a CO Person ID from an External Identity (or legacy OrgIdentity) ID via cache. + * + * @param int $externalIdentityId + * @return int|null + */ + private function getPersonIdFromExternalIdentity(int $externalIdentityId): ?int + { + $personId = $this->cache['external_identities']['id'][$externalIdentityId]['person_id'] ?? null; + return $personId !== null ? (int)$personId : null; + } + + /** + * Resolve a CO ID from a Group ID via cache. + * + * @param int $groupId + * @return int|null + */ + private function getCoIdFromGroupId(int $groupId): ?int + { + if (isset($this->cache['groups']['id'][$groupId]['co_id'])) { + return (int)$this->cache['groups']['id'][$groupId]['co_id']; + } + return null; + } + + /** + * Resolve a CO ID starting from an External Identity (or legacy OrgIdentity) ID via cache. + * + * @param int $externalIdentityId + * @return int|null + */ + private function getCoIdFromExternalIdentityId(int $externalIdentityId): ?int + { + $personId = $this->getPersonIdFromExternalIdentity($externalIdentityId); + return $personId !== null ? $this->getCoIdFromPersonId($personId) : null; + } + + /** + * Resolve a CO ID from a Match Server ID via cache. + * + * @param int $matchServerId Match Server ID to resolve + * @return int|null CO ID if found, null otherwise + * @since COmanage Registry v5.2.0 + */ + private function getCoIdFromMatchServer(int $matchServerId): ?int + { + if (isset($this->cache['match_servers']['id'][$matchServerId]['server_id'])) { + $serverId = (int)$this->cache['match_servers']['id'][$matchServerId]['server_id']; + return $this->cache['servers']['id'][$serverId]['co_id'] ?? null; + } + return null; + } + + /** + * Resolve a CO ID from a Person Role ID via cache. + * + * @param int $personRoleId Person Role ID to resolve + * @return int|null CO ID if found, null otherwise + * @since COmanage Registry v5.2.0 + */ + private function getCoIdFromPersonRoleId(int $personRoleId): ?int + { + if (isset($this->cache['person_roles']['id'][$personRoleId]['person_id'])) { + $personId = (int)$this->cache['person_roles']['id'][$personRoleId]['person_id']; + return $this->getCoIdFromPersonId($personId); + } + return null; + } +} \ No newline at end of file diff --git a/app/plugins/Transmogrify/src/Lib/Traits/HookRunnersTrait.php b/app/plugins/Transmogrify/src/Lib/Traits/HookRunnersTrait.php new file mode 100644 index 000000000..8ddf6c05c --- /dev/null +++ b/app/plugins/Transmogrify/src/Lib/Traits/HookRunnersTrait.php @@ -0,0 +1,157 @@ +tables[$table]['preTable'])) { return; } + $method = $this->tables[$table]['preTable']; + if(!method_exists($this, $method)) { + throw new \RuntimeException("Unknown preTable hook: $method"); + } + + $this->cmdPrinter->verbose('Running pre-table hook: ' . ucfirst(preg_replace('/([A-Z])/', ' $1', $method))); + $this->{$method}(); + } + + /** + * Run post-table hook if configured for given table + * + * @param string $table Table name + * @return void + * @throws \RuntimeException If hook method doesn't exist + * @since COmanage Registry v5.2.0 + */ + private function runPostTableHook(string $table): void { + if(empty($this->tables[$table]['postTable'])) { return; } + $method = $this->tables[$table]['postTable']; + if(!method_exists($this, $method)) { + throw new \RuntimeException("Unknown postTable hook: $method"); + } + $this->cmdPrinter->verbose('Running post-table hook: ' . ucfirst(preg_replace('/([A-Z])/', ' $1', $method))); + $this->{$method}(); + } + + /** + * Run pre-row hook if configured for given table + * + * @param string $table Table name + * @param array $origRow Original row data by reference + * @param array $row Current row data by reference + * @return void + * @throws \RuntimeException If hook method doesn't exist + * @since COmanage Registry v5.2.0 + */ + private function runPreRowHook(string $table, array &$origRow, array &$row): void { + if(empty($this->tables[$table]['preRow'])) { return; } + $method = $this->tables[$table]['preRow']; + if(!method_exists($this, $method)) { + throw new \RuntimeException("Unknown preRow hook: $method"); + } + $this->cmdPrinter->verbose('Running pre-row hook: ' . ucfirst(preg_replace('/([A-Z])/', ' $1', $method))); + $this->{$method}($origRow, $row); + } + + /** + * Run post-row hook if configured for given table + * + * @param string $table Table name + * @param array $origRow Original row data by reference + * @param array $row Current row data by reference + * @return void + * @throws \RuntimeException If hook method doesn't exist + * @since COmanage Registry v5.2.0 + */ + private function runPostRowHook(string $table, array &$origRow, array &$row): void { + if(empty($this->tables[$table]['postRow'])) { return; } + $method = $this->tables[$table]['postRow']; + if(!method_exists($this, $method)) { + throw new \RuntimeException("Unknown postRow hook: $method"); + } + $this->cmdPrinter->verbose('Running post-row hook: ' . ucfirst(preg_replace('/([A-Z])/', ' $1', $method))); + $this->{$method}($origRow, $row); + } + + /** + * Run SQL select hook if configured for given table + * + * @param string $table Table name + * @param string $qualifiedTableName Fully qualified table name + * @return string SQL select statement + * @throws \RuntimeException If hook method doesn't exist + * @since COmanage Registry v5.2.0 + */ + private function runSqlSelectHook(string $table, string $qualifiedTableName): string { + $method = $this->tables[$table]['sqlSelect'] ?? ''; + if($method === '') { + return RawSqlQueries::buildSelectAllOrderedById($qualifiedTableName); + } + if(!method_exists(RawSqlQueries::class, $method)) { + throw new \RuntimeException("Unknown sqlSelect hook: $method"); + } + $this->cmdPrinter->verbose('Running SQL select hook: ' . ucfirst(preg_replace('/([A-Z])/', ' $1', $method))); + + // Special handling for MVEA-style selects: derive FK columns from fieldMap keys dynamically + if ($method === 'mveaSqlSelect') { + // Known FK candidates we care about for MVEA presence checks + $fkCandidates = [ + 'co_person_id', + 'co_person_role_id', + 'org_identity_id', + 'co_group_id', + 'co_department_id', + 'co_provisioning_target_id', + 'organization_id' + ]; + $fieldMap = $this->tables[$table]['fieldMap'] ?? []; + $presentFks = array_values(array_intersect($fkCandidates, array_keys($fieldMap))); + + // Fallback to names-like FKs if nothing matched (defensive) + if (empty($presentFks)) { + $presentFks = ['co_person_id', 'org_identity_id']; + } + + return RawSqlQueries::mveaSqlSelect( + $qualifiedTableName, + $this->inconn->isMySQL(), + $presentFks + ); + } + + return RawSqlQueries::{$method}($qualifiedTableName, $this->inconn->isMySQL()); + } +} \ No newline at end of file diff --git a/app/plugins/Transmogrify/src/Lib/Traits/ManageDefaultsTrait.php b/app/plugins/Transmogrify/src/Lib/Traits/ManageDefaultsTrait.php new file mode 100644 index 000000000..d1f4a8ec9 --- /dev/null +++ b/app/plugins/Transmogrify/src/Lib/Traits/ManageDefaultsTrait.php @@ -0,0 +1,142 @@ +get('Groups'); + + $iterator = new PaginatedSqlIterator($Groups, []); + + foreach($iterator as $k => $group) { + try { + // Because PaginatedSqlIterator will pick up new Groups as we create them, + // we need to check for any Owners groups (that we just created) and skip them. + if(!$group->isOwners()) { + $ownersGid = $Groups->createOwnersGroup($group); + + // We need to manually populate the cache + $this->cache['groups']['id'][$group->id]['owners_group_id'] = $ownersGid; + } + } + catch(\Exception $e) { + $this->cache['error'] += 1; + $this->cmdPrinter->error("Failed to create owners group for " + . $group->name . " (" . $group->id . "): " + . $e->getMessage()); + } + } + } + + /** + * Insert default CO Settings for COs that don't have settings. + * + * @since COmanage Registry v5.0.0 + * @return void + */ + protected function insertDefaultSettings(): void + { + // Create a CoSetting for any CO that didn't previously have one. + + $createdSettings = []; + $createdCos = array_keys($this->cache['cos']['id']); + + foreach($this->cache['co_settings']['id'] as $co_setting_id => $cached) { + $createdSettings[] = $cached['co_id']; + } + + $emptySettings = array_values(array_diff($createdCos, $createdSettings)); + + if(!empty($emptySettings)) { + $CoSettings = TableRegistry::getTableLocator()->get('CoSettings'); + + foreach($emptySettings as $coId) { + // Insert a default row into CoSettings for this CO ID + try { + $CoSettings->addDefaults($coId); + } catch (\ConflictException $e) { + // skip + } + } + } + } + + /** + * Insert default Pronoun types for all COs. + * + * @since COmanage Registry v5.0.0 + * @return void + */ + protected function insertPronounTypes(): void + { + // Since the Pronoun MVEA didn't exist in v4, we'll need to create the + // default types for all COs. + + $Types = TableRegistry::getTableLocator()->get('Types'); + + foreach(array_keys($this->cache['cos']['id']) as $coId) { + $Types->addDefault($coId, 'Pronouns.type'); + } + } + + /** + * Set a default value for CO Settings Permitted Telephone Number Fields. + * Returns CANE as the default permitted telephone number field value. + * + * @param array $row Row of table data + * @return string Default value CANE + * @since COmanage Registry v5.0.0 + */ + + protected function populateCoSettingsPhone(array $row): string + { + return PermittedTelephoneNumberFieldsEnum::CANE; + } +} diff --git a/app/plugins/Transmogrify/src/Lib/Traits/RowTransformationTrait.php b/app/plugins/Transmogrify/src/Lib/Traits/RowTransformationTrait.php new file mode 100644 index 000000000..dd7d1194c --- /dev/null +++ b/app/plugins/Transmogrify/src/Lib/Traits/RowTransformationTrait.php @@ -0,0 +1,469 @@ +args?->getOption('groups-colon-replacement') ?? ''); + if ($replacement === '' && ($this->args?->getOption('groups-colon-replacement-dash') === true)) { + $replacement = '-'; + } + if ($replacement !== '') { + $newName = str_replace(':', $replacement, $name); + if ($newName === '') { + // Guard against accidental empty name + throw new \InvalidArgumentException('Replacing ":" produced an empty Standard CoGroup name; adjust --groups-colon-replacement'); + } + $row['name'] = $newName; + $this->cmdPrinter?->verbose(sprintf('Replaced ":" in Standard CoGroup name "%s" -> "%s"', $name, $newName)); + } else { + // Default: error out (no auto-replacement) + throw new \InvalidArgumentException('Standard CoGroup names cannot contain a colon by default: ' . $name); + } + } + } + } + + + /** + * Check if a group membership is actually asserted, and reassign ownerships. + * + * @since COmanage Registry v5.0.0 + * @param array $origRow Row of table data (original data) + * @param array $row Row of table data (post fixes) + * @throws InvalidArgumentException + */ + protected function reconcileGroupMembershipOwnership(array $origRow, array &$row): void + { + // We need to handle the various member+owner scenarios, but basically + // (1) If 'owner' is set, manually create a Group Membership in the appropriate + // Owners Group (we need to be called via preRow to do this) + // (2) If 'member' is NOT set, throw an exception so we don't create + // in invalid membership + // (3) Otherwise just return so the Membership gets created + + $tableName = 'group_members'; + if($origRow['owner'] && !$origRow['deleted'] && !$origRow['co_group_member_id']) { + // Create a membership in the appropriate owners group, but not + // on changelog entries + + if(!empty($this->cache['groups']['id'][ $origRow['co_group_id'] ]['owners_group_id'])) { + $ownerRow = [ + 'group_id' => $this->cache['groups']['id'][ $origRow['co_group_id'] ]['owners_group_id'], + 'person_id' => $origRow['co_person_id'], + 'created' => $origRow['created'], + 'modified' => $origRow['modified'], + 'group_member_id' => null, + 'revision' => 0, + 'deleted' => 'f', + 'actor_identifier' => $origRow['actor_identifier'] + ]; + + $qualifiedTableName = $this->outconn->qualifyTableName($tableName); + $this->outconn->insert($qualifiedTableName, $ownerRow); + } else { + $this->cmdPrinter->error("Could not find owners group for CoGroupMember " . $origRow['id']); + $this->cache['error'] += 1; + } + } + + if(!$row['member'] && !$row['owner']) { + // we are caching the rejected rows so we can reject the rows that reference them as well. + $this->cache['rejected'][$tableName][$row['id']] = $row; + throw new \InvalidArgumentException('member not set on GroupMember'); + } + } + + /** + * Filter Jobs. + * + * @since COmanage Registry v5.0.0 + * @param array $origRow Row of table data (original data) + * @param array $row Row of table data (post fixes) + * @throws InvalidArgumentException + */ + protected function validateJobIsTransmogrifiable(array $origRow, array &$row): void + { + // We don't update any of the attributes, but for rows with unsupported data + // we throw an exception so they don't transmogrify. + + if($row['status'] == 'GO' || $row['status'] == 'Q') { + throw new \InvalidArgumentException("Job is Queued or In Progress"); + } + + if($row['job_type'] == 'EX' || $row['job_type'] == 'OS') { + throw new \InvalidArgumentException("Legacy Job types cannot be transmogrified"); + } + } + + /** + * Translate booleans to string literals to work around DBAL Postgres boolean handling. + * + * @since COmanage Registry v5.0.0 + * @param string $table Table Name + * @param array $row Row of attributes, fixed in place + */ + protected function normalizeBooleanFieldsForDb(string $table, array &$row): void + { + $attrs = ['deleted']; + + // We could introspect this from the schema file... + if(!empty($this->tables[$table]['booleans'])) { + $attrs = array_merge($attrs, $this->tables[$table]['booleans']); + } + + foreach($attrs as $a) { + if(isset($row[$a]) && gettype($row[$a]) == 'boolean') { + // DBAL Postgres boolean handling seems to be somewhat buggy, see history in + // this issue: https://github.com/doctrine/dbal/issues/1847 + // We need to (more generically than this hack) convert from boolean to char + // to avoid errors on insert + if($this->outconn->isMySQL()) { + $row[$a] = ($row[$a] ? '1' : '0'); + } else { + $row[$a] = ($row[$a] ? 't' : 'f'); + } + } + } + } + + /** + * Populate empty Changelog data from legacy records + * + * @since COmanage Registry v5.0.0 + * @param string $table Table Name + * @param array $row Row of attributes, fixed in place + * @param bool $force If true, always create keys + */ + protected function populateChangelogDefaults(string $table, array &$row, bool $force=false): void + { + if ($force || (array_key_exists('deleted', $row) && is_null($row['deleted']))) { + $row['deleted'] = false; + } + + if ($force || (array_key_exists('revision', $row) && is_null($row['revision']))) { + $row['revision'] = 0; + } + + if ($force || (array_key_exists('actor_identifier', $row) && is_null($row['actor_identifier']))) { + $row['actor_identifier'] = 'Transmogrification'; + } + } + + /** + * Map fields that have been renamed from Registry Classic to Registry PE. + * + * @since COmanage Registry v5.0.0 + * @param string $table Table Name + * @param array $row Row of attributes, fixed in place + * @throws InvalidArgumentException + */ + protected function mapLegacyFieldNames(string $table, array &$row): void + { + // oldname => newname, or &newname, which is a function to call. + // Note functions can return more than one mapping + + if(empty($this->tables[$table]['fieldMap'])) { + return; + } + + $fields = $this->tables[$table]['fieldMap']; + + foreach ($fields as $oldname => $newname) { + // Determine the first character only if $newname is a non-empty string + $first = is_string($newname) && $newname !== '' ? $newname[0] : null; + + match (true) { + // No mapping: unset the legacy field + !$newname => $this->performNoMapping($row, $oldname), + + // Function mapping: compute value by helper method named after the &-prefixed token + $first === '&' => $this->performFunctionMapping($row, $oldname, substr((string)$newname, 1), $table), + + // Default value mapping: set only if current value is null + $first === '?' => $this->applyDefaultIfNull($row, $oldname, substr((string)$newname, 1)), + + // Direct rename: copy to new name and unset the old one + default => $this->renameField($row, $oldname, (string)$newname), + }; + } + } + + /** + * Process Extended Attributes by converting them to Ad Hoc Attributes. + * + * @since COmanage Registry v5.0.0 + */ + protected function migrateExtendedAttributesToAdHocAttributes(): void + { + // This is intended to run AFTER AdHocAttributes so that we don't stomp on + // the row identifiers. + + // First, pull the old Extended Attribute configuration. + $extendedAttrs = []; + + $tableName = "cm_co_extended_attributes"; + $qualifiedTableName = $this->inconn->qualifyTableName($tableName); + $insql = RawSqlQueries::buildSelectAllOrderedById($qualifiedTableName); + $stmt = $this->inconn->query($insql); + + while($row = $stmt->fetch()) { + $extendedAttrs[ $row['co_id'] ][] = $row['name']; + } + + if(empty($extendedAttrs)) { + // No need to do anything further if no attributes are configured + return; + } + + foreach(array_keys($extendedAttrs) as $coId) { + $tableName = "cm_co" . $coId . "_person_extended_attributes"; + $qualifiedTableName = $this->inconn->qualifyTableName($tableName); + $insql = RawSqlQueries::buildSelectAll($qualifiedTableName); + $stmt = $this->inconn->query($insql); + + while($eaRow = $stmt->fetch()) { + // If we didn't transmogrify the parent row for some reason then trying + // to insert the ad_hoc_attributes will throw an error. + if(!empty($this->cache['person_roles']['id'][ $eaRow['co_person_role_id'] ])) { + foreach($extendedAttrs[$coId] as $ea) { + $adhocRow = [ + 'person_role_id' => $eaRow['co_person_role_id'], + 'tag' => $ea, + 'value' => $eaRow[$ea], + 'created' => $eaRow['created'], + 'modified' => $eaRow['modified'] + ]; + + // Extended Attributes were not changelog enabled + $this->populateChangelogDefaults('ad_hoc_attributes', $adhocRow, true); + $this->normalizeBooleanFieldsForDb('ad_hoc_attributes', $adhocRow); + + try { + $tableName = 'ad_hoc_attributes'; + $qualifiedTableName = $this->outconn->qualifyTableName($tableName); + $this->outconn->insert($qualifiedTableName, $adhocRow); + } catch (\Doctrine\DBAL\Exception\UniqueConstraintViolationException $e) { + $this->cmdPrinter->warning("record already exists: " . print_r($adhocRow, true)); + } + } + } + } + } + } + + /** + * Split an External Identity into an External Identity Role. + * + * @param array $origRow Row of table data (original data) + * @param array $row Row of table data (post fixes) + * @throws Exception + * @since COmanage Registry v5.0.0 + */ + + protected function mapExternalIdentityToExternalIdentityRole(array $origRow, array $row): void + { + // the original row has the co_id + $roleRow = []; + + // We could set the row ID to be the same as the original parent, but then + // we'd have to reset the sequence after the table is finished migrating. + + foreach([ + // Parent Key + 'id' => 'external_identity_id', + 'o' => 'organization', + 'ou' => 'department', + 'manager_identifier' => 'manager_identifier', + 'sponsor_identifier' => 'sponsor_identifier', + 'status' => 'status', + 'title' => 'title', + 'valid_from' => 'valid_from', + 'valid_through' => 'valid_through', + // Fix up changelog + 'org_identity_id' => 'external_identity_role_id', + 'revision' => 'revision', + 'deleted' => 'deleted', + 'actor_identifier' => 'actor_identifier', + 'created' => 'created', + 'modified' => 'modified' + ] as $oldKey => $newKey) { + $roleRow[$newKey] = $origRow[$oldKey]; + } + + // Rationale: mapOrgIdentitycoPersonId accepts only the first mapping it finds + // (later mappings are treated as legacy/unpooled anomalies and ignored). + // Therefore, each ExternalIdentity produces at most one ExternalIdentityRole. + // To avoid inserting a bogus self-referential changelog link, do not carry any + // legacy key into external_identity_role_id. Start a fresh changelog chain. + $roleRow['external_identity_role_id'] = null; + $roleRow['revision'] = 0; + + + try { + if (!empty($origRow['affiliation'])) { + $row['affiliation'] = $origRow['affiliation']; + $roleRow['affiliation_type_id'] = $this->mapAffiliationType( + row: $row, + coId: $origRow['co_id'] ?? null + ); + } + } catch (\Exception $e) { + $this->cmdPrinter->warning("Failed to map affiliation type: " . $e->getMessage()); + if ( + isset($origRow['co_id']) + && (!isset($this->cache['cos'][$origRow['co_id']]) + || ($this->cache['cos'][$origRow['co_id']]['status']) + && $this->cache['cos'][$origRow['co_id']]['status'] == 'TR') + ) { + // This CO has been deleted, so we can't map the type. We will return null + $roleRow['affiliation_type_id'] = null; + } else { + // Rethrow the exception + throw $e; + } + } + + $tableName = 'external_identity_roles'; + + // Fix up changelog and booleans prior to insert + $this->populateChangelogDefaults($tableName, $roleRow, true); + $this->normalizeBooleanFieldsForDb($tableName, $roleRow); + + $qualifiedTableName = $this->outconn->qualifyTableName($tableName); + + $this->outconn->insert($qualifiedTableName, $roleRow); + } + + /** + * Unset a legacy field when it has no mapping. + * + * @param array &$row Row data to modify + * @param string $oldname Name of field to remove + * @return void + */ + private function performNoMapping(array &$row, string $oldname): void + { + unset($row[$oldname]); + } + + + /** + * Compute value for a field via a mapping function name (without the leading &). + * Reuses the original field name in-place. Throws if the mapping yields a falsy value. + * + * @param array &$row Row data to modify + * @param string $oldname Name of field to map + * @param string $funcName Name of mapping function to call + * @param string $table Table name for error reporting + * @return void + * @throws \InvalidArgumentException When mapping returns falsy value or function not found + */ + private function performFunctionMapping(array &$row, string $oldname, string $funcName, string $table): void + { + if (!method_exists($this, $funcName)) { + throw new \InvalidArgumentException("Mapping function {$funcName} does not exist for {$table} {$oldname}"); + } + + // We always pass the entire row so the mapping function can implement arbitrary logic + $row[$oldname] = $this->$funcName($row); + + // NOTE: mapAffiliationType can return null since extended types might not exist for deleted COs + if (!$row[$oldname] && $funcName !== 'mapAffiliationType') { + throw new \InvalidArgumentException("Could not find value for {$table} {$oldname}"); + } + } + + /** + * Apply a default value only when the current value is strictly null. + * + * @param array &$row Row data to modify + * @param string $oldname Name of field to check/update + * @param string $default Default value to apply if field is null + * @return void + */ + private function applyDefaultIfNull(array &$row, string $oldname, string $default): void + { + if (array_key_exists($oldname, $row) && $row[$oldname] === null) { + $row[$oldname] = $default; + } + } + + /** + * Rename a field by copying its value to a new key and removing the old key. + * + * @param array &$row Row data to modify + * @param string $oldname Original field name + * @param string $newname New field name + * @return void + */ + private function renameField(array &$row, string $oldname, string $newname): void + { + // Only copy if the old field exists to avoid notices + if (array_key_exists($oldname, $row)) { + $row[$newname] = $row[$oldname]; + unset($row[$oldname]); + } + } +} \ No newline at end of file diff --git a/app/plugins/Transmogrify/src/Lib/Traits/TypeMapperTrait.php b/app/plugins/Transmogrify/src/Lib/Traits/TypeMapperTrait.php new file mode 100644 index 000000000..c6cfdbb33 --- /dev/null +++ b/app/plugins/Transmogrify/src/Lib/Traits/TypeMapperTrait.php @@ -0,0 +1,433 @@ + + */ + + public const SERVER_TYPE_MAP = [ + 'HT' => 'CoreServer.HttpServers', + 'KA' => 'CoreServer.KafkaServers', + 'KC' => 'CoreServer.KdcServers', + 'LD' => 'CoreServer.LdapServers', + 'MT' => 'CoreServer.MatchServers', + 'O2' => 'CoreServer.Oauth2Servers', + 'SQ' => 'CoreServer.SqlServers', + ]; + + /** + * Map match attribute types to corresponding model names + * + * @var array + * @since COmanage Registry v5.2.0 + */ + public const MATCH_ATTRIBUTE_TYPE_MAP = [ + 'emailAddress' => 'EmailAddresses', + 'name' => 'Names', + 'identifier' => 'Identifiers', + 'dateOfBirth' => '', + ]; + + /** + * Map address type to corresponding type ID + * + * @param array $row Row data containing address type + * @return int|null Mapped type ID + * @since COmanage Registry v5.0.0 + */ + protected function mapAddressType(array $row): ?int + { + return $this->mapType($row, 'Addresses.type', $this->findCoId($row)); + } + + /** + * Map affiliation type to corresponding type ID + * + * @param array $row Row data containing affiliation + * @param int|null $coId CO Id or null if not known + * @return int|null Mapped type ID + * @since COmanage Registry v5.0.0 + */ + protected function mapAffiliationType(array $row, ?int $coId = null): ?int + { + return $this->mapType( + $row, + 'PersonRoles.affiliation_type', + $coId ?? $this->findCoId($row), + 'affiliation' + ); + } + + /** + * Map email type to corresponding type ID + * + * @param array $row Row data containing email type + * @return int|null Mapped type ID + * @since COmanage Registry v5.0.0 + */ + protected function mapEmailType(array $row): ?int + { + return $this->mapType($row, 'EmailAddresses.type', $this->findCoId($row)); + } + + /** + * Map an Extended Type attribute name for model name changes. + * + * @since COmanage Registry v5.0.0 + * @param array $row Row of table data + * @return string Updated attribute name + */ + + protected function mapExtendedType(array $row): string + { + switch($row['attribute']) { + case 'CoDepartment.type': + return 'Departments.type'; + case 'CoPersonRole.affiliation': + return 'PersonRoles.affiliation_type'; + } + + // For everything else, we need to pluralize the model name + $bits = explode('.', $row['attribute'], 2); + + return Inflector::pluralize($bits[0]) . "." . $bits[1]; + } + + /** + * Map identifier type to corresponding type ID + * + * @param array $row Row data containing identifier type + * @return int|null Mapped type ID + * @since COmanage Registry v5.0.0 + */ + protected function mapIdentifierType(array $row): ?int + { + return $this->mapType($row, 'Identifiers.type', $this->findCoId($row)); + } + + + /** + * Map match attribute type to corresponding type ID based on attribute key + * + * @param array $row Row data containing match attribute + * @return int|null Mapped type ID or null if attribute not found/mapped + * @since COmanage Registry v5.2.0 + */ + protected function mapMatchAttributeTypeId(array $row): ?int + { + // Determine the model based on the match attribute map (eg, emailAddress -> EmailAddresses) + $attributeKey = $row['attribute'] ?? null; + if (!$attributeKey || !array_key_exists($attributeKey, self::MATCH_ATTRIBUTE_TYPE_MAP)) { + return null; + } + + $model = self::MATCH_ATTRIBUTE_TYPE_MAP[$attributeKey]; + + // If the mapping is empty (eg, dateOfBirth), return null + if ($model === '') { + return null; + } + return $this->mapType($row, $model . '.type', $this->findCoId($row)); + } + + /** + * Map login identifiers, in accordance with the configuration. + * + * @since COmanage Registry v5.0.0 + * @param array $origRow Row of table data (original data) + * @param array $row Row of table data (post fixes) + * @throws \InvalidArgumentException + */ + protected function mapLoginIdentifiers(array $origRow, array &$row): void { + // There might be multiple reasons to copy the row, but we only want to + // copy it once. + $copyRow = false; + + if(!empty($origRow['org_identity_id'])) { + if($this->args->getOption('login-identifier-copy') + && $origRow['login']) { + $copyRow = true; + } + + // Note the argument here is the old v4 string (eg "eppn") and not the + // PE foreign key + if($this->args->getOption('login-identifier-type') + && $origRow['type'] == $this->args->getOption('login-identifier-type')) { + $copyRow = true; + } + + // Identifiers attached to External Identities do not have login flags in PE + $row['login'] = false; + } + + if($copyRow) { + // Find the Person ID associated with this External Identity ID + + if(!empty($this->cache['external_identities']['id'][ $origRow['org_identity_id'] ]['person_id'])) { + // Insert a new row attached to the Person, leave the original record + // (ie: $row) untouched + + $copiedRow = [ + 'person_id' => $this->map_org_identity_co_person_id(['id' => $origRow['org_identity_id']]), + 'identifier' => $origRow['identifier'], + 'type_id' => $this->map_identifier_type($origRow), + 'status' => $origRow['status'], + 'login' => true, + 'created' => $origRow['created'], + 'modified' => $origRow['modified'] + ]; + + // Set up changelog and fix booleans + $this->fixChangelog('identifiers', $copiedRow, true); + $this->fixBooleans('identifiers', $copiedRow); + + try { + $tableName = 'identifiers'; + $qualifiedTableName = $this->outconn->qualifyTableName($tableName); + $this->outconn->insert($qualifiedTableName, $copiedRow); + } catch (UniqueConstraintViolationException $e) { + $this->cmdPrinter->warning("record already exists: " . print_r($copiedRow, true)); + } + } + } + } + + /** + * Map name type to corresponding type ID + * + * @param array $row Row data containing name type + * @return int|null Mapped type ID + * @since COmanage Registry v5.0.0 + */ + protected function mapNameType(array $row): ?int + { + return $this->mapType($row, 'Names.type', $this->findCoId($row)); + } + + /** + * Return a timestamp equivalent to now. + * + * @since COmanage Registry v5.0.0 + * @param array $row Row of table data (ignored) + * @return string Timestamp + */ + + protected function mapNow(array $row) { + if(empty($this->cache['now'])) { + $created = new \Datetime('now'); + $this->cache['now'] = $created->format('Y-m-d H:i:s'); + } + + return $this->cache['now']; + } + + /** + * Map an Org Identity ID to a CO Person ID + * + * @param array $row Row of Org Identity table data + * @return int|null CO Person ID + * @throws Exception + * @since COmanage Registry v5.0.0 + */ + protected function mapOrgIdentitycoPersonId(array $row): ?int + { + // PE eliminates OrgIdentityLink, so we need to map each Org Identity to + // a Person ID. Historically, an Org Identity could have been relinked and + // had multiple historical mappings. We now fetch only the current row, + // so we no longer need to track or select by revision. + + // Before Transmogrification, we require that Org Identities are unpooled. + // (This is probably how most deployments are set up, but there may be some + // legacy deployments out there.) This ensures whatever CO Person the Org + // Identity currently maps to through CoOrgIdentityLink is in the same CO. + + // There may be multiple mappings if the Org Identity was relinked. Basically + // we're going to lose the multiple mappings, since we can only return one + // value here. (Ideally, we would inject multiple OrgIdentities into the new + // table, but this ends up being rather tricky, since we have to figure out + // what row id to assign, and for the moment we don't have a mechanism to + // do that.) Historical information remains available in history_records, + // and if the deployer keeps an archive of the old database. + + $rowId = (int)$row['id']; + + if (empty($this->cache['org_identities']['co_people'])) { + $this->cache['org_identities']['co_people'] = []; + + // Build cache on first use + $this->cmdPrinter->verbose('Populating org identity map...'); + + $tableName = 'cm_co_org_identity_links'; + $changelogFK = 'co_org_identity_link_id'; + $qualifiedTableName = $this->inconn->qualifyTableName($tableName); + + // Only fetch current rows (historical/changelog rows are filtered out) + $mapsql = RawSqlQueries::buildSelectAllWithNoChangelong( + qualifiedTableName: $qualifiedTableName, + changelogFK: $changelogFK + ); + + $stmt = $this->inconn->executeQuery($mapsql); + + while ($r = $stmt->fetchAssociative()) { + $oid = $r['org_identity_id'] ?? null; + + if(!empty($oid)) { + $rowRev = $r['revision']; + if(isset($this->cache['org_identities']['co_people'][ $oid ][ $rowRev ])) { + // If for some reason we already have a record, it's probably due to + // improper unpooling from a legacy deployment. We'll accept only the + // first record and throw warnings on the others. + + $this->cmdPrinter->verbose("Found existing CO Person for Org Identity " . $oid . ", skipping"); + } else { + // Store as-is; we'll resolve the latest revision on lookup + $this->cache['org_identities']['co_people'][ $oid ][ $rowRev ] = (int)$r['co_person_id']; + } + } + } + + // Preserve keys while providing the deterministic order of revisions + foreach ($this->cache['org_identities']['co_people'] as &$revisions) { + if (is_array($revisions)) { + ksort($revisions, SORT_NUMERIC); + } + } + unset($revisions); + } + + if (!empty($this->cache['org_identities']['co_people'][$rowId])) { + // Return the record with the highest revision number + $revisions = $this->cache['org_identities']['co_people'][$rowId]; + $rev = max(array_keys($revisions)); + + return (int)$revisions[$rev]; + } + + // No current mapping found for this Org Identity + return null; + } + + /** + * Map server type code to corresponding plugin path + * + * @param array $row Row data containing server type + * @return string|null Mapped plugin path or null if not found + * @since COmanage Registry v5.2.0 + */ + protected function mapServerTypeToPlugin(array $row): ?string + { + return (string)self::SERVER_TYPE_MAP[$row['server_type']] ?? null; + } + + /** + * Map telephone type to corresponding type ID + * + * @param array $row Row data containing telephone type + * @return int|null Mapped type ID + * @since COmanage Registry v5.0.0 + */ + protected function mapTelephoneType(array $row): ?int + { + return $this->mapType($row, 'TelephoneNumbers.type', $this->findCoId($row)); + } + + /** + * Map a type value to its corresponding ID + * + * @param array $row Row data containing the type value + * @param string $type Type attribute name + * @param int $coId CO ID + * @param string $attr Attribute name in row data + * @return int|null Mapped type ID + * @throws \InvalidArgumentException When type not found + * @since COmanage Registry v5.0.0 + */ + protected function mapType(array $row, string $type, int $coId, string $attr = 'type'): ?int + { + // If we delete a CO, we permanently delete the extended attributes. As a result, the type + // is no longer available because there is no changelog. Types are used by Person Roles and + // External Identity Roles. + if(!$coId) { + throw new \InvalidArgumentException("CO ID not provided for $type " . ($row['id'] ?? '')); + } + $value = $row[$attr] ?? null; + $key = $coId . "+" . $type . "+" . $value . "+"; + if(empty($this->cache['types']['co_id+attribute+value+'][$key])) { + if ( + !isset($this->cache['cos'][$coId]) + || ($this->cache['cos'][$coId]['status'] && in_array($this->cache['cos'][$coId]['status'], ['TR'])) + ) { + // This CO has been deleted, so we can't map the type. We will return null + return null; + } + throw new \InvalidArgumentException("Type not found for " . $key); + } + return (int)$this->cache['types']['co_id+attribute+value+'][$key]; + } + + /** + * Map URL type to corresponding type ID + * + * @param array $row Row data containing URL type + * @return int|null Mapped type ID + * @since COmanage Registry v5.0.0 + */ + protected function mapUrlType(array $row): ?int + { + return $this->mapType($row, 'Urls.type', $this->findCoId($row)); + } +} diff --git a/app/plugins/Transmogrify/src/Lib/Util/CommandLinePrinter.php b/app/plugins/Transmogrify/src/Lib/Util/CommandLinePrinter.php new file mode 100644 index 000000000..80f2f5a04 --- /dev/null +++ b/app/plugins/Transmogrify/src/Lib/Util/CommandLinePrinter.php @@ -0,0 +1,577 @@ +io = $io; + $this->barColor = in_array($barColor, ['blue', 'green'], true) ? $barColor : 'blue'; + $this->barWidth = max(10, $barWidth); + $this->useColors = $useColors; + + // Info is forced to be white + $this->io->setStyle('info', ['text' => '0;39']); + $this->io->setStyle('question', ['text' => '0;39']); + } + + /** + * Start displaying a new progress bar + * + * @param int $total Total number of steps + * @return void + * @since COmanage Registry v5.2.0 + */ + public function start(int $total): void + { + // When verbose is enabled, do not draw the progress bar at all + if ($this->getVerboseLevel() > 1) { + return; + } + + $this->total = max(0, $total); + $this->current = 0; + $this->messageLines = 0; + $this->barActive = true; + + // Hard reset the current line, then move to a brand new line + // \r -> move to column 0 + // \033[K -> clear to end of line + // \n -> go to next line + $this->rawWrite("\r\033[2K"); + + // Draw initial progress bar line (no leading \r, we are already at col 0) + $this->rawWrite($this->formatBar(0)); + + // Save cursor position at the end of the progress bar line + $this->rawWrite("\033[s"); + + // Move to message area (one line below the bar) + $this->rawWrite("\033[1B\r"); + } + + /** + * Update the progress bar to show current progress + * + * @param int $current Current step number + * @return void + * @since COmanage Registry v5.2.0 + */ + public function update(int $current): void + { + // When verbose is enabled, do not draw the progress bar at all + if ($this->getVerboseLevel() > 1 || !$this->barActive) { + return; + } + + $this->current = min(max(0, $current), $this->total); + + // Restore to bar line, redraw bar, save, then go back to message area + $this->rawWrite("\033[u"); // restore saved cursor (bar line end) + $this->rawWrite("\r" . $this->formatBar($this->current)); + $this->rawWrite("\033[s"); // save again at end of bar + $down = 1 + $this->messageLines; // one below bar + existing messages + $this->rawWrite("\033[" . $down . "B\r"); + } + + /** + * Complete and cleanup the progress bar display + * + * @return void + * @since COmanage Registry v5.2.0 + */ + public function finish(): void + { + // When verbose is enabled, do not draw the progress bar at all + if ($this->getVerboseLevel() > 1 || !$this->barActive) { + return; + } + + $this->update($this->total); + + // Move the cursor to the line AFTER the whole message area, + // so subsequent output doesn't overwrite the bar. + $this->rawWrite("\033[u"); // restore saved position at end of bar line + $down = 1 + $this->messageLines; // one below bar + all message lines + $this->rawWrite("\033[" . $down . "B\r"); // move down + $this->rawWrite(PHP_EOL); // clean newline below + + // Re-anchor saved cursor to this clean line so future restores are safe + $this->rawWrite("\033[s"); + + // Reset internal counters so next run starts fresh + $this->messageLines = 0; + $this->barActive = false; + $this->total = 0; + $this->current = 0; + } + + /** + * Display an out level message + * + * @param string $message Message to display + * @return void + * @since COmanage Registry v5.2.0 + */ + public function out(string $message): void + { + $this->message($message, 'out'); + } + + /** + * Display an info level message + * + * @param string $message Message to display + * @return void + * @since COmanage Registry v5.2.0 + */ + public function info(string $message): void + { + $this->message($message, 'info'); + } + + /** + * Display a warning level message (alias of warn()) + * + * @param string $message Message to display + * @return void + * @since COmanage Registry v5.2.0 + */ + public function warning(string $message): void + { + $this->message($message, 'warn'); + } + + /** + * Display an error level message + * + * @param string $message Message to display + * @return void + * @since COmanage Registry v5.2.0 + */ + public function error(string $message): void + { + $this->message($message, 'error'); + } + + /** + * Display a debug level message + * + * @param string $message Message to display + * @return void + * @since COmanage Registry v5.2.0 + */ + public function debug(string $message): void + { + $this->message($message, 'debug'); + } + + /** + * Display a verbose level message + * + * @param string $message Message to display + * @return void + * @since COmanage Registry v5.2.0 + */ + public function verbose(string $message): void + { + $this->message($message, 'verbose'); + } + + /** + * Display a message with the specified level + * + * @param string $message Message to display + * @param string $level Message level (info, warn, error, debug, verbose) + * @return void + * @since COmanage Registry v5.2.0 + */ + public function message(string $message, string $level = 'info'): void + { + // Suppress verbose messages unless verbose mode is enabled + if (!$this->shouldPrintLevel($level)) { + return; + } + + $lines = $this->colorizeLevel($level, $message); + // Ensure the message ends with a newline so we can count line advances + if ($lines === '' || substr($lines, -1) !== "\n") { + $lines .= "\n"; + } + + + if ($this->barActive) { + // Always print below the bar + $this->rawWrite("\033[u"); // restore to end of bar line + $down = 1 + $this->messageLines; + $this->rawWrite("\033[" . $down . "B"); // move to message area line + // Clear the entire message line and reset cursor to column 0 + $this->rawWrite("\r\033[2K"); + $this->rawWrite($lines); + $this->messageLines += substr_count($lines, "\n"); + + // Return to bar line and re-save for next update + $this->rawWrite("\033[u"); + $this->rawWrite("\033[s"); + } else { + // No bar: clear current line completely and print from column 0 + $this->rawWrite("\r\033[2K"); + $this->rawWrite($lines); + } + } + + /** + * Prompt for user input + * + * @param string $prompt Prompt to display + * @param string|null $default Default value if no input provided + * @return string|null User input or default value + * @since COmanage Registry v5.2.0 + */ + public function ask(string $prompt, ?string $default = null): ?string + { + // Ensure the message area exists (one line below the bar) + if ($this->messageLines === 0) { + $this->rawWrite(PHP_EOL); + } + + $answer = null; + + if ($this->io) { + // ConsoleIo handles rendering the prompt and reading input + $answer = $this->io->ask($prompt, $default); + } else { + // Fallback to STDOUT/STDIN + $this->rawWrite($prompt . ' '); + $line = fgets(STDIN); + $answer = ($line === false) ? null : rtrim($line, "\r\n"); + if ($answer === null && $default !== null) { + $answer = $default; + } + // Ensure the cursor advances to the next line after the prompt + $this->rawWrite(PHP_EOL); + } + + // A prompt line was added to the message area + $this->messageLines += 1; + + // Redraw progress bar and return cursor to the end of the message area + $this->rawWrite("\033[u"); // restore to saved bar position + $this->rawWrite("\r" . $this->formatBar($this->current)); + $this->rawWrite("\033[s"); // save bar position again + $this->rawWrite("\033[" . $this->messageLines . "B\r"); // move down to message area + + return $answer; + } + + /** + * Pause execution until user presses enter + * + * @param string $prompt Prompt to display + * @return void + * @since COmanage Registry v5.2.0 + */ + public function pause(string $prompt = 'Press to continue...'): void + { + $this->ask($prompt, ''); + } + + /** + * Add color formatting to a message based on its level + * + * @param string $level Message level + * @param string $message Message to colorize + * @return string Formatted message + * @since COmanage Registry v5.2.0 + */ + private function colorizeLevel(string $level, string $message): string + { + $level = strtolower($level); + switch ($level) { + case 'warn': + case 'warning': + $prefix = '[WARN] '; + break; + case 'error': + $prefix = '[ERROR] '; + break; + case 'debug': + $prefix = '[DEBUG] '; + break; + case 'verbose': + $prefix = '[VERBOSE] '; + break; + case 'info': + $prefix = '[INFO] '; + break; + default: + $prefix = ''; + } + + // For INFO, render the label (up to the first colon) in white, value unchanged (or green if white info default) + if ($level === 'info' || $level === 'out') { + $formatted = $this->useColors ? $this->formatLabelWhite($message) : $message; + return $prefix . $formatted; + } + + $color = $this->defaultColorForLevel($level); + $text = $prefix . $message; + if ($this->useColors && $color) { + return $this->wrapColor($text, $color); + } + return $text; + } + + /** + * Get the default color for a message level + * + * @param string $level Message level + * @return string|null Color name or null if no color + * @since COmanage Registry v5.2.0 + */ + private function defaultColorForLevel(string $level): ?string + { + $level = strtolower($level); + return match ($level) { + 'warn', 'warning' => 'yellow', + 'error' => 'red', + 'debug' => 'cyan', + default => 'white', + }; + } + + /** + * Format the progress bar string + * + * @param int $current Current progress value + * @return string Formatted progress bar + * @since COmanage Registry v5.2.0 + */ + private function formatBar(int $current): string + { + $total = max(1, $this->total); + $pct = (int) floor(($current / $total) * 100); + + $width = $this->barWidth; + $filled = (int) floor(($pct / 100) * $width); + $remaining = max(0, $width - $filled); + + $filledStr = str_repeat('=', max(0, $filled - 1)) . ($filled > 0 ? '>' : ''); + $emptyStr = str_repeat('.', $remaining); + + $bar = sprintf('[%s%s] %3d%% (%d/%d)', $filledStr, $emptyStr, $pct, $current, $this->total); + + // Clear the line to the right to avoid remnants on shorter redraws + $bar .= "\033[K"; + + if ($this->useColors) { + $color = $this->barColor === 'green' ? 'green' : 'blue'; + $bar = $this->wrapColor($bar, $color); + } + return $bar; + } + + /** + * Wrap text in ANSI color codes + * + * @param string $text Text to colorize + * @param string $color Color name + * @return string Color-wrapped text + * @since COmanage Registry v5.2.0 + */ + private function wrapColor(string $text, string $color): string + { + $map = [ + 'red' => '0;31', + 'green' => '0;32', + 'yellow' => '0;33', + 'blue' => '0;34', + 'magenta'=> '0;35', + 'cyan' => '0;36', + 'white' => '0;39', + ]; + $code = $map[$color] ?? null; + if (!$code) { return $text; } + return "\033[{$code}m{$text}\033[0m"; + } + + /** + * Write raw string to output + * + * @param string $str String to write + * @return void + * @since COmanage Registry v5.2.0 + */ + private function rawWrite(string $str): void + { + if ($this->io) { + // ConsoleIo::out() defaults to a trailing newline; we want raw text + $this->io->out($str, 0, 0); + } else { + // Fallback to STDOUT + echo $str; + } + } + + /** + * Format info message with white label + * + * @param string $message Message to format + * @return string Formatted message + * @since COmanage Registry v5.2.0 + */ + private function formatLabelWhite(string $message): string + { + $lines = explode("\n", $message); + foreach ($lines as $i => $line) { + if ($line === '') { continue; } + if (preg_match('/^([^:\r\n]+:)(.*)$/', $line, $m)) { + $second = $m[2]; + if ( + $this->useColors + && ($this->defaultColorForLevel('info') === 'white' || $this->defaultColorForLevel('out') === 'white') + && $second !== '' + ) { + $second = $this->wrapColor($second, 'green'); + } + $lines[$i] = $this->wrapColor($m[1], 'white') . $second; + } + } + return implode("\n", $lines); + } + + /** + * Detect verbose level from ConsoleIO instance + * + * @return int Verbose level + * @since COmanage Registry v5.2.0 + */ + private function detectVerboseFromIo(): int + { + $default = 0; + if ($this->io === null) { + return $default; + } + if (method_exists($this->io, 'level')) { + try { + return $this->io->level(); + } catch (\Throwable $e) { + return $default; + } + } + return $default; + } + + /** + * Determine if a message of the given level should be printed based on verbosity setting + * + * @param string $level Message level to check + * @return bool Whether the message should be printed + * @since COmanage Registry v5.2.0 + */ + private function shouldPrintLevel(string $level): bool + { + $level = strtolower($level); + $verboseLevel = $this->detectVerboseFromIo(); + + return match ($verboseLevel) { + 0 => in_array($level, ['out', 'error'], true), // only errors and out + 1 => in_array($level, ['out', 'info', 'warn', 'warning', 'error'], true), // no verbose/debug + 2 => in_array($level, ['out', 'info', 'warn', 'warning', 'error', 'debug', 'verbose'], true), // all messages + default => true, + }; + } + + /** + * Get the current verbosity level + * + * @return int Verbosity level (0=quiet, 1=normal, 2=verbose) + * @since COmanage Registry v5.2.0 + */ + private function getVerboseLevel(): int + { + return $this->detectVerboseFromIo(); + } +} diff --git a/app/plugins/Transmogrify/src/Lib/Util/DbInfoPrinter.php b/app/plugins/Transmogrify/src/Lib/Util/DbInfoPrinter.php new file mode 100644 index 000000000..da6616291 --- /dev/null +++ b/app/plugins/Transmogrify/src/Lib/Util/DbInfoPrinter.php @@ -0,0 +1,246 @@ +io = $io; + $this->service = $service; + $this->pluginRoot = dirname(__DIR__, 3); + } + + /** + * Initializes the singleton instance + * @param ConsoleIo $io Console IO instance for output + * @param DbInfoService $service Database info service + * @return self The singleton instance + */ + public static function initialize(ConsoleIo $io, DbInfoService $service): self + { + if (self::$instance === null) { + self::$instance = new self($io, $service); + } + return self::$instance; + } + + + /** + * Prints database connection and schema information + * @param bool $asJson Whether to output as JSON + * @param bool $withPing Whether to test database connectivity + * @param bool $withSchema Whether to include schema information + * @param string|null $schemaRole Limit schema info to specific role + */ + public function print(bool $asJson, bool $withPing, bool $withSchema = false, ?string $schemaRole = null): void + { + $aliases = [ 'source' => 'transmogrify', 'target' => 'default' ]; + $data = []; + + foreach ($aliases as $role => $alias) { + // Base connection info + $info = $this->service->getConnectionInfo($role); + + if ($withPing) { + $info['status'] = $this->service->ping($alias); + } + + if ($withSchema && ($schemaRole === null || $schemaRole === $role)) { + try { + + $info['schema'] = $this->service->loadSchemaInfo( + $alias, + $this->pluginRoot . DS . Transmogrify::TABLES_JSON_PATH); + } catch (\Throwable $e) { + $info['schema'] = [ 'error' => $e->getMessage() ]; + } + } + + $data[$role] = $info; + } + + if ($asJson) { + $this->io->out(json_encode($data, JSON_PRETTY_PRINT)); + return; + } + + foreach (['source', 'target'] as $role) { + $i = $data[$role] ?? []; + $header = ucfirst($role) . ':'; + $this->io->out(self::STYLE_BOLD . $header . self::COLOR_RESET); + $this->io->out(str_repeat('-', strlen($header))); + $this->io->out(' alias: ' . ($i['alias'] ?? '')); + $this->io->out(' configured:' . ((($i['configured'] ?? false)) ? ' yes' : ' no')); + if (!empty($i['driver'])) $this->io->out(' driver: ' . $i['driver']); + if (!empty($i['host'])) $this->io->out(' host: ' . $i['host']); + if (!empty($i['port'])) $this->io->out(' port: ' . $i['port']); + if (!empty($i['database'])) $this->io->out(' database: ' . $i['database']); + if (!empty($i['username'])) $this->io->out(' username: ' . $i['username']); + if (!empty($i['dsn'])) $this->io->out(' dsn: ' . $i['dsn']); + if (!empty($i['schema'])) { + // Determine a schema name (for PostgreSQL) and strip schema prefixes for sample tables display + $schemaName = null; + $bareSample = []; + $s = $i['schema']; + if (!empty($s['sample_tables'])) { + foreach ($s['sample_tables'] as $t) { + $dot = strrpos($t, '.'); + if ($dot !== false) { + $schemaName = $schemaName ?? substr($t, 0, $dot); + $bareSample[] = substr($t, $dot + 1); + } else { + $bareSample[] = $t; + } + } + } + $s = $i['schema']; + $schemaHeader = 'Schema:'; + $this->io->out(' ' . self::STYLE_BOLD . $schemaHeader . self::COLOR_RESET); + $this->io->out(' ' . str_repeat('-', strlen($schemaHeader))); + if (!empty($schemaName)) { + $this->io->out(' name: ' . $schemaName); + } + $this->io->out(' empty: ' . (($s['empty'] ?? false) ? 'yes' : 'no')); + $this->io->out(' tables: ' . ($s['table_count'] ?? 0)); + if (!empty($s['sample_tables'])) { + $sampleHeader = 'Sample tables:'; + $this->io->out(' ' . self::STYLE_BOLD . $sampleHeader . self::COLOR_RESET); + $this->io->out(' ' . str_repeat('-', strlen($sampleHeader))); + $list = !empty($bareSample) ? $bareSample : $s['sample_tables']; + foreach ($list as $t) { $this->io->out(' - ' . $t); } + } + if (!empty($s['tables_compare'])) { + $cmp = $s['tables_compare']; + // Add a blank line then print comparison headers at top level (aligned with SOURCE/TARGET) + $this->io->out(''); + $header = 'Tables present (json ∧ db):'; + $this->io->out(self::STYLE_BOLD . $header . self::COLOR_RESET); + $this->io->out(str_repeat('-', strlen($header))); + // Tables present in both JSON and DB + foreach (($cmp['both'] ?? []) as $t) { + $this->io->out(' ' . self::COLOR_GREEN . '✔' . self::COLOR_RESET . ' ' . $t); + } + // Only declared in tables.json + if (!empty($cmp['only_in_json'])) { + $onlyJsonHeader = 'Only in tables.json:'; + $this->io->out(self::STYLE_BOLD . $onlyJsonHeader . self::COLOR_RESET); + $this->io->out(str_repeat('-', strlen($onlyJsonHeader))); + foreach ($cmp['only_in_json'] as $t) { $this->io->out(' - ' . $t); } + } + // Present only in the database (render in 3 columns) + if (!empty($cmp['only_in_db'])) { + $onlyDbHeader = 'Only in database:'; + $this->io->out(self::STYLE_BOLD . $onlyDbHeader . self::COLOR_RESET); + $this->io->out(str_repeat('-', strlen($onlyDbHeader))); + $list = array_values($cmp['only_in_db']); + $cols = 3; + $rows = (int)ceil(count($list) / $cols); + // Determine max width per column + $widths = array_fill(0, $cols, 0); + for ($c = 0; $c < $cols; $c++) { + for ($r = 0; $r < $rows; $r++) { + $idx = $r + $rows * $c; + if ($idx < count($list)) { + $len = strlen((string)$list[$idx]); + if ($len > $widths[$c]) { $widths[$c] = $len; } + } + } + } + // Print rows + for ($r = 0; $r < $rows; $r++) { + $line = ' '; + for ($c = 0; $c < $cols; $c++) { + $idx = $r + $rows * $c; + if ($idx < count($list)) { + $cell = (string)$list[$idx]; + // No padding after last printed column + if ($c === $cols - 1 || ($r + $rows * ($c + 1)) >= count($list)) { + $line .= $cell; + } else { + $line .= str_pad($cell, $widths[$c]) . ' '; + } + } + } + $this->io->out($line); + } + } + } + } + + if (isset($i['status'])) { + $st = $i['status']; + + $this->io->out(' connectivity: ' . ($st['ok'] + ? self::COLOR_GREEN . '✔ OK' . self::COLOR_RESET + : self::COLOR_RED . '✘ ERROR' . self::COLOR_RESET)); + if (!empty($st['server'])) $this->io->out(' server: ' . $st['server']); + if (!$st['ok'] && !empty($st['error'])) $this->io->out(' error: ' . $st['error']); + } + $this->io->out(''); + } + } + +} diff --git a/app/plugins/Transmogrify/src/Lib/Util/GroupsHealth.php b/app/plugins/Transmogrify/src/Lib/Util/GroupsHealth.php new file mode 100644 index 000000000..5e7a68981 --- /dev/null +++ b/app/plugins/Transmogrify/src/Lib/Util/GroupsHealth.php @@ -0,0 +1,141 @@ +out('Running Groups health check (AR-Group-9)...'); + $sql = RawSqlQueries::STANDARD_GROUP_ARG9_SQL_QUERY; + + try { + $rows = $inconn->fetchAllAssociative($sql); + } catch (\Throwable $e) { + $io->err('Groups health check failed: ' . $e->getMessage()); + return; + } + + if (empty($rows)) { + $io->out('No results.'); + return; + } + + // Detect available columns + $first = $rows[0]; + $hasIncluded = array_key_exists('included_count', $first); + $hasExcluded = array_key_exists('excluded_count', $first); + $hasIndicator = array_key_exists('indicator', $first); + $hasCount = array_key_exists('count', $first); + + // Prepare headers based on detected columns + if ($hasIncluded || $hasExcluded) { + $headers = ['Reason', 'Included', 'Excluded']; + if ($hasIndicator) { + $headers[] = 'Indicator'; + } + } else { + // Fallback to simple reason + count (and indicator if present) + $headers = ['Reason', 'Count']; + if ($hasIndicator) { + $headers[] = 'Indicator'; + } + } + + // Compute column widths + $widths = array_fill(0, count($headers), 0); + $reasonIdx = 0; + $incIdx = array_search('Included', $headers, true); + $excIdx = array_search('Excluded', $headers, true); + $cntIdx = array_search('Count', $headers, true); + $indIdx = array_search('Indicator', $headers, true); + + // Initialize with header widths + foreach ($headers as $i => $h) { + $widths[$i] = max($widths[$i], mb_strlen($h)); + } + + // Measure data + foreach ($rows as $r) { + $reasonLen = mb_strlen((string)($r['reason'] ?? '')); + $widths[$reasonIdx] = max($widths[$reasonIdx], $reasonLen); + + if ($incIdx !== false) { + $widths[$incIdx] = max($widths[$incIdx], mb_strlen((string)($r['included_count'] ?? ''))); + } + if ($excIdx !== false) { + $widths[$excIdx] = max($widths[$excIdx], mb_strlen((string)($r['excluded_count'] ?? ''))); + } + if ($cntIdx !== false) { + $widths[$cntIdx] = max($widths[$cntIdx], mb_strlen((string)($r['count'] ?? ''))); + } + if ($indIdx !== false) { + $widths[$indIdx] = max($widths[$indIdx], mb_strlen((string)($r['indicator'] ?? ''))); + } + } + + // Helper to pad a cell + $pad = static function (string $s, int $w): string { + $len = mb_strlen($s); + if ($len >= $w) { + return $s; + } + return $s . str_repeat(' ', $w - $len); + }; + + // Print header + $lineParts = []; + foreach ($headers as $i => $h) { + $lineParts[] = $pad($h, $widths[$i]); + } + $io->out(implode(' | ', $lineParts)); + + // Print separator + $sepParts = array_map(static fn($w) => str_repeat('-', $w), $widths); + $io->out(implode('--+--', $sepParts)); + + // Print rows + foreach ($rows as $r) { + $rowParts = []; + $rowParts[] = $pad((string)($r['reason'] ?? ''), $widths[$reasonIdx]); + + if ($incIdx !== false) { + $rowParts[] = $pad((string)($r['included_count'] ?? ''), $widths[$incIdx]); + } + if ($excIdx !== false) { + $rowParts[] = $pad((string)($r['excluded_count'] ?? ''), $widths[$excIdx]); + } + if ($cntIdx !== false) { + $rowParts[] = $pad((string)($r['count'] ?? ''), $widths[$cntIdx]); + } + if ($indIdx !== false) { + $indRaw = (string)($r['indicator'] ?? ''); + $cell = $pad($indRaw, $widths[$indIdx]); + + // Colorize first visible char, leave padding spaces uncolored to preserve alignment + if ($indRaw === 'x' || $indRaw === '✓') { + $color = ($indRaw === 'x') ? "\033[31m" : "\033[32m"; // red for x, green for ✓ + $reset = "\033[0m"; + $first = mb_substr($cell, 0, 1); + $rest = mb_substr($cell, 1); + $cell = $color . $first . $reset . $rest; + } + + $rowParts[] = $cell; + } + + $io->out(implode(' | ', $rowParts)); + } + } +} diff --git a/app/plugins/Transmogrify/src/Lib/Util/OrgIdentitiesHealth.php b/app/plugins/Transmogrify/src/Lib/Util/OrgIdentitiesHealth.php new file mode 100644 index 000000000..b585c071f --- /dev/null +++ b/app/plugins/Transmogrify/src/Lib/Util/OrgIdentitiesHealth.php @@ -0,0 +1,141 @@ +out('Running Org Identities health check...'); + $sql = RawSqlQueries::ORGIDENTITIES_HEALTH_SQL_QUERY; + + try { + $rows = $inconn->fetchAllAssociative($sql); + } catch (\Throwable $e) { + $io->err('Org Identities health check failed: ' . $e->getMessage()); + return; + } + + if (empty($rows)) { + $io->out('No results.'); + return; + } + + // Detect available columns + $first = $rows[0]; + $hasIncluded = array_key_exists('included_count', $first); + $hasExcluded = array_key_exists('excluded_count', $first); + $hasIndicator = array_key_exists('indicator', $first); + $hasCount = array_key_exists('count', $first); + + // Prepare headers based on detected columns + if ($hasIncluded || $hasExcluded) { + $headers = ['Reason', 'Included', 'Excluded']; + if ($hasIndicator) { + $headers[] = 'Indicator'; + } + } else { + // Fallback to simple reason + count (and indicator if present) + $headers = ['Reason', 'Count']; + if ($hasIndicator) { + $headers[] = 'Indicator'; + } + } + + // Compute column widths + $widths = array_fill(0, count($headers), 0); + $reasonIdx = 0; + $incIdx = array_search('Included', $headers, true); + $excIdx = array_search('Excluded', $headers, true); + $cntIdx = array_search('Count', $headers, true); + $indIdx = array_search('Indicator', $headers, true); + + // Initialize with header widths + foreach ($headers as $i => $h) { + $widths[$i] = max($widths[$i], mb_strlen($h)); + } + + // Measure data + foreach ($rows as $r) { + $reasonLen = mb_strlen((string)($r['reason'] ?? '')); + $widths[$reasonIdx] = max($widths[$reasonIdx], $reasonLen); + + if ($incIdx !== false) { + $widths[$incIdx] = max($widths[$incIdx], mb_strlen((string)($r['included_count'] ?? ''))); + } + if ($excIdx !== false) { + $widths[$excIdx] = max($widths[$excIdx], mb_strlen((string)($r['excluded_count'] ?? ''))); + } + if ($cntIdx !== false) { + $widths[$cntIdx] = max($widths[$cntIdx], mb_strlen((string)($r['count'] ?? ''))); + } + if ($indIdx !== false) { + $widths[$indIdx] = max($widths[$indIdx], mb_strlen((string)($r['indicator'] ?? ''))); + } + } + + // Helper to pad a cell + $pad = static function (string $s, int $w): string { + $len = mb_strlen($s); + if ($len >= $w) { + return $s; + } + return $s . str_repeat(' ', $w - $len); + }; + + // Print header + $lineParts = []; + foreach ($headers as $i => $h) { + $lineParts[] = $pad($h, $widths[$i]); + } + $io->out(implode(' | ', $lineParts)); + + // Print separator + $sepParts = array_map(static fn($w) => str_repeat('-', $w), $widths); + $io->out(implode('--+--', $sepParts)); + + // Print rows + foreach ($rows as $r) { + $rowParts = []; + $rowParts[] = $pad((string)($r['reason'] ?? ''), $widths[$reasonIdx]); + + if ($incIdx !== false) { + $rowParts[] = $pad((string)($r['included_count'] ?? ''), $widths[$incIdx]); + } + if ($excIdx !== false) { + $rowParts[] = $pad((string)($r['excluded_count'] ?? ''), $widths[$excIdx]); + } + if ($cntIdx !== false) { + $rowParts[] = $pad((string)($r['count'] ?? ''), $widths[$cntIdx]); + } + if ($indIdx !== false) { + $indRaw = (string)($r['indicator'] ?? ''); + $cell = $pad($indRaw, $widths[$indIdx]); + + // Colorize first visible char, leave padding spaces uncolored to preserve alignment + if ($indRaw === 'x' || $indRaw === '✓') { + $color = ($indRaw === 'x') ? "\033[31m" : "\033[32m"; // red for x, green for ✓ + $reset = "\033[0m"; + $first = mb_substr($cell, 0, 1); + $rest = mb_substr($cell, 1); + $cell = $color . $first . $reset . $rest; + } + + $rowParts[] = $cell; + } + + $io->out(implode(' | ', $rowParts)); + } + } +} diff --git a/app/plugins/Transmogrify/src/Lib/Util/RawSqlQueries.php b/app/plugins/Transmogrify/src/Lib/Util/RawSqlQueries.php new file mode 100644 index 000000000..d716c5736 --- /dev/null +++ b/app/plugins/Transmogrify/src/Lib/Util/RawSqlQueries.php @@ -0,0 +1,775 @@ +qualifyTableName($sourceTable); + $maxId = $inconn->fetchOne(self::buildSelectMaxId($qualifiedTableName)); + $maxId = ((int)($maxId ?? 0)) + 1; + + $qualifiedTableName = $outconn->qualifyTableName($targetTable); + $cmdPrinter->info("Resetting primary key sequence for $qualifiedTableName to $maxId"); + + // Strictly speaking we should use prepared statements, but we control the + // data here, and also we're executing a maintenance operation (so query + // optimization is less important). + $outsql = RawSqlQueries::buildSequenceReset($qualifiedTableName, $maxId, $outconn->isMySQL()); + try { + $outconn->executeQuery($outsql); + } catch (\Exception $e) { + return false; + } + + return true; + } + + /** + * Return SQL used to select COUs from inbound database. + * + * @since COmanage Registry v5.0.0 + * @param string $tableName Name of the SQL table + * @param bool $isMySQL Whether the database is MySQL + * @return string SQL string to select rows from inbound database + */ + public static function couSqlSelect(string $tableName, bool $isMySQL): string { + if($isMySQL) { + $sqlTemplate = RawSqlQueries::COU_SQL_SELECT_TEMPLATE_MYSQL; + } else { + $sqlTemplate = RawSqlQueries::COU_SQL_SELECT_TEMPLATE_POSTGRESQL; + } + + return str_replace('{table}', $tableName, $sqlTemplate); + } + + /** + * Select history records that are "current" (no changelog link) and whose + * org_identity_id is either NULL or refers to an included Org Identity + * (has a current link to a non-null co_person_id). + * + * @param string $tableName + * @param bool $isMySQL Unused here; kept for consistent signature + * @return string + */ + public static function historyRecordsSqlSelect(string $tableName, bool $isMySQL): string { + return str_replace('{table}', $tableName, RawSqlQueries::HISTORY_RECORDS_SQL_SELECT); + } + + /** + * Select job history records that are "current" (no changelog link) and whose + * org_identity_id is either NULL or refers to an included Org Identity + * (has a current link to a non-null co_person_id). + * + * @param string $tableName + * @param bool $isMySQL Unused here; kept for consistent signature + * @return string + */ + public static function jobHistoryRecordsSqlSelect(string $tableName, bool $isMySQL): string { + return str_replace('{table}', $tableName, RawSqlQueries::HISTORY_RECORDS_SQL_SELECT); + } + + /** + * Return SQL used to select COUs from inbound database. + * + * @since COmanage Registry v5.0.0 + * @param string $tableName Name of the SQL table + * @param bool $isMySQL Whether the database is MySQL + * @return string SQL string to select rows from inbound database + */ + public static function roleSqlSelect(string $tableName, bool $isMySQL): string { + return RawSqlQueries::ROLE_SQL_SELECT; + } + + + /** + * Builds SQL query to select Multiple Value Element Attributes (MVEAs) for valid org identities + * + * @param string $tableName Name of the database table containing MVEAs + * @param bool $isMySQL Whether the target database is MySQL (true) or PostgreSQL (false) + * @return string SQL query string to select MVEA rows that are linked to valid org identities + * @since COmanage Registry v5.2.0 + */ + public static function mveaSqlSelect(string $tableName, bool $isMySQL, array $fkColumns = []): string { + // Defaults to co_person_id and org_identity_id + if (empty($fkColumns)) { + $fkColumns = ['co_person_id', 'org_identity_id']; + } + + // XXX Unsupported FKs for now (until their models are implemented) + // co_department_id, co_provisioning_target_id, organization_id + $unsupportedFks = [ + 'co_department_id', + 'co_provisioning_target_id', + 'organization_id' + ]; + + // In full mode, treat all as supported; otherwise split into supported/unsupported + if (false /* $fullMode */) { + $supportedInUse = array_values($fkColumns); + $unsupportedInUse = []; + } else { + // Split provided FKs into supported (for OR non-null) and unsupported (must be NULL) + $supportedInUse = array_values(array_diff($fkColumns, $unsupportedFks)); + $unsupportedInUse = array_values(array_intersect($fkColumns, $unsupportedFks)); + } + + // Require at least one SUPPORTED FK is not NULL (unsupported FKs are excluded from this OR) + $nonnullClauses = array_map( + fn(string $c) => 'n.' . $c . ' IS NOT NULL', + $supportedInUse + ); + // Keep SQL valid even if no supported FKs present + $nonnullAny = empty($nonnullClauses) ? '1=1' : '(' . implode(' OR ', $nonnullClauses) . ')'; + + // Unsupported FKs (that are present) must be NULL (AND clause) + $unsupportedNullClause = ''; + if (!empty($unsupportedInUse)) { + $unsupportedNullClause = 'AND ' . implode( + ' AND ', + array_map(fn(string $c) => "n.$c IS NULL", $unsupportedInUse) + ); + } + + // If org_identity_id is one of the FKs, apply the org identity validity EXISTS + $orgIdCheck = ''; + if (in_array('org_identity_id', $fkColumns, true)) { + // If we decide to enable the soft delete we need to take into account + // the type of database + // - (p.deleted IS NULL OR p.deleted = false) on PostgreSQL + // - (p.deleted IS NULL OR p.deleted = 0) on MySQL + $orgIdCheck = <<plugin( + 'Transmogrify', + ['path' => '/transmogrify'], + function (RouteBuilder $builder) { + // Add custom routes here + + $builder->fallbacks(); + } + ); + parent::routes($routes); + } + + /** + * Add middleware for the plugin. + * + * @param \Cake\Http\MiddlewareQueue $middlewareQueue The middleware queue to update. + * @return \Cake\Http\MiddlewareQueue + */ + public function middleware(MiddlewareQueue $middlewareQueue): MiddlewareQueue + { + // Add your middlewares here + // remove this method hook if you don't need it + + return $middlewareQueue; + } + + /** + * Add commands for the plugin. + * + * @param \Cake\Console\CommandCollection $commands The command collection to update. + * @return \Cake\Console\CommandCollection + */ + public function console(CommandCollection $commands): CommandCollection + { + $commands->add('transmogrify', TransmogrifyCommand::class); + $commands->add('transmogrify source-to-target', TransmogrifySourceToTargetCommand::class); + + return parent::console($commands); + } + + /** + * Register application container services. + * + * @param \Cake\Core\ContainerInterface $container The Container to update. + * @return void + * @link https://book.cakephp.org/5/en/development/dependency-injection.html#dependency-injection + */ + public function services(ContainerInterface $container): void + { + // Add your services here + // remove this method hook if you don't need it + $container->add(TransmogrifyCommand::class) + ->addArguments([DbInfoService::class, ConfigLoaderService::class]); + $container->add(TransmogrifySourceToTargetCommand::class) + ->addArgument(ConfigLoaderService::class); + } +} diff --git a/app/plugins/Transmogrify/webroot/.gitkeep b/app/plugins/Transmogrify/webroot/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/app/src/Application.php b/app/src/Application.php index fbed5ede5..ae0b8a6dd 100644 --- a/app/src/Application.php +++ b/app/src/Application.php @@ -16,7 +16,10 @@ */ namespace App; +use App\Service\ConfigLoaderService; +use App\Service\DbInfoService; use Cake\Core\Configure; +use Cake\Core\ContainerInterface; use Cake\Core\Exception\MissingPluginException; use Cake\Error\Middleware\ErrorHandlerMiddleware; use Cake\Http\BaseApplication; @@ -66,6 +69,12 @@ public function bootstrap(): void foreach($activePlugins as $p) { $this->addPlugin($p->plugin); } + + // The Transmogrify plugin is required for all applications + // It is a special use case. + if (!\Cake\Core\Plugin::isLoaded('Transmogrify')) { + $this->addPlugin('Transmogrify'); + } } catch(\Cake\Database\Exception\DatabaseException $e) { // Most likely we are performing the initial database setup and @@ -124,4 +133,20 @@ protected function bootstrapCli(): void // Load more plugins here } + + + /** + * Register application container services. + * + * @param \Cake\Core\ContainerInterface $container The Container to update + * @return void + */ + public function services(ContainerInterface $container): void + { + parent::services($container); + + // Register services so the container can resolve them (constructor autowiring) + $container->add(DbInfoService::class); + $container->add(ConfigLoaderService::class); + } } diff --git a/app/src/Command/TransmogrifyCommand.php b/app/src/Command/TransmogrifyCommand.php deleted file mode 100644 index a3d24f886..000000000 --- a/app/src/Command/TransmogrifyCommand.php +++ /dev/null @@ -1,1388 +0,0 @@ - [ - 'source' => 'cm_cos', - 'displayField' => 'name', - 'addChangelog' => true, - // We don't really need status, but we need something cached for co_settings - 'cache' => [ 'status' ] - ], - 'types' => [ - 'source' => 'cm_co_extended_types', - 'displayField' => 'display_name', - 'postTable' => 'insertPronounTypes', - 'fieldMap' => [ - 'attribute' => '&map_extended_type', - 'name' => 'value', - // For some reason, cm_co_extended_types never had created/modified metadata - 'created' => '&map_now', - 'modified' => '&map_now' - ], - 'cache' => [ [ 'co_id', 'attribute', 'value' ] ] - ], - 'co_settings' => [ - 'source' => 'cm_co_settings', - 'displayField' => 'co_id', - 'addChangelog' => true, - 'booleans' => [], - 'postTable' => 'insertDefaultSettings', - 'cache' => [ 'co_id' ], - 'fieldMap' => [ - 'global_search_limit' => 'search_global_limit', - 'required_fields_addr' => 'required_fields_address', - 'permitted_fields_telephone_number' => '&populate_co_settings_phone', - // XXX CFM-80 these fields are not yet migrated - // be sure to add appropriate fields to 'booleans' - 'enable_nsf_demo' => null, // CFM-123 - 'disable_expiration' => null, - 'disable_ois_sync' => null, - 'group_validity_sync_window' => null, - 'garbage_collection_interval' => null, - 'enable_normalization' => null, - 'enable_empty_cou' => null, - 'invitation_validity' => null, - 't_and_c_login_mode' => null, - 'sponsor_eligibility' => null, - 'sponsor_co_group_id' => null, - 'theme_stacking' => null, - 'default_co_pipeline_id' => null, // XXX was this ever used? - 'elect_strategy_primary_name' => null, - 'co_dashboard_id' => null, - 'co_theme_id' => null, - 'person_picker_email_type' => null, - 'person_picker_identifier_type' => null, - 'person_picker_display_types' => null, - // No longer supported in PE, see CFM-316 - 'group_create_admin_only' => null - ] - ], - 'authentication_events' => [ - 'source' => 'cm_authentication_events', - 'displayField' => 'authenticated_identifier' - ], - 'api_users' => [ - 'source' => 'cm_api_users', - 'displayField' => 'username', - 'booleans' => [ 'privileged' ], - 'fieldMap' => [ - 'password' => 'api_key' - ] - ], - 'cous' => [ - 'source' => 'cm_cous', - 'displayField' => 'name', - 'sqlSelect' => 'couSqlSelect' - ], - //'dashboards' => [ 'source' => 'cm_co_dashboards' ] - 'people' => [ - 'source' => 'cm_co_people', - 'displayField' => 'id', - 'cache' => [ 'co_id' ], - 'fieldMap' => [ - // Rename the changelog key - 'co_person_id' => 'person_id' - ] - ], - 'person_roles' => [ - 'source' => 'cm_co_person_roles', - 'sqlSelect' => 'roleSqlSelect', - 'displayField' => 'id', - // We don't currently need status specifically, just that the role exists - 'cache' => [ 'status' ], - 'fieldMap' => [ - 'co_person_id' => 'person_id', - // Rename the changelog key - 'co_person_role_id' => 'person_role_id', - // We need to map affiliation_type_id before we null out affiliation - 'affiliation_type_id' => '&map_affiliation_type', - 'affiliation' => null, - 'manager_co_person_id' => 'manager_person_id', - 'sponsor_co_person_id' => 'sponsor_person_id', - 'o' => 'organization', - 'ou' => 'department', -// XXX temporary until tables are migrated - 'source_org_identity_id' => null - ] - ], - 'external_identities' => [ - 'source' => 'cm_org_identities', - 'displayField' => 'id', - 'fieldMap' => [ - 'co_id' => null, - 'person_id' => '&map_org_identity_co_person_id', - // Rename the changelog key - 'org_identity_id' => 'external_identity_id', - // These fields are migrated to external_identity_roles by split_external_identity() - 'title' => null, - 'o' => null, - 'ou' => null, - 'affiliation' => null, - 'manager_identifier' => null, - 'sponsor_identifier' => null, - 'valid_from' => null, - 'valid_through' => null - ], - 'postRow' => 'split_external_identity', - 'cache' => [ 'person_id' ] - ], - 'groups' => [ - 'source' => 'cm_co_groups', - 'displayField' => 'name', - 'cache' => [ 'co_id', 'owners_group_id' ], - 'booleans' => [ 'nesting_mode_all', 'open' ], - 'fieldMap' => [ - // auto is implied by group_type - 'auto' => null, - // Rename the changelog key - 'co_group_id' => 'group_id', - // Make sure group_type is populated if not already set - 'group_type' => '?S' - ], - 'postTable' => 'createOwnersGroups' - ], - 'group_nestings' => [ - 'source' => 'cm_co_group_nestings', - 'displayField' => 'id', - 'booleans' => [ 'negate' ], - 'fieldMap' => [ - 'co_group_id' => 'group_id', - 'target_co_group_id' => 'target_group_id', - // Rename the changelog key - 'co_group_nesting_id' => 'group_nesting_id' - ] - ], - 'group_members' => [ - 'source' => 'cm_co_group_members', - 'displayField' => 'id', - 'booleans' => [ 'member', 'owner' ], - 'fieldMap' => [ - 'co_group_id' => 'group_id', - 'co_person_id' => 'person_id', - 'member' => null, - 'owner' => null, - 'co_group_nesting_id' => 'group_nesting_id', - // Rename the changelog key - 'co_group_member_id' => 'group_member_id', - // Temporary until implemented - 'source_org_identity_id' => null - ], - 'preRow' => 'check_group_membership' - ], - 'names' => [ - 'source' => 'cm_names', - 'displayField' => 'id', - 'booleans' => [ 'primary_name' ], - 'fieldMap' => [ - 'co_person_id' => 'person_id', - 'org_identity_id' => 'external_identity_id', - // We need to map type_id before we null out type - 'type_id' => '&map_name_type', - 'type' => null - ] - ], - 'ad_hoc_attributes' => [ - 'source' => 'cm_ad_hoc_attributes', - 'displayField' => 'id', - 'fieldMap' => [ - 'co_person_role_id' => 'person_role_id', - 'org_identity_id' => 'external_identity_id', -// XXX temporary until tables are migrated - 'co_department_id' => null, - 'organization_id' => null - ], - 'postTable' => 'processExtendedAttributes' - ], - 'addresses' => [ - 'source' => 'cm_addresses', - 'displayField' => 'id', - 'fieldMap' => [ - 'co_person_role_id' => 'person_role_id', - 'org_identity_id' => 'external_identity_id', - 'type_id' => '&map_address_type', - 'type' => null, -// XXX temporary until tables are migrated - 'co_department_id' => null, - 'organization_id' => null - ] - ], - 'email_addresses' => [ - 'source' => 'cm_email_addresses', - 'displayField' => 'id', - 'booleans' => [ 'verified' ], - 'fieldMap' => [ - 'co_person_id' => 'person_id', - 'org_identity_id' => 'external_identity_id', - 'type_id' => '&map_email_type', - 'type' => null, -// XXX temporary until tables are migrated - 'co_department_id' => null, - 'organization_id' => null - ] - ], - 'identifiers' => [ - 'source' => 'cm_identifiers', - 'displayField' => 'id', - 'booleans' => [ 'login' ], - 'fieldMap' => [ - 'co_group_id' => 'group_id', - 'co_person_id' => 'person_id', - 'org_identity_id' => 'external_identity_id', - 'type_id' => '&map_identifier_type', - 'type' => null, -// XXX temporary until tables are migrated - 'co_department_id' => null, - 'co_provisioning_target_id' => null, - 'organization_id' => null - ], - 'preRow' => 'map_login_identifiers' - ], - 'telephone_numbers' => [ - 'source' => 'cm_telephone_numbers', - 'displayField' => 'id', - 'fieldMap' => [ - 'co_person_role_id' => 'person_role_id', - 'org_identity_id' => 'external_identity_id', - 'type_id' => '&map_telephone_type', - 'type' => null, -// XXX temporary until tables are migrated - 'co_department_id' => null, - 'organization_id' => null - ] - ], - 'urls' => [ - 'source' => 'cm_urls', - 'displayField' => 'id', - 'fieldMap' => [ - 'co_person_id' => 'person_id', - 'org_identity_id' => 'external_identity_id', - 'type_id' => '&map_url_type', - 'type' => null, -// XXX temporary until tables are migrated - 'co_department_id' => null, - 'organization_id' => null - ] - ], - 'history_records' => [ - 'source' => 'cm_history_records', - 'displayField' => 'id', - 'fieldMap' => [ - 'actor_co_person_id' => 'actor_person_id', - 'co_person_id' => 'person_id', - 'co_person_role_id' => 'person_role_id', - 'co_group_id' => 'group_id', - 'org_identity_id' => 'external_identity_id', -// XXX temporary until tables are migrated - 'co_email_list_id' => null, - 'co_service_id' => null - ] - ], - 'jobs' => [ - 'source' => 'cm_co_jobs', - 'displayField' => 'id', - 'fieldMap' => [ - 'job_type' => 'plugin', - 'job_mode' => null, - 'queue_time' => 'register_time', - 'complete_time' => 'finish_time', - 'job_type_fk' => null, - 'job_params' => 'parameters', - 'requeued_from_co_job_id' => 'requeued_from_job_id', - // XXX CFM-246 not yet supported - 'max_retry' => null, - 'max_retry_count' => null - ], - 'preRow' => 'filterJobs' - ], - 'job_history_records' => [ - 'source' => 'cm_co_job_history_records', - 'displayField' => 'id', - 'fieldMap' => [ - 'co_job_id' => 'job_id', - 'co_person_id' => 'person_id', - 'org_identity_id' => 'external_identity_id' - ] - ] - ]; - - // Table specific field mapping cache - protected $cache = []; - - // Make some objects more easily accessible - protected $inconn = null; - protected $outconn = null; - - // Shell arguments, for easier access - protected $args = null; - protected $io = null; - - /** - * Build an Option Parser. - * - * @since COmanage Registry v5.0.0 - * @param ConsoleOptionParser $parser ConsoleOptionParser - * @return ConsoleOptionParser ConsoleOptionParser - */ - - protected function buildOptionParser(ConsoleOptionParser $parser): ConsoleOptionParser { - $parser->addOption('login-identifier-copy', [ - 'help' => __d('command', 'tm.login-identifier-copy'), - 'boolean' => true - ]); - - $parser->addOption('login-identifier-type', [ - 'help' => __d('command', 'tm.login-identifier-type') - ]); - - $parser->setEpilog(__d('command', 'tm.epilog')); - - return $parser; - } - - /** - * Cache results as configured for the specified table. - * - * @since COmanage Registry v5.0.0 - * @param string $table Table to cache - * @param array $row Row of table data - */ - - protected function cacheResults(string $table, array $row) { - if(!empty($this->tables[$table]['cache'])) { - // Cache the requested fields. For now, at least, we key on row ID only. - foreach($this->tables[$table]['cache'] as $field) { - if(is_array($field)) { - // This is a list of fields, create a composite key that point to the row ID - - $label = ""; - $key = ""; - - foreach($field as $subfield) { - // eg: co_id+attribute+value+ - $label .= $subfield . "+"; - - // eg: 2+Identifier.type+eppn+ - $key .= $row[$subfield] . "+"; - } - - $this->cache[$table][$label][$key] = $row['id']; - } else { - // If the row has the field then map id to the requested field. - if(array_key_exists($field, $row)) { - $this->cache[$table]['id'][ $row['id'] ][$field] = $row[$field]; - } - } - } - } - } - - /** - * Check if a group membership is actually asserted, and reassign ownerships. - * - * @since COmanage Registry v5.0.0 - * @param array $origRow Row of table data (original data) - * @param array $row Row of table data (post fixes) - * @throws InvalidArgumentException - */ - - protected function check_group_membership(array $origRow, array $row) { - // We need to handle the various member+owner scenarios, but basically - // (1) If 'owner' is set, manually create a Group Membership in the appropriate - // Owners Group (we need to be called via preRow to do this) - // (2) If 'member' is NOT set, throw an exception so we don't create - // in invalid membership - // (3) Otherwise just return so the Membership gets created - - if($origRow['owner'] && !$origRow['deleted'] && !$origRow['co_group_member_id']) { - // Create a membership in the appropriate owners group, but not - // on changelog entries - - if(!empty($this->cache['groups']['id'][ $origRow['co_group_id'] ]['owners_group_id'])) { - $ownerRow = [ - 'group_id' => $this->cache['groups']['id'][ $origRow['co_group_id'] ]['owners_group_id'], - 'person_id' => $origRow['co_person_id'], - 'created' => $origRow['created'], - 'modified' => $origRow['modified'], - 'group_member_id' => null, - 'revision' => 0, - 'deleted' => 'f', - 'actor_identifier' => $origRow['actor_identifier'] - ]; - - $tableName = 'group_members'; - $qualifiedTableName = $this->outconn->qualifyTableName($tableName); - $this->outconn->insert($qualifiedTableName, $ownerRow); - } else { - $this->io->error("Could not find owners group for CoGroupMember " . $origRow['id']); - } - } - - if(!$row['member'] && !$row['owner']) { - throw new \InvalidArgumentException('member not set on GroupMember'); - } - } - - /** - * Return SQL used to select COUs from inbound database. - * - * @since COmanage Registry v5.0.0 - * @param string $tableName Name of the SQL table - * @return string SQL string to select rows from inbound database - */ - - protected function couSqlSelect(string $tableName): string { - if($this->inconn->isMySQL()) { - $sqlTemplate = TransmogrifyUtilities::COU_SQL_SELECT_TEMPLATE_MYSQL; - } else { - $sqlTemplate = TransmogrifyUtilities::COU_SQL_SELECT_TEMPLATE_POSTGRESQL; - } - - $sql = str_replace('{table}', $tableName, $sqlTemplate); - - return $sql; - } - - /** - * Create an Owners Group for an existing Group. - * - * @since COmanage Registry v5.0.0 - * @param array $origRow Row of table data (original data) - * @param array $row Row of table data (post fixes) - */ - - protected function createOwnersGroups() { - // Pull all Groups and create Owners Group for them. Deployments generally - // don't have so many Groups that we need PaginatedSqlIterator, but we'll - // use it here anyway just in case. - - // By doing this once for the table we avoid having to sort through - // changelog metadata to figure out which rows to actually create owners - // groups for. - - $Groups = TableRegistry::getTableLocator()->get('Groups'); - - $iterator = new PaginatedSqlIterator($Groups, []); - - foreach($iterator as $k => $group) { - try { - // Because PaginatedSqlIterator will pick up new Groups as we create them, - // we need to check for any Owners groups (that we just created) and skip them. - if(!$group->isOwners()) { - $ownersGid = $Groups->createOwnersGroup($group); - - // We need to manually populate the cache - $this->cache['groups']['id'][$group->id]['owners_group_id'] = $ownersGid; - } - } - catch(\Exception $e) { - $this->io->error("Failed to create owners group for " - . $group->name . " (" . $group->id . "): " - . $e->getMessage()); - } - } - } - - /** - * Execute the Transmogrify Command. - * - * @since COmanage Registry v5.0.0 - * @param Arguments $args Command Arguments - * @param ConsoleIo $io Console IO - */ - - public function execute(Arguments $args, ConsoleIo $io) { - $this->args = $args; - $this->io = $io; - - // Load data from the inbound "transmogrify" database to a newly created - // (and empty) v5 database. The schema should already be applied to the - // new database. - - // First, open connections to both old and new databases. - $this->inconn = DBALConnection::factory($io, 'transmogrify'); - $this->outconn = DBALConnection::factory($io, 'default'); - - // We accept a list of table names, mostly for testing purposes - $atables = $args->getArguments(); - - // Register the current version for future upgrade purposes - - $metaTable = $this->getTableLocator()->get('Meta'); - $metaTable->setUpgradeVersion(); - - foreach(array_keys($this->tables) as $t) { - // If the command line args include a list of tables skip this table - // if it is not in that list. - if(!empty($atables) && !in_array($t, $atables)) - continue; - - $io->info(message: sprintf("Transmogrifying table %s(%s)", Inflector::classify($t), $t)); - - // Find the maximum id in the inbound table and reset the sequence for the outbound - // table to be that value plus one so that as we process the rows any entirely new rows - // inserted (entirely new rows have no existing id and will be given the id by the - // auto sequence) do not have a primary key id that conflicts with an existing row - // from the inbound table. - $qualifiedTableName = $this->inconn->qualifyTableName($this->tables[$t]['source']); - $maxId = $this->inconn->fetchOne('SELECT MAX(id) FROM ' . $qualifiedTableName); - $maxId++; - - $qualifiedTableName = $this->outconn->qualifyTableName($t); - $this->io->info("Resetting sequence for $qualifiedTableName to $maxId"); - - // Strictly speaking we should use prepared statements, but we control the - // data here, and also we're executing a maintenance operation (so query - // optimization is less important). - if($this->outconn->isMySQL()) { - $outsql = "ALTER TABLE $qualifiedTableName AUTO_INCREMENT = " . $maxId; - } else { - $outsql = "ALTER SEQUENCE " . $qualifiedTableName . "_id_seq RESTART WITH " . $maxId; - } - $this->outconn->executeQuery($outsql); - - // Run any preprocessing functions for the table. - if(!empty($this->tables[$t]['preTable'])) { - $p = $this->tables[$t]['preTable']; - - $this->$p(); - } - - // Check if the outbound table contains any rows and skip the table if it does. - $Model = $this->getTableLocator()->get($t); - if($Model->find()->count() > 0) { - $io->warning("Skipping Transmogrification. Table (" . $t . ") is not empty. Drop the database (or truncate) and start over."); - continue; - } - - // Count the rows in the inbound table so that we can log the percentage processed. - $qualifiedTableName = $this->inconn->qualifyTableName($this->tables[$t]['source']); - $count = $this->inconn->fetchOne("SELECT COUNT(*) FROM " . $qualifiedTableName); - - // Select all the rows from the inbound table. - if(!empty($this->tables[$t]['sqlSelect'])) { - $p = $this->tables[$t]['sqlSelect']; - $insql = $this->$p($qualifiedTableName); - } else { - $insql = "SELECT * FROM " . $qualifiedTableName . " ORDER BY id ASC"; - } - $stmt = $this->inconn->executeQuery($insql); - - $tally = 0; - $warns = 0; - $err = 0; - - // Loop over each row from the inbound table. - while($row = $stmt->fetchAssociative()) { - if(!empty($row[ $this->tables[$t]['displayField'] ])) { - $io->verbose("$t " . $row[ $this->tables[$t]['displayField'] ]); - } - - try { - // Make a copy of the original data for any postprocessing followups. - $origRow = $row; - - // Run any preprocessing functions for the row. - - if(!empty($this->tables[$t]['preRow'])) { - $p = $this->tables[$t]['preRow']; - - $this->$p($origRow, $row); - } - - // Do this before fixBooleans since we'll insert some - $this->fixChangelog($t, $row, isset($this->tables[$t]['addChangelog']) && $this->tables[$t]['addChangelog']); - - $this->fixBooleans($t, $row); - - $this->mapFields($t, $row); - - $qualifiedTableName = $this->outconn->qualifyTableName($t); - $this->outconn->insert($qualifiedTableName, $row); - - $this->cacheResults($t, $row); - - // Run any post processing functions for the row. - - if(!empty($this->tables[$t]['postRow'])) { - $p = $this->tables[$t]['postRow']; - - $this->$p($origRow, $row); - } - } - catch(ForeignKeyConstraintViolationException $e) { - // A foreign key associated with this record did not load, so we can't - // load this record. This can happen, eg, because the source_field_id - // did not load, perhaps because it was associated with an Org Identity - // not linked to a CO Person that was not migrated. - $warns++; - $io->warning("Skipping $t record " . $row['id'] . " due to invalid foreign key: " . $e->getMessage()); - } - catch(\InvalidArgumentException $e) { - // If we can't find a value for mapping we skip the record - // (ie: mapFields basically requires a successful mapping) - $warns++; - $io->warning("Skipping $t record " . $row['id'] . ": " . $e->getMessage()); - } - catch(\Exception $e) { - $err++; - $io->error("$t record " . $row['id'] . ": " . $e->getMessage()); - } - - $tally++; - - if(!$this->args->getOption('quiet') && !$this->args->getOption('verbose')) { - // We don't output the progress bar for quiet for obvious reasons, - // or for verbose so we don't interfere with the extra output - $this->cliLogPercentage($tally, $count); - } - } - - // Log warning and error count. - $io->out("(Warnings: " . $warns . ")"); - $io->out("(Errors: " . $err . ")"); - - - // Run any post processing functions for the table. - if(!empty($this->tables[$t]['postTable'])) { - $p = $this->tables[$t]['postTable']; - - $this->$p(); - } - } - } - - /** - * Filter Jobs. - * - * @since COmanage Registry v5.0.0 - * @param array $origRow Row of table data (original data) - * @param array $row Row of table data (post fixes) - * @throws InvalidArgumentException - */ - - protected function filterJobs(array $origRow, array $row) { - // We don't update any of the attributes, but for rows with unsupported data - // we throw an exception so they don't transmogrify. - - if($row['status'] == 'GO' || $row['status'] == 'Q') { - throw new \InvalidArgumentException("Job is Queued or In Progress"); - } - - if($row['job_type'] == 'EX' || $row['job_type'] == 'OS') { - throw new \InvalidArgumentException("Legacy Job types cannot be transmogrified"); - } - } - - /** - * Find the CO for a row of table data, based on a foreign key. - * - * @since COmanage Registry v5.0.0 - * @param array $row Row of table data - * @return int CO ID - * @throws InvalidArgumentException - */ - - protected function findCoId(array $row) { - // By the time we're called, we should have transmogrified the Org Identity - // and CO Person data, so we can just walk the caches - - if(!empty($row['person_id'])) { - if(isset($this->cache['people']['id'][ $row['person_id'] ]['co_id'])) { - return $this->cache['people']['id'][ $row['person_id'] ]['co_id']; - } - } elseif(!empty($row['external_identity_id'])) { - // Map the OrgIdentity to a CO Person, then to the CO - if(!empty($this->cache['external_identities']['id'][ $row['external_identity_id'] ]['person_id'])) { - $personId = $this->cache['external_identities']['id'][ $row['external_identity_id'] ]['person_id']; - - if(isset($this->cache['people']['id'][ $personId ]['co_id'])) { - return $this->cache['people']['id'][ $personId ]['co_id']; - } - } - } elseif(!empty($row['group_id'])) { - if(isset($this->cache['groups']['id'][ $row['group_id'] ]['co_id'])) { - return $this->cache['groups']['id'][ $row['group_id'] ]['co_id']; - } - } - // We also support being called using the old keys for use in the preRow context - elseif(!empty($row['org_identity_id'])) { - // Map the OrgIdentity to a CO Person, then to the CO - if(!empty($this->cache['external_identities']['id'][ $row['org_identity_id'] ]['person_id'])) { - $personId = $this->cache['external_identities']['id'][ $row['org_identity_id'] ]['person_id']; - - if(isset($this->cache['people']['id'][ $personId ]['co_id'])) { - return $this->cache['people']['id'][ $personId ]['co_id']; - } - } - } elseif(!empty($row['co_person_id'])) { - if(isset($this->cache['people']['id'][ $row['co_person_id'] ]['co_id'])) { - return $this->cache['people']['id'][ $row['co_person_id'] ]['co_id']; - } - } - - throw new \InvalidArgumentException('CO not found for record'); - } - - /** - * Translate booleans to string literals to work around DBAL Postgres boolean handling. - * - * @since COmanage Registry v5.0.0 - * @param string $table Table Name - * @param array $row Row of attributes, fixed in place - */ - - protected function fixBooleans(string $table, array &$row) { - $attrs = ['deleted']; - - // We could introspect this from the schema file... - if(!empty($this->tables[$table]['booleans'])) { - $attrs = array_merge($attrs, $this->tables[$table]['booleans']); - } - - foreach($attrs as $a) { - if(isset($row[$a]) && gettype($row[$a]) == 'boolean') { - // DBAL Postgres boolean handling seems to be somewhat buggy, see history in - // this issue: https://github.com/doctrine/dbal/issues/1847 - // We need to (more generically than this hack) convert from boolean to char - // to avoid errors on insert - if($this->outconn->isMySQL()) { - $row[$a] = ($row[$a] ? '1' : '0'); - } else { - $row[$a] = ($row[$a] ? 't' : 'f'); - } - } - } - } - - /** - * Populate empty Changelog data from legacy records - * - * @since COmanage Registry v5.0.0 - * @param string $table Table Name - * @param array $row Row of attributes, fixed in place - * @param bool $force If true, always create keys - */ - - protected function fixChangelog(string $table, array &$row, bool $force=false) { - if($force || (array_key_exists('deleted', $row) && is_null($row['deleted']))) { - $row['deleted'] = false; - } - - if($force || (array_key_exists('revision', $row) && is_null($row['revision']))) { - $row['revision'] = 0; - } - - if($force || (array_key_exists('actor_identifier', $row) && is_null($row['actor_identifier']))) { - $row['actor_identifier'] = 'Transmogrification'; - } - - // The parent FK should remain NULL since this is the original record. - /* - // If the table was renamed, we need to rename the changelog key as well. - // NOTE: We don't actually do this here because it creates issues with the - // order of field processing. Instead, each key must be renamed - // manually in the fieldMap. - // eg: cm_org_identities -> org_identity_id - $oldfk = Inflector::singularize(substr($this->tables[$table]['source'], 3)) . "_id"; - // eg: external_identities -> external_identity_id - $newfk = Inflector::singularize($table) . "_id"; - - if($oldfk != $newfk && array_key_exists($oldfk, $row)) { - $row[$newfk] = $row[$oldfk]; - unset($row[$oldfk]); - }*/ - } - - /** - * Insert default CO Settings. - * - * @since COmanage Registry v5.0.0 - */ - - protected function insertDefaultSettings() { - // Create a CoSetting for any CO that didn't previously have one. - - $createdSettings = []; - $createdCos = array_keys($this->cache['cos']['id']); - - foreach($this->cache['co_settings']['id'] as $co_setting_id => $cached) { - $createdSettings[] = $cached['co_id']; - } - - $emptySettings = array_values(array_diff($createdCos, $createdSettings)); - - if(!empty($emptySettings)) { - $CoSettings = TableRegistry::getTableLocator()->get('CoSettings'); - - foreach($emptySettings as $coId) { - // Insert a default row into CoSettings for this CO ID - try { - $CoSettings->addDefaults($coId); - } catch (\ConflictException $e) { - // skip - } - } - } - } - - /** - * Insert default Pronoun types. - * - * @since COmanage Registry v5.0.0 - */ - - protected function insertPronounTypes() { - // Since the Pronoun MVEA didn't exist in v4, we'll need to create the - // default types for all COs. - - $Types = TableRegistry::getTableLocator()->get('Types'); - - foreach(array_keys($this->cache['cos']['id']) as $coId) { - $Types->addDefault($coId, 'Pronouns.type'); - } - } - - /** - * Map fields that have been renamed from Registry Classic to Registry PE. - * - * @since COmanage Registry v5.0.0 - * @param string $table Table Name - * @param array $row Row of attributes, fixed in place - * @throws InvalidArgumentException - */ - - protected function mapFields(string $table, array &$row) { - // oldname => newname, or &newname, which is a function to call. - // Note functions can returns more than one mapping - $fields = []; - - if(!empty($this->tables[$table]['fieldMap'])) { - $fields = $this->tables[$table]['fieldMap']; - } - - foreach($fields as $oldname => $newname) { - if(!$newname) { - // This attribute doesn't map, so simply unset it - unset($row[$oldname]); - } elseif($newname[0] == '&') { - // This is a function to map the field, in which case we reuse the old name - $f = substr($newname, 1); - - // We always pass the entire row so the mapping function can implement - // whatever logic it needs - $row[$oldname] = $this->$f($row); - - if(!$row[$oldname]) { - throw new \InvalidArgumentException("Could not find value for $table $oldname"); - } - } elseif($newname[0] == '?') { - // This is a default value to populate if the current value is null - $v = substr($newname, 1); - - if($row[$oldname] === null) { - $row[$oldname] = $v; - } - } else { - // Copy the value to the new name, then unset the old name - $row[$newname] = $row[$oldname]; - unset($row[$oldname]); - } - } - } - - /** - * Map an address type string to a foreign key. - * - * @since COmanage Registry v5.0.0 - * @param array $row Row of table data - * @return int type_id - */ - - protected function map_address_type(array $row) { - return $this->map_type($row, 'Addresses.type', $this->findCoId($row)); - } - - /** - * Map an affiliation type string to a foreign key. - * - * @since COmanage Registry v5.0.0 - * @param array $row Row of table data - * @return int type_id - */ - - protected function map_affiliation_type(array $row) { - return $this->map_type($row, 'PersonRoles.affiliation_type', $this->findCoId($row), 'affiliation'); - } - - /** - * Map an email type string to a foreign key. - * - * @since COmanage Registry v5.0.0 - * @param array $row Row of table data - * @return int type_id - */ - - protected function map_email_type(array $row) { - return $this->map_type($row, 'EmailAddresses.type', $this->findCoId($row)); - } - - /** - * Map an Extended Type attribute name for model name changes. - * - * @since COmanage Registry v5.0.0 - * @param array $row Row of table data - * @return string Updated attribute name - */ - - protected function map_extended_type(array $row) { - switch($row['attribute']) { - case 'CoDepartment.type': - return 'Departments.type'; - case 'CoPersonRole.affiliation': - return 'PersonRoles.affiliation_type'; - } - - // For everything else, we need to pluralize the model name - $bits = explode('.', $row['attribute'], 2); - - return Inflector::pluralize($bits[0]) . "." . $bits[1]; - } - - /** - * Map login identifiers, in accordance with the configuration. - * - * @since COmanage Registry v5.0.0 - * @param array $origRow Row of table data (original data) - * @param array $row Row of table data (post fixes) - * @throws InvalidArgumentException - */ - - protected function map_login_identifiers(array $origRow, array $row) { - // There might be multiple reasons to copy the row, but we only want to - // copy it once. - $copyRow = false; - - if(!empty($origRow['org_identity_id'])) { - if($this->args->getOption('login-identifier-copy') - && $origRow['login']) { - $copyRow = true; - } - - // Note the argument here is the old v4 string (eg "eppn") and not the - // PE foreign key - if($this->args->getOption('login-identifier-type') - && $origRow['type'] == $this->args->getOption('login-identifier-type')) { - $copyRow = true; - } - - // Identifiers attached to External Identities do not have login flags in PE - $row['login'] = false; - } - - if($copyRow) { - // Find the Person ID associated with this External Identity ID - - if(!empty($this->cache['external_identities']['id'][ $origRow['org_identity_id'] ]['person_id'])) { - // Insert a new row attached to the Person, leave the original record - // (ie: $row) untouched - - $copiedRow = [ - 'person_id' => $this->map_org_identity_co_person_id(['id' => $origRow['org_identity_id']]), - 'identifier' => $origRow['identifier'], - 'type_id' => $this->map_identifier_type($origRow), - 'status' => $origRow['status'], - 'login' => true, - 'created' => $origRow['created'], - 'modified' => $origRow['modified'] - ]; - - // Set up changelog and fix booleans - $this->fixChangelog('identifiers', $copiedRow, true); - $this->fixBooleans('identifiers', $copiedRow); - - try { - $tableName = 'identifiers'; - $qualifiedTableName = $this->outconn->qualifyTableName($tableName); - $this->outconn->insert($qualifiedTableName, $copiedRow); - } catch (\Doctrine\DBAL\Exception\UniqueConstraintViolationException $e) { - $this->io->warning("record already exists: " . print_r($copiedRow, true)); - } - } - } - } - - /** - * Map an identifier type string to a foreign key. - * - * @since COmanage Registry v5.0.0 - * @param array $row Row of table data (ignored) - * @return int type_id - */ - - protected function map_identifier_type(array $row) { - return $this->map_type($row, 'Identifiers.type', $this->findCoId($row)); - } - - /** - * Map a name type string to a foreign key. - * - * @since COmanage Registry v5.0.0 - * @param array $row Row of table data (ignored) - * @return int type_id - */ - - protected function map_name_type(array $row) { - return $this->map_type($row, 'Names.type', $this->findCoId($row)); - } - - /** - * Return a timestamp equivalent to now. - * - * @since COmanage Registry v5.0.0 - * @param array $row Row of table data (ignored) - * @return string Timestamp - */ - - protected function map_now(array $row) { - if(empty($this->cache['now'])) { - $created = new \Datetime('now'); - $this->cache['now'] = $created->format('Y-m-d H:i:s'); - } - - return $this->cache['now']; - } - - /** - * Map an Org Identity ID to a CO Person ID - * - * @since COmanage Registry v5.0.0 - * @param array $row Row of Org Identity table data - * @return int CO Person ID - */ - - protected function map_org_identity_co_person_id(array $row) { - // PE eliminates OrgIdentityLink, so we need to map each Org Identity to - // a Person ID. This is a bit trickier than it sounds, since an Org Identity - // could have been relinked. - - // Before Transmogrification, we require that Org Identities are unpooled. - // (This is probably how most deployments are set up, but there may be some - // legacy deployments out there.) This ensures whatever CO Person the Org - // Identity currently maps to through CoOrgIdentityLink is in the same CO. - - // There may be multiple mappings if the Org Identity was relinked. Basically - // we're going to lose the multiple mappings, since we can only return one - // value here. (Ideally, we would inject multiple OrgIdentities into the new - // table, but this ends up being rather tricky, since we have to figure out - // what row id to assign, and for the moment we don't have a mechanism to - // do that.) Historical information remains available in history_records, - // and if the deployer keeps an archive of the old database. - - // To figure out which person_id to use, we pull the record with the - // highest revision number. Note we might be transmogrifying a deleted row, - // so we can't ignore deleted rows here. - - if(empty($this->cache['org_identities']['co_people'])) { - $this->io->info('Populating org identity map...'); - - // We pull deleted rows because we might be migrating deleted rows - $tableName = "cm_co_org_identity_links"; - $qualifiedTableName = $this->inconn->qualifyTableName($tableName); - $mapsql = "SELECT * FROM $qualifiedTableName"; - $stmt = $this->inconn->query($mapsql); - - while($r = $stmt->fetch()) { - if(!empty($r['org_identity_id'])) { - if(isset($this->cache['org_identities']['co_people'][ $r['org_identity_id'] ][ $r['revision'] ])) { - // If for some reason we already have a record, it's probably due to - // improper unpooling from a legacy deployment. We'll accept only the - // first record and throw warnings on the others. - - $this->io->warning("Found existing CO Person for Org Identity " . $r['org_identity_id'] . ", skipping"); - } else { - $this->cache['org_identities']['co_people'][ $r['org_identity_id'] ][ $r['revision'] ] = $r['co_person_id']; - } - } - } - } - - if(!empty($this->cache['org_identities']['co_people'][ $row['id'] ])) { - // Return the record with the highest revision number - $rev = max(array_keys($this->cache['org_identities']['co_people'][ $row['id'] ])); - - return $this->cache['org_identities']['co_people'][ $row['id'] ][$rev]; - } - - return null; - } - - /** - * Map a telephone type string to a foreign key. - * - * @since COmanage Registry v5.0.0 - * @param array $row Row of table data - * @return int type_id - */ - - protected function map_telephone_type(array $row) { - return $this->map_type($row, 'TelephoneNumbers.type', $this->findCoId($row)); - } - - /** - * Map a type string to a foreign key. - * - * @since COmanage Registry v5.0.0 - * @param array $row Row of table data - * @param string $type Type to map (types:attribute) - * @param int $coId CO ID - * @param string $attr Row column to use for type value - * @return int type_id - * @throws InvalidArgumentException - */ - - protected function map_type(array $row, string $type, $coId, string $attr="type") { - if(!$coId) { - throw new \InvalidArgumentException("CO ID not provided for $type " . $row['id']); - } - - $key = $coId . "+" . $type . "+" . $row[$attr] . "+"; - - if(empty($this->cache['types']['co_id+attribute+value+'][$key])) { - throw new \InvalidArgumentException("Type not found for " . $key); - } - - return $this->cache['types']['co_id+attribute+value+'][$key]; - } - - /** - * Map a URL type string to a foreign key. - * - * @since COmanage Registry v5.0.0 - * @param array $row Row of table data - * @return int type_id - */ - - protected function map_url_type(array $row) { - return $this->map_type($row, 'Urls.type', $this->findCoId($row)); - } - - /** - * Set a default value for CO Settings Permitted Telephone Number Fields. - * - * @since COmanage Registry v5.0.0 - * @param array $row Row of table data - * @return string Default value - */ - - protected function populate_co_settings_phone(array $row) { - return \App\Lib\Enum\PermittedTelephoneNumberFieldsEnum::CANE; - } - - /** - * Process Extended Attributes by converting them to Ad Hoc Attributes. - * - * @since COmanage Registry v5.0.0 - */ - - protected function processExtendedAttributes() { - // This is intended to run AFTER AdHocAttributes so that we don't stomp on - // the row identifiers. - - // First, pull the old Extended Attribute configuration. - $extendedAttrs = []; - - $tableName = "cm_co_extended_attributes"; - $qualifiedTableName = $this->inconn->qualifyTableName($tableName); - $insql = "SELECT * FROM $qualifiedTableName ORDER BY id ASC"; - $stmt = $this->inconn->query($insql); - - while($row = $stmt->fetch()) { - $extendedAttrs[ $row['co_id'] ][] = $row['name']; - } - - if(empty($extendedAttrs)) { - // No need to do anything further if no attributes are configured - return; - } - - foreach(array_keys($extendedAttrs) as $coId) { - $tableName = "cm_co" . $coId . "_person_extended_attributes"; - $qualifiedTableName = $this->inconn->qualifyTableName($tableName); - $insql = "SELECT * FROM $qualifiedTableName"; - $stmt = $this->inconn->query($insql); - - while($eaRow = $stmt->fetch()) { - // If we didn't transmogrify the parent row for some reason then trying - // to insert the ad_hoc_attributes will throw an error. - if(!empty($this->cache['person_roles']['id'][ $eaRow['co_person_role_id'] ])) { - foreach($extendedAttrs[$coId] as $ea) { - $adhocRow = [ - 'person_role_id' => $eaRow['co_person_role_id'], - 'tag' => $ea, - 'value' => $eaRow[$ea], - 'created' => $eaRow['created'], - 'modified' => $eaRow['modified'] - ]; - - // Extended Attributes were not changelog enabled - $this->fixChangelog('ad_hoc_attributes', $adhocRow, true); - $this->fixBooleans('ad_hoc_attributes', $adhocRow); - - try { - $tableName = 'ad_hoc_attributes'; - $qualifiedTableName = $this->outconn->qualifyTableName($tableName); - $this->outconn->insert($qualifiedTableName, $adhocRow); - } catch (\Doctrine\DBAL\Exception\UniqueConstraintViolationException $e) { - $this->io->warning("record already exists: " . print_r($adhocRow, true)); - } - } - } - } - } - } - - /** - * Return SQL used to select CO Person Roles from inbound database. - * - * @since COmanage Registry v5.0.0 - * @param string $tableName Name of the SQL table - * @return string SQL string to select rows from inbound database - */ - - protected function roleSqlSelect(string $tableName): string { - // Cast the affiliation value to 'member' when NULL or empty string. - define("ROLE_SQL_SELECT", << 'external_identity_id', - 'o' => 'organization', - 'ou' => 'department', - 'manager_identifier' => 'manager_identifier', - 'sponsor_identifier' => 'sponsor_identifier', - 'status' => 'status', - 'title' => 'title', - 'valid_from' => 'valid_from', - 'valid_through' => 'valid_through', - // Fix up changelog - 'org_identity_id' => 'external_identity_role_id', - 'revision' => 'revision', - 'deleted' => 'deleted', - 'actor_identifier' => 'actor_identifier', - 'created' => 'created', - 'modified' => 'modified' - ] as $oldKey => $newKey) { - $roleRow[$newKey] = $origRow[$oldKey]; - } - - // Affiliation requires special handling. We need to use the post-fixed $row - // because map_affiliation_type calls findCoId which uses the foreign key to - // lookup the CO ID in the cache, however by the time we've been called - // affiliation has been null'd out (since we're moving it to the role row). - // So shove it back in before calling map_affiliation_type. - if(!empty($origRow['affiliation'])) { - $row['affiliation'] = $origRow['affiliation']; - $roleRow['affiliation_type_id'] = $this->map_affiliation_type($row); - } - - // Fix up changelog - // Since we're creating a new row, we have to manually fix up booleans - $roleRow['deleted'] = ($roleRow['deleted'] ? 't' : 'f'); - - $tableName = 'external_identity_roles'; - $qualifiedTableName = $this->outconn->qualifyTableName($tableName); - $this->outconn->insert($qualifiedTableName, $roleRow); - } -} \ No newline at end of file diff --git a/app/src/Lib/Util/TransmogrifyUtilities.php b/app/src/Lib/Util/TransmogrifyUtilities.php deleted file mode 100644 index 65be538f8..000000000 --- a/app/src/Lib/Util/TransmogrifyUtilities.php +++ /dev/null @@ -1,179 +0,0 @@ -loadGeneric($path); + // Filter out documentation or comment keys (eg, keys starting with "__") + $cfg = array_filter($cfg, static function ($value, $key) { + return !(is_string($key) && str_starts_with($key, '__')); + }, ARRAY_FILTER_USE_BOTH); + return $cfg; + } + + /** + * Generic config loader that supports JSON (.json) and XML (.xml) files. + * Returns associative array representation of the root object. + * + * @param string $path Absolute or project-relative path + * @return array + */ + public function loadGeneric(string $path): array + { + // Resolve path if relative + if (!is_readable($path)) { + if (defined('ROOT')) { + $alt = ROOT . DIRECTORY_SEPARATOR . ltrim($path, DIRECTORY_SEPARATOR); + if (is_readable($alt)) { + $path = $alt; + } + } + } + if (!is_readable($path)) { + throw new \RuntimeException('Config not readable: ' . $path); + } + + $ext = strtolower(pathinfo($path, PATHINFO_EXTENSION)); + $raw = file_get_contents($path); + if ($raw === false) { + throw new \RuntimeException('Failed to read config: ' . $path); + } + + if ($ext === 'json') { + // Decode JSON and detect corruption with detailed error messages + $data = json_decode($raw, true); + if ($data === null && json_last_error() !== JSON_ERROR_NONE) { + $err = function_exists('json_last_error_msg') ? json_last_error_msg() : ('code ' . json_last_error()); + throw new \RuntimeException('Invalid JSON config: ' . $path . ' Details: ' . $err); + } + if (!is_array($data)) { + // Enforce that root must be an object/array for config purposes + throw new \RuntimeException('Invalid JSON config: ' . $path . ' Details: Expected root object or array'); + } + return $data; + } + + if ($ext === 'xml') { + // Use libxml internal error handling instead of deprecated @ suppression + $prev = libxml_use_internal_errors(true); + $xml = simplexml_load_string($raw, 'SimpleXMLElement', LIBXML_NONET | LIBXML_NOERROR | LIBXML_NOWARNING); + if ($xml === false) { + $errors = libxml_get_errors(); + libxml_clear_errors(); + libxml_use_internal_errors($prev); + $messages = array_map(static function ($e) { + return trim($e->message ?? '') . ' at line ' . ($e->line ?? ''); + }, $errors ?: []); + $detail = $messages ? (' Details: ' . implode(' | ', $messages)) : ''; + throw new \RuntimeException('Invalid XML config: ' . $path . $detail); + } + // Clear any accumulated libxml errors and restore previous setting + libxml_clear_errors(); + libxml_use_internal_errors($prev); + + $json = json_encode($xml); + $arr = json_decode($json, true); + if (!is_array($arr)) { + throw new \RuntimeException('Failed to convert XML to array: ' . $path); + } + return $arr; + } + + throw new \RuntimeException('Unsupported config extension (expected .json or .xml): ' . $path); + } +} diff --git a/app/src/Service/DbInfoService.php b/app/src/Service/DbInfoService.php new file mode 100644 index 000000000..bae30d07b --- /dev/null +++ b/app/src/Service/DbInfoService.php @@ -0,0 +1,184 @@ + 'transmogrify', 'target' => 'default' ]; + $role = $roleOrAlias; + $alias = $map[$roleOrAlias] ?? $roleOrAlias; + + $cfg = null; + try { + $cfg = ConnectionManager::getConfig($alias); + } catch (\Throwable $e) { + $cfg = null; + } + + return [ + 'alias' => $alias, + 'role' => ($alias === ($map['source'] ?? 'transmogrify')) ? 'source' : (($alias === ($map['target'] ?? 'default')) ? 'target' : $role), + 'configured' => $cfg !== null, + 'driver' => $cfg['driver'] ?? null, + 'host' => $cfg['host'] ?? ($cfg['hostname'] ?? null), + 'port' => $cfg['port'] ?? null, + 'database' => $cfg['database'] ?? ($cfg['dbname'] ?? null), + 'username' => $cfg['username'] ?? ($cfg['user'] ?? null), + 'password' => null, + 'dsn' => $cfg['url'] ?? null, + ]; + } + + /** + * Ping a connection alias and return status info. + * @param string $alias Cake connection alias + * @return array{ok:bool,server:?(string),error:?(string)} + */ + public function ping(string $alias): array + { + $status = [ 'ok' => false, 'error' => null, 'server' => null ]; + try { + $conn = DBALConnection::factory(connection: $alias); + if ($conn->isMySQL()) { + $ver = $conn->fetchOne('SELECT VERSION()'); + } else { + $ver = $conn->fetchOne('SHOW server_version'); + if (!$ver) { $ver = $conn->fetchOne('SELECT version()'); } + } + $status['ok'] = true; + $status['server'] = $ver; + } catch (\Throwable $e) { + $status['ok'] = false; + $status['error'] = $e->getMessage(); + } + return $status; + } + + /** + * Load schema information for a given alias. + * @param string $alias Cake connection alias + * @param string|null $tablesJsonPath Path to the tables.json file + * @return array{table_count:int,empty:bool,sample_tables:array,tables_compare:array,loaded_schema:?(string)} + */ + public function loadSchemaInfo(string $alias, ?string $tablesJsonPath = null): array + { + if ($tablesJsonPath === null) { + return []; + } + $conn = DBALConnection::factory(connection: $alias); + return $this->loadSchemaInfoFromConnection($conn, $tablesJsonPath); + } + + /** + * Exposed for reuse when a DBALConnection already exists. + * @param DBALConnection $conn The database connection + * @param string $tablesJsonPath Path to the tables.json file + * @return array Schema information array + * + * todo: The new version should render all the tables. And if i pass a the parameter transmogrify then + * it should render the ones that have been transmogrified. + */ + public function loadSchemaInfoFromConnection(DBALConnection $conn, string $tablesJsonPath): array + { + // Load declared tables from tables.json + $declared = []; + $loadedSchemaName = basename($tablesJsonPath); + try { + + if (is_readable($tablesJsonPath)) { + $json = file_get_contents($tablesJsonPath); + $cfg = json_decode($json, true); + if (is_array($cfg)) { + // Filter out documentation keys (eg, keys starting with "__") to avoid printing template entries + $cfg = array_filter($cfg, static function ($value, $key) { + return !(is_string($key) && str_starts_with($key, '__')); + }, ARRAY_FILTER_USE_BOTH); + $declared = array_keys($cfg); + } + } + } catch (\Throwable $e) { + // ignore; we'll fall back to empty list + } + + // Gather list of non-system tables + $tables = []; + if ($conn->isMySQL()) { + $db = $conn->fetchOne('SELECT DATABASE()'); + $rows = $conn->fetchAllAssociative('SELECT table_name FROM information_schema.tables WHERE table_schema = ? AND table_type = ? ORDER BY table_name ASC', [$db, 'BASE TABLE']); + $tables = array_map(fn($r) => $r['table_name'], $rows); + } else { + // PostgreSQL + $rows = $conn->fetchAllAssociative("SELECT schemaname, tablename FROM pg_catalog.pg_tables WHERE schemaname NOT IN ('pg_catalog','information_schema') ORDER BY schemaname, tablename"); + foreach ($rows as $r) { $tables[] = ($r['schemaname'] . '.' . $r['tablename']); } + } + + // Check if DB is empty (no tables) or all tables empty (zero rows) + $info = [ + 'table_count' => count($tables), + 'sample_tables' => array_slice($tables, 0, self::TABLE_SAMPLE_LIMIT), + 'empty' => false, + 'loaded_schema' => $loadedSchemaName, + ]; + + $info['empty'] = (count($tables) === 0); + if (!$info['empty']) { + // If there are tables, check if any table has rows; if none, still empty of data + $hasData = false; + foreach ($info['sample_tables'] as $t) { + try { + // For qualified names, don't double-quote + $count = (int)$conn->fetchOne('SELECT COUNT(*) FROM ' . $t); + if ($count > 0) { $hasData = true; break; } + } catch (\Throwable $e) { + // ignore per-table errors + } + } + $info['empty'] = !$hasData; + } + + // Build comparison lists between tables.json and actual DB tables, ignoring schema names + $normalize = function(string $t): string { + // Strip any schema prefix such as public.table or mysch.table + $p = strrpos($t, '.'); + return $p === false ? $t : substr($t, $p + 1); + }; + $actualBare = array_map($normalize, $tables); + $declaredBare = array_map($normalize, $declared); + // Now compute sets on bare names + $actual = $actualBare; + $declared = $declaredBare; + sort($actual); + sort($declared); + $both = array_values(array_unique(array_intersect($declared, $actual))); + $onlyJson = array_values(array_unique(array_diff($declared, $actual))); + $onlyDb = array_values(array_unique(array_diff($actual, $declared))); + + $info['tables_compare'] = [ + 'both' => $both, + 'only_in_json' => $onlyJson, + 'only_in_db' => $onlyDb, + ]; + + return $info; + } +} diff --git a/app/vendor/cakephp-plugins.php b/app/vendor/cakephp-plugins.php index 79b785483..c161ef702 100644 --- a/app/vendor/cakephp-plugins.php +++ b/app/vendor/cakephp-plugins.php @@ -16,5 +16,6 @@ 'Migrations' => $baseDir . '/vendor/cakephp/migrations/', 'OrcidSource' => $baseDir . '/plugins/OrcidSource/', 'SshKeyAuthenticator' => $baseDir . '/plugins/SshKeyAuthenticator/', + 'Transmogrify' => $baseDir . '/plugins/Transmogrify/', ], ];