From 31ff6cdce3a43ae3494ee053c127ae3a74a9801d Mon Sep 17 00:00:00 2001 From: Ioannis Igoumenos Date: Fri, 3 Oct 2025 11:19:54 +0300 Subject: [PATCH 01/15] Transmogrify plugin --- app/composer.json | 6 +- app/plugins/Transmogrify/README.md | 125 ++ app/plugins/Transmogrify/config/plugin.json | 7 + .../Transmogrify/config/schema/tables.json | 323 ++++ .../src/Command/TransmogrifyCommand.php | 550 +++++++ .../TransmogrifySourceToTargetCommand.php | 573 +++++++ .../src/Controller/AppController.php | 10 + .../src/Lib/Enum/TransmogrifyEnum.php | 36 + .../src/Lib/Traits/CacheTrait.php | 172 ++ .../src/Lib/Traits/HookRunnersTrait.php | 123 ++ .../src/Lib/Traits/ManageDefaultsTrait.php | 141 ++ .../src/Lib/Traits/RowTransformationTrait.php | 388 +++++ .../src/Lib/Traits/TypeMapperTrait.php | 324 ++++ .../src/Lib/Util/CommanLinePrinter.php | 184 +++ .../src/Lib/Util/CommandLinePrinter.php | 203 +++ .../src/Lib/Util/DbInfoPrinter.php | 246 +++ .../src/Lib/Util/RawSqlQueries.php | 421 +++++ .../Transmogrify/src/TransmogrifyPlugin.php | 104 ++ app/plugins/Transmogrify/webroot/.gitkeep | 0 app/src/Application.php | 19 + app/src/Command/TransmogrifyCommand.php | 1388 ----------------- app/src/Lib/Util/TransmogrifyUtilities.php | 179 --- app/src/Service/ConfigLoaderService.php | 120 ++ app/src/Service/DbInfoService.php | 184 +++ app/vendor/cakephp-plugins.php | 1 + 25 files changed, 4258 insertions(+), 1569 deletions(-) create mode 100644 app/plugins/Transmogrify/README.md create mode 100644 app/plugins/Transmogrify/config/plugin.json create mode 100644 app/plugins/Transmogrify/config/schema/tables.json create mode 100644 app/plugins/Transmogrify/src/Command/TransmogrifyCommand.php create mode 100644 app/plugins/Transmogrify/src/Command/TransmogrifySourceToTargetCommand.php create mode 100644 app/plugins/Transmogrify/src/Controller/AppController.php create mode 100644 app/plugins/Transmogrify/src/Lib/Enum/TransmogrifyEnum.php create mode 100644 app/plugins/Transmogrify/src/Lib/Traits/CacheTrait.php create mode 100644 app/plugins/Transmogrify/src/Lib/Traits/HookRunnersTrait.php create mode 100644 app/plugins/Transmogrify/src/Lib/Traits/ManageDefaultsTrait.php create mode 100644 app/plugins/Transmogrify/src/Lib/Traits/RowTransformationTrait.php create mode 100644 app/plugins/Transmogrify/src/Lib/Traits/TypeMapperTrait.php create mode 100644 app/plugins/Transmogrify/src/Lib/Util/CommanLinePrinter.php create mode 100644 app/plugins/Transmogrify/src/Lib/Util/CommandLinePrinter.php create mode 100644 app/plugins/Transmogrify/src/Lib/Util/DbInfoPrinter.php create mode 100644 app/plugins/Transmogrify/src/Lib/Util/RawSqlQueries.php create mode 100644 app/plugins/Transmogrify/src/TransmogrifyPlugin.php create mode 100644 app/plugins/Transmogrify/webroot/.gitkeep delete mode 100644 app/src/Command/TransmogrifyCommand.php delete mode 100644 app/src/Lib/Util/TransmogrifyUtilities.php create mode 100644 app/src/Service/ConfigLoaderService.php create mode 100644 app/src/Service/DbInfoService.php diff --git a/app/composer.json b/app/composer.json index b5a21792a..d3872f6e9 100644 --- a/app/composer.json +++ b/app/composer.json @@ -47,7 +47,8 @@ "PipelineToolkit\\": "availableplugins/PipelineToolkit/src/", "SqlConnector\\": "availableplugins/SqlConnector/src/", "SshKeyAuthenticator\\": "plugins/SshKeyAuthenticator/src/", - "CoreJob\\": "plugins/CoreJob/src/" + "CoreJob\\": "plugins/CoreJob/src/", + "Transmogrify\\": "plugins/Transmogrify/src/" } }, "autoload-dev": { @@ -66,7 +67,8 @@ "PipelineToolkit\\Test\\": "availableplugins/PipelineToolkit/tests/", "SqlConnector\\Test\\": "availableplugins/SqlConnector/tests/", "SshKeyAuthenticator\\Test\\": "plugins/SshKeyAuthenticator/tests/", - "CoreJob\\Test\\": "plugins/CoreJob/tests/" + "CoreJob\\Test\\": "plugins/CoreJob/tests/", + "Transmogrify\\": "plugins/Transmogrify/src/" } }, "scripts": { diff --git a/app/plugins/Transmogrify/README.md b/app/plugins/Transmogrify/README.md new file mode 100644 index 000000000..05c513bb5 --- /dev/null +++ b/app/plugins/Transmogrify/README.md @@ -0,0 +1,125 @@ +# Transmogrify (COmanage Registry Plugin) + +Transmogrify is a command‑line migration tool bundled as a CakePHP plugin for COmanage Registry PE. It copies and transforms data from a legacy/source Registry schema (cm_… tables) into the PE target schema using a configurable mapping (config/schema/tables.json) plus code hooks for special cases. + +It is designed to be: +- Safe by default: target tables that already contain data are skipped and not overwritten. +- Transparent: shows a progress bar with in‑line warnings and errors. +- Configurable: table selection, custom tables.json path, and optional helper modes (info, schema, etc). + + +## Prerequisites +- Configure database connections in local/config/database.php + - Source DB (used by Transmogrify) and target DB must both be reachable. Transmogrify initializes two Doctrine DBAL connections internally. +- The default tables mapping file is at: + - app/plugins/Transmogrify/config/schema/tables.json + + +## Command +Invoke from your app root: + +- bin/cake transmogrify [options] + +Cake’s standard flags also apply (eg: --verbose, --quiet, -h). + + +## Options +These options come directly from TransmogrifyCommand::buildOptionParser. + +- --tables-config PATH + - Path to the transmogrify tables JSON config (defaults to the plugin’s tables.json). + +- --dump-tables-config + - Print the effective tables configuration (after schema extension) and exit. + +- --table NAME (repeatable) + - Migrate only the specified table(s). Repeat --table to select multiple. + - If omitted, Transmogrify processes all known tables in the configured order. + +- --list-tables + - List available target tables from the transmogrify config and exit. + +- --info + - Print source/target database configuration and exit. + +- --info-json + - Output info in JSON (use with --info). + +- --info-ping + - Ping connections and include connectivity + server version details (use with --info or --info-schema). + +- --info-schema + - Print schema information and whether the database is empty (defaults to inspecting the target DB). + +- --info-schema-role ROLE + - When used with --info-schema, select which database to inspect: source or target (default: target). + +- --login-identifier-copy + - Enable helper logic to copy/set up login identifiers during migration (see your deployment’s identifier policy). Use together with --login-identifier-type to choose the identifier type. + +- --login-identifier-type TYPE + - Identifier type value to use for login identifiers when --login-identifier-copy is set. + + +## Typical usage + +- Migrate everything using the default mapping + - bin/cake transmogrify + +- Preview environment information + - bin/cake transmogrify --info + - bin/cake transmogrify --info --info-json + - bin/cake transmogrify --info --info-ping + +- Inspect schema state (target by default, or choose source) + - bin/cake transmogrify --info-schema + - bin/cake transmogrify --info-schema --info-schema-role source + +- List the tables Transmogrify knows how to process + - bin/cake transmogrify --list-tables + +- Dump the effective tables configuration + - bin/cake transmogrify --dump-tables-config + +- Migrate a subset of tables (in safe order) + - bin/cake transmogrify --table types --table people --table person_roles + +- Use a custom tables.json mapping + - bin/cake transmogrify --tables-config /path/to/your/tables.json + +- Migrate with login identifier help + - bin/cake transmogrify --table identifiers --login-identifier-copy --login-identifier-type eppn + +Hints: +- Combine Cake’s verbosity flags for extra diagnostics: add --verbose to see per‑row details emitted by some hooks; add --quiet to minimize output (progress still shows). + + +## Behavior notes +- Target table already has data? Transmogrify will skip that table and warn (no overwrite). +- Primary key sequences are aligned automatically to preserve/accept explicit IDs where possible. +- Ordering and foreign keys: + - The tool emits rows in a dependency‑friendly order and may retry deferred rows to satisfy FK dependencies. + - You’ll see warnings for rows skipped due to unresolved foreign keys or missing type mappings. +- Progress UI: + - A single‑line progress bar updates in place. + - Warnings and errors appear under the bar as they happen. + + +## Mapping and hooks +- Mapping is defined in config/schema/tables.json. + - You can specify per‑table field maps, boolean normalization, caches, and pre/post hooks. +- Some tables use custom SELECTs to ensure parent‑before‑child ordering (eg recursive chains). +- Type lookups depend on the ‘types’ cache; migrate ‘types’ early or include it in the same run if you migrate data that requires type IDs. + + +## Exit codes +- 0: Success +- Non‑zero: Error (check the emitted [ERROR] lines) + + +## Getting help +- bin/cake transmogrify -h +- Consult this plugin’s code for advanced behavior: + - src/Command/TransmogrifyCommand.php + - config/schema/tables.json + - src/Lib/Traits/* (type mapping, caching, row transformations, hooks) diff --git a/app/plugins/Transmogrify/config/plugin.json b/app/plugins/Transmogrify/config/plugin.json new file mode 100644 index 000000000..92d32ce1e --- /dev/null +++ b/app/plugins/Transmogrify/config/plugin.json @@ -0,0 +1,7 @@ +{ + "name": "Transmogrify", + "version": "1.0.0", + "description": "Data migration command to transmogrify data from a source database into the target schema.", + "types": {}, + "schema": null +} diff --git a/app/plugins/Transmogrify/config/schema/tables.json b/app/plugins/Transmogrify/config/schema/tables.json new file mode 100644 index 000000000..656737920 --- /dev/null +++ b/app/plugins/Transmogrify/config/schema/tables.json @@ -0,0 +1,323 @@ +{ + "__COMMENT__": "Template for adding new table configurations to the Transmogrify mapping. Keys starting with double underscores (__) are ignored by processors and exist only for documentation.", + "__EXAMPLE_TABLE_TEMPLATE__": { + "__INSTRUCTIONS__": "Copy this object, rename the key to your logical table name (eg, 'my_items'), and adjust values. Do NOT leave this example enabled in production.", + "source": "cm_my_items", + "displayField": "name", + "addChangelog": true, + "booleans": ["is_active", "is_primary"], + "cache": [ + "co_id", + ["co_id", "status"], + ["co_id", "name", "value"] + ], + "sqlSelect": "myItemsSqlSelect", + "preTable": "beforeInsertMyItems", + "postTable": "afterInsertMyItems", + "preRow": "beforeInsertMyItemRow", + "postRow": "afterInsertMyItemRow", + "fieldMap": { + "id": null, + "co_id": "co_id", + "name": "name", + "description": "details", + "status": "status", + "value": "amount", + "created": "&mapNow", + "modified": "&mapNow", + "type_id": "&mapExtendedType", + "owner_identifier": "&mapIdentifierToPersonId", + "valid_from": "valid_from", + "valid_through": "valid_through" + }, + "__NOTES__": "Common keys: source (DB table), displayField (for printing), addChangelog (add created/modified history), booleans (type coercion), cache (key(s) to cache rows by), sqlSelect (custom SELECT provider function), pre/postTable and pre/postRow (hook function names), fieldMap (left=new table column, right=source column name or &function). Use &mapNow to inject current timestamp, &mapExtendedType to resolve cm_co_extended_types, and mapping helpers found in Transmogrify traits. Note: There is no generic config-driven enforcement for required/defaults/unique; enforce uniqueness via database constraints." + }, + "cos": { + "source": "cm_cos", + "displayField": "name", + "addChangelog": true, + "cache": ["status"] + }, + "types": { + "source": "cm_co_extended_types", + "displayField": "display_name", + "postTable": "insertPronounTypes", + "fieldMap": { + "attribute": "&mapExtendedType", + "name": "value", + "created": "&mapNow", + "modified": "&mapNow" + }, + "cache": [["co_id", "attribute", "value"]] + }, + "co_settings": { + "source": "cm_co_settings", + "displayField": "co_id", + "addChangelog": true, + "booleans": [], + "postTable": "insertDefaultSettings", + "cache": ["co_id"], + "fieldMap": { + "global_search_limit": "search_global_limit", + "required_fields_addr": "required_fields_address", + "permitted_fields_telephone_number": "&populateCoSettingsPhone", + "enable_nsf_demo": null, + "disable_expiration": null, + "disable_ois_sync": null, + "group_validity_sync_window": null, + "garbage_collection_interval": null, + "enable_normalization": null, + "enable_empty_cou": null, + "invitation_validity": null, + "t_and_c_login_mode": null, + "sponsor_eligibility": null, + "sponsor_co_group_id": null, + "theme_stacking": null, + "default_co_pipeline_id": null, + "elect_strategy_primary_name": null, + "co_dashboard_id": null, + "co_theme_id": null, + "person_picker_email_type": null, + "person_picker_identifier_type": null, + "person_picker_display_types": null, + "group_create_admin_only": null + } + }, + "authentication_events": { + "source": "cm_authentication_events", + "displayField": "authenticated_identifier" + }, + "api_users": { + "source": "cm_api_users", + "displayField": "username", + "booleans": ["privileged"], + "fieldMap": { + "password": "api_key" + } + }, + "cous": { + "source": "cm_cous", + "displayField": "name", + "sqlSelect": "couSqlSelect" + }, + "people": { + "source": "cm_co_people", + "displayField": "id", + "cache": ["co_id"], + "fieldMap": { + "co_person_id": "person_id" + } + }, + "person_roles": { + "source": "cm_co_person_roles", + "sqlSelect": "roleSqlSelect", + "displayField": "id", + "cache": ["status"], + "fieldMap": { + "co_person_id": "person_id", + "co_person_role_id": "person_role_id", + "affiliation_type_id": "&mapAffiliationType", + "affiliation": null, + "manager_co_person_id": "manager_person_id", + "sponsor_co_person_id": "sponsor_person_id", + "o": "organization", + "ou": "department", + "source_org_identity_id": null + } + }, + "external_identities": { + "source": "cm_org_identities", + "displayField": "id", + "fieldMap": { + "co_id": null, + "person_id": "&mapOrgIdentitycoPersonId", + "org_identity_id": "external_identity_id", + "title": null, + "o": null, + "ou": null, + "affiliation": null, + "manager_identifier": null, + "sponsor_identifier": null, + "valid_from": null, + "valid_through": null + }, + "postRow": "mapExternalIdentityToExternalIdentityRole", + "cache": ["person_id"] + }, + "groups": { + "source": "cm_co_groups", + "displayField": "name", + "cache": ["co_id", "owners_group_id"], + "booleans": ["nesting_mode_all", "open"], + "fieldMap": { + "auto": null, + "co_group_id": "group_id", + "group_type": "?S" + }, + "postTable": "createOwnersGroups" + }, + "group_nestings": { + "source": "cm_co_group_nestings", + "displayField": "id", + "booleans": ["negate"], + "fieldMap": { + "co_group_id": "group_id", + "target_co_group_id": "target_group_id", + "co_group_nesting_id": "group_nesting_id" + } + }, + "group_members": { + "source": "cm_co_group_members", + "displayField": "id", + "booleans": ["member", "owner"], + "fieldMap": { + "co_group_id": "group_id", + "co_person_id": "person_id", + "member": null, + "owner": null, + "co_group_nesting_id": "group_nesting_id", + "co_group_member_id": "group_member_id", + "source_org_identity_id": null + }, + "preRow": "reconcileGroupMembershipOwnership" + }, + "names": { + "source": "cm_names", + "displayField": "id", + "booleans": ["primary_name"], + "fieldMap": { + "co_person_id": "person_id", + "org_identity_id": "external_identity_id", + "type_id": "&mapNameType", + "type": null + } + }, + "ad_hoc_attributes": { + "source": "cm_ad_hoc_attributes", + "displayField": "id", + "fieldMap": { + "co_person_role_id": "person_role_id", + "org_identity_id": "external_identity_id", + "co_department_id": null, + "organization_id": null + }, + "postTable": "migrateExtendedAttributesToAdHocAttributes" + }, + "addresses": { + "source": "cm_addresses", + "displayField": "id", + "fieldMap": { + "co_person_role_id": "person_role_id", + "org_identity_id": "external_identity_id", + "type_id": "&mapAddressType", + "type": null, + "co_department_id": null, + "organization_id": null + } + }, + "email_addresses": { + "source": "cm_email_addresses", + "displayField": "id", + "booleans": ["verified"], + "fieldMap": { + "co_person_id": "person_id", + "org_identity_id": "external_identity_id", + "type_id": "&mapEmailType", + "type": null, + "co_department_id": null, + "organization_id": null + } + }, + "identifiers": { + "source": "cm_identifiers", + "displayField": "id", + "booleans": ["login"], + "fieldMap": { + "co_group_id": "group_id", + "co_person_id": "person_id", + "org_identity_id": "external_identity_id", + "type_id": "&mapIdentifierType", + "type": null, + "co_department_id": null, + "co_provisioning_target_id": null, + "organization_id": null + }, + "preRow": "mapLoginIdentifiers" + }, + "telephone_numbers": { + "source": "cm_telephone_numbers", + "displayField": "id", + "fieldMap": { + "co_person_role_id": "person_role_id", + "org_identity_id": "external_identity_id", + "type_id": "&mapTelephoneType", + "type": null, + "co_department_id": null, + "organization_id": null + } + }, + "urls": { + "source": "cm_urls", + "displayField": "id", + "fieldMap": { + "co_person_id": "person_id", + "org_identity_id": "external_identity_id", + "type_id": "&mapUrlType", + "type": null, + "co_department_id": null, + "organization_id": null + } + }, + "history_records": { + "source": "cm_history_records", + "displayField": "id", + "fieldMap": { + "actor_co_person_id": "actor_person_id", + "co_person_id": "person_id", + "co_person_role_id": "person_role_id", + "co_group_id": "group_id", + "org_identity_id": "external_identity_id", + "co_email_list_id": null, + "co_service_id": null + } + }, + "jobs": { + "source": "cm_co_jobs", + "displayField": "id", + "fieldMap": { + "job_type": "plugin", + "job_mode": null, + "queue_time": "register_time", + "complete_time": "finish_time", + "job_type_fk": null, + "job_params": "parameters", + "requeued_from_co_job_id": "requeued_from_job_id", + "max_retry": null, + "max_retry_count": null + }, + "preRow": "validateJobIsTransmogrifiable" + }, + "job_history_records": { + "source": "cm_co_job_history_records", + "displayField": "id", + "fieldMap": { + "co_job_id": "job_id", + "co_person_id": "person_id", + "org_identity_id": "external_identity_id" + } + }, + "servers": { + "source": "cm_servers", + "displayField": "description", + "addChangelog": false, + "cache": ["status"], + "sqlSelect": null, + "preTable": null, + "postTable": null, + "preRow": null, + "postRow": null, + "fieldMap": { + "plugin": "&mapServerTypeToPlugin" + } + } +} diff --git a/app/plugins/Transmogrify/src/Command/TransmogrifyCommand.php b/app/plugins/Transmogrify/src/Command/TransmogrifyCommand.php new file mode 100644 index 000000000..17aecffe4 --- /dev/null +++ b/app/plugins/Transmogrify/src/Command/TransmogrifyCommand.php @@ -0,0 +1,550 @@ +pluginRoot = dirname(__DIR__, 2); + parent::__construct(); + } + + /** + * Override run command + * + * @param array $argv + * @param ConsoleIo $io + * @return int + */ + public function run(array $argv, ConsoleIo $io): int + { + $this->inconn = DBALConnection::factory($io, 'transmogrify'); + $this->outconn = DBALConnection::factory($io); + return parent::run($argv, $io); + } + + /** + * Build an Option Parser. + * + * @since COmanage Registry v5.0.0 + * @param ConsoleOptionParser $parser ConsoleOptionParser + * @return ConsoleOptionParser ConsoleOptionParser + */ + + protected function buildOptionParser(ConsoleOptionParser $parser): ConsoleOptionParser + { + // Allow overriding the tables config path + $parser->addOption('tables-config', [ + 'help' => 'Path to transmogrify tables JSON config', + 'default' => TransmogrifyEnum::TABLES_JSON_PATH + ]); + $parser->addOption('dump-tables-config', [ + 'help' => 'Output the effective tables configuration (after schema extension) and exit', + 'boolean' => true + ]); + // Specify a table (or repeat option) to migrate only a subset + $parser->addOption('table', [ + 'help' => 'Migrate only the specified table. Repeat the option to migrate multiple tables', + 'multiple' => true + ]); + // List available target tables and exit + $parser->addOption('list-tables', [ + 'help' => 'List available target tables from the transmogrify config and exit', + 'boolean' => true + ]); + // Info options integrated into TransmogrifyCommand + $parser->addOption('info', [ + 'help' => 'Print source and target database configuration and exit', + 'boolean' => true + ]); + $parser->addOption('info-json', [ + 'help' => 'Output info in JSON (use with --info)', + 'boolean' => true + ]); + $parser->addOption('info-ping', [ + 'help' => 'Ping connections and include connectivity + server version (use with --info or --info-schema)', + 'boolean' => true + ]); + $parser->addOption('info-schema', [ + 'help' => 'Print schema information and whether the database is empty (defaults to target). Use --info-schema-role to select source/target', + 'boolean' => true + ]); + $parser->addOption('info-schema-role', [ + 'help' => 'When using --info-schema, which database to inspect: source or target (default: target)' + ]); + $parser->addOption('login-identifier-copy', [ + 'help' => __d('command', 'tm.login-identifier-copy'), + 'boolean' => true + ]); + + $parser->addOption('login-identifier-type', [ + 'help' => __d('command', 'tm.login-identifier-type') + ]); + + $parser->setEpilog(__d('command', 'tm.epilog')); + + return $parser; + } + + /** + * Execute the Transmogrify Command. + * + * @param Arguments $args Command Arguments + * @param ConsoleIo $io Console IO + * @throws Exception + * @since COmanage Registry v5.0.0 + */ + + public function execute(Arguments $args, ConsoleIo $io): int + { + $this->args = $args; + $this->io = $io; + + // Validate "info" option combinations and handle errors + $code = $this->validateInfoOptions($io); + if ($code !== null) { + return $code; + } + + // Handle info modes early (no tables config needed unless ping) + $code = $this->maybeHandleInfo(); + if ($code !== null) { + return $code; + } + + // Load tables configuration (from JSON) and extend it with schema data + $this->loadTablesConfig(); + + // Dump tables config if requested + $code = $this->maybeDumpTablesConfig($io); + if ($code !== null) { + return $code; + } + + // List tables and exit + $code = $this->maybeListTables($io); + if ($code !== null) { + return $code; + } + + // Build list of tables to migrate from --table option and positional args + $selected = $this->buildSelectedTables($args); + + // Validate and warn for subset selection + $code = $this->maybeValidateSelectedTables($selected, $io); + if ($code !== null) { + return $code; + } + + // Register the current version for future upgrade purposes + $this->metaTable = TableRegistry::getTableLocator()->get('Meta'); + $this->metaTable->setUpgradeVersion(); + + foreach(array_keys($this->tables) as $t) { + $modeltableEmpty = true; + $notSelected = false; + + + $io->info(message: sprintf("Transmogrifying table %s(%s)", Inflector::classify($t), $t)); + + + // Step 1: Check if source table exists and warn if not present + if(!empty($this->tables[$t]['source'])) { + $src = $this->tables[$t]['source']; + if (!$this->tableExists($src)) { + $this->io->warning("Source table '$src' does not exist in source database, skipping table '$t'"); + continue; + } + } + // Step 2: Skip tables not in selected subset if specified + // We will print a warning and we will parse all the tables because we need to construct the cache. + // Nevertheless, we will not allow any database processing + if (!empty($selected) && !in_array($t, $selected)) { + $notSelected = true; + $io->warning("Skipping Transmogrification. Table ($t) is not in the selected subset."); + } + + // Step 3: Configure sequence ID for the target table + if(!RawSqlQueries::setSequenceId( + $this->inconn, + $this->outconn, + $this->tables[$t]['source'], + $t, + $this->io + )) { + $io->warning("Skipping Transmogrification. Can not properly configure the Sequence for the primary key for the Table (\"$t\""); + return BaseCommand::CODE_ERROR; + } + + // Step 5: Execute any pre-processing hooks for the current table + $this->runPreTableHook($t); + + // Step 6: Skip if target table already contains data + $Model = TableRegistry::getTableLocator()->get($t); + if($Model->find()->count() > 0) { + $modeltableEmpty = false; + $io->warning("Table (" . $t . ") is not empty. We will not overwrite existing data."); + } + + // Step 7: Get total count of source records for progress tracking + $qualifiedTableName = $this->inconn->qualifyTableName($this->tables[$t]['source']); + $count = $this->inconn->fetchOne(RawSqlQueries::buildCountAll($qualifiedTableName)); + + // Step 8: Build and execute query to fetch all source records + $insql = match(true) { + !empty($this->tables[$t]['sqlSelect']) => $this->runSqlSelectHook($t, $qualifiedTableName), + default => RawSqlQueries::buildSelectAllOrderedById($qualifiedTableName) + }; + $stmt = $this->inconn->executeQuery($insql); + + $progress = new CommanLinePrinter($io, 'green', 50, true); + $progress->start($count); + $tally = 0; + $warns = 0; + $err = 0; + + while ($row = $stmt->fetchAssociative()) { + if(!empty($row[ $this->tables[$t]['displayField'] ])) { + $io->verbose("$t " . $row[ $this->tables[$t]['displayField'] ]); + } + + try { + // Step 1: Create a copy of the original row data to preserve it for post-processing + $origRow = $row; + + // Step 2: Execute any pre-processing hooks to transform or validate the row data + $this->runPreRowHook($t, $origRow, $row); + + // Step 3: Set changelog defaults (created/modified timestamps, user IDs) + // Must be done before boolean normalization as it adds new fields + $this->populateChangelogDefaults( + $t, + $row, + isset($this->tables[$t]['addChangelog']) && $this->tables[$t]['addChangelog'] + ); + + // Step 4: Convert boolean values to database-compatible format + $this->normalizeBooleanFieldsForDb($t, $row); + + // Step 5: Map old field names to new schema field names + $this->mapLegacyFieldNames($t, $row); + + // Step 6: Insert the transformed row into the target database + $qualifiedTableName = $this->outconn->qualifyTableName($t); + + if($modeltableEmpty && !$notSelected) { + $this->outconn->insert($qualifiedTableName, $row); + // Step 8: Execute any post-processing hooks after successful insertion + $this->runPostRowHook($t, $origRow, $row); + } + + // Step 7: Store row data in cache for potential later use + $this->cacheResults($t, $row); + } catch(ForeignKeyConstraintViolationException $e) { + // A foreign key associated with this record did not load, so we can't + // load this record. This can happen, eg, because the source_field_id + // did not load, perhaps because it was associated with an Org Identity + // not linked to a CO Person that was not migrated. + $warns++; + $progress->warn("Skipping $t record " . $row['id'] . " due to invalid foreign key: " . $e->getMessage()); + } + catch(\InvalidArgumentException $e) { + // If we can't find a value for mapping we skip the record + // (ie: mapLegacyFieldNames basically requires a successful mapping) + $warns++; + $progress->warn("Skipping $t record " . $row['id'] . ": " . $e->getMessage()); + } + catch(\Exception $e) { + $err++; + $progress->error("$t record " . $row['id'] . ": " . $e->getMessage()); + } + + $tally++; + + if (!$this->args->getOption('quiet') && !$this->args->getOption('verbose')) { + $progress->update($tally); + } + } + + $progress->finish(); + // Step 10: Output final warning and error counts for the table + $io->warning(sprintf('Warnings: %d', $warns)); + $io->error(sprintf('Errors: %d', $err)); + + // Step 11: Execute any post-processing hooks for the table + if ($modeltableEmpty && !$notSelected) { + $this->runPostTableHook($t); + } + + // Prompt for confirmation before processing table + $tables = array_keys($this->tables); + $currentIndex = array_search($t, $tables); + if (isset($tables[$currentIndex + 1])) { + $io->info("Next table to process: " . $tables[$currentIndex + 1]); + } else { + $io->info("This is the last table to process."); + } + + $io->ask('Press to continue...'); + } + + return BaseCommand::CODE_SUCCESS; + } + + + /** + * Validate incompatible/invalid "info" related options. + * Returns an exit code when invalid, or null to continue. + * + * @param ConsoleIo $io Console IO object for output + * @return int|null Command exit code or null to continue + * @since COmanage Registry v5.2.0 + */ + private function validateInfoOptions(ConsoleIo $io): ?int + { + if ( + $this->args->getOption('info-ping') + && !$this->args->getOption('info') + && !$this->args->getOption('info-schema') + ) { + $io->err('Option --info-ping must be used together with --info or --info-schema.'); + $io->err('Examples:'); + $io->err(' bin/cake transmogrify --info --info-ping'); + $io->err(' bin/cake transmogrify --info-schema --info-ping [--info-schema-role source|target]'); + return BaseCommand::CODE_ERROR; + } + + if ( + $this->args->getOption('info-schema-role') !== null + && !$this->args->getOption('info-schema') + ) { + $io->err('Option --info-schema-role must be used together with --info-schema.'); + $io->err('Examples:'); + $io->err(' bin/cake transmogrify --info-schema --info-schema-role target'); + $io->err(' bin/cake transmogrify --info-schema --info-ping --info-schema-role source'); + $io->err(' bin/cake transmogrify --info-schema --info-json --info-schema-role source'); + return BaseCommand::CODE_ERROR; + } + + return null; + } + + /** + * Handle --info / --info-schema early-exit modes. + * Returns exit code if handled, or null to continue normal execution. + * + * @return int|null Command exit code if handled, null to continue execution + * @since COmanage Registry v5.2.0 + */ + private function maybeHandleInfo(): ?int + { + if ($this->args->getOption('info')) { + (DbInfoPrinter::initialize($this->io, $this->dbInfoService))->print( + (bool)$this->args->getOption('info-json'), + (bool)$this->args->getOption('info-ping') + ); + return BaseCommand::CODE_SUCCESS; + } + + if ($this->args->getOption('info-schema')) { + $role = $this->args->getOption('info-schema-role') ?: 'target'; + if (!in_array($role, ['source', 'target'], true)) { $role = 'target'; } + (DbInfoPrinter::initialize($this->io, $this->dbInfoService))->print( + (bool)$this->args->getOption('info-json'), + (bool)$this->args->getOption('info-ping'), + true, + $role + ); + return BaseCommand::CODE_SUCCESS; + } + + return null; + } + + /** + * Load tables configuration from JSON and attach to $this->tables. + * + * @return void + * @since COmanage Registry v5.2.0 + */ + private function loadTablesConfig(): void + { + $path = $this->args->getOption('tables-config') ?? Transmogrify::TABLES_JSON_PATH; + if (!str_starts_with($path, $this->pluginRoot . DS)) { + $path = $this->pluginRoot . DS . $path; + } + $this->tables = $this->configLoader->load($path); + } + + /** + * If requested, dump effective tables configuration and exit. + * + * @param ConsoleIo $io Console IO object for output + * @return int|null Command exit code if dumped, null to continue + * @since COmanage Registry v5.2.0 + */ + private function maybeDumpTablesConfig(ConsoleIo $io): ?int + { + if ($this->args->getOption('dump-tables-config')) { + $io->out(json_encode($this->tables, JSON_PRETTY_PRINT)); + return BaseCommand::CODE_SUCCESS; + } + return null; + } + + /** + * If requested, list available tables and exit. + * + * @param ConsoleIo $io Console IO object for output + * @return int|null Command exit code if listed, null to continue + * @since COmanage Registry v5.2.0 + */ + private function maybeListTables(ConsoleIo $io): ?int + { + if ($this->args->getOption('list-tables')) { + $io->out(implode("\n", array_keys($this->tables))); + return BaseCommand::CODE_SUCCESS; + } + return null; + } + + /** + * Build list of tables from --table options and positional args. + * + * @param Arguments $args Command arguments + * @return array List of selected table names + * @since COmanage Registry v5.2.0 + */ + private function buildSelectedTables(Arguments $args): array + { + $selected = $args->getArrayOption('table') ?? []; + $positional = $args->getArguments(); + if (!empty($positional)) { + $selected = array_merge($selected, $positional); + } + return array_values(array_unique($selected)); + } + + /** + * Validate selected tables against config and warn about partial migration. + * Returns exit code on error, or null if OK. + * + * @param array $selected List of selected table names + * @param ConsoleIo $io Console IO object for output + * @return int|null Command exit code on error, null if valid + * @since COmanage Registry v5.2.0 + */ + private function maybeValidateSelectedTables(array $selected, ConsoleIo $io): ?int + { + if (!empty($selected)) { + $unknown = array_diff($selected, array_keys($this->tables)); + if (!empty($unknown)) { + $io->err('Unknown table(s): ' . implode(', ', $unknown)); + $io->err('Use --list-tables to see available options.'); + return BaseCommand::CODE_ERROR; + } + $io->warning('Migrating a subset of tables may lead to foreign key or type mapping warnings if dependencies are not loaded (eg, types, people, groups).'); + $io->out('Selected tables: ' . implode(', ', $selected)); + } + return null; + } + + /** + * Check if a table exists in the source database + * + * @param string $tableName Name of table to check + * @return bool True if table exists + * @throws \Exception + */ + protected function tableExists(string $tableName): bool + { + $dbSchemaManager = $this->inconn->createSchemaManager(); + $tableList = $dbSchemaManager->listTableNames(); + return in_array($tableName, $tableList); + } +} \ No newline at end of file diff --git a/app/plugins/Transmogrify/src/Command/TransmogrifySourceToTargetCommand.php b/app/plugins/Transmogrify/src/Command/TransmogrifySourceToTargetCommand.php new file mode 100644 index 000000000..a074b9a58 --- /dev/null +++ b/app/plugins/Transmogrify/src/Command/TransmogrifySourceToTargetCommand.php @@ -0,0 +1,573 @@ +setDescription('Print a JSON template shaped like __EXAMPLE_TABLE_TEMPLATE__ for mapping a source table to a target table. Target config is used only to read the example template; no values are copied. Paths must be absolute.'); + + // Switch from positional arguments to named options (long and short) + $parser->addOption('source-path', [ + 'short' => 's', + 'help' => 'Absolute path to the SOURCE configuration file (JSON or XML).', + ]); + $parser->addOption('target-path', [ + 'short' => 't', + 'help' => 'Absolute path to the TARGET configuration file (JSON or XML).', + ]); + $parser->addOption('source-table', [ + 'short' => 'S', + 'help' => 'Table key/name in the source configuration to map from.', + ]); + $parser->addOption('target-table', [ + 'short' => 'T', + 'help' => 'Table key/name in the target configuration to map to.', + ]); + $parser->addOption('source-prefix', [ + 'short' => 'p', + 'help' => 'Prefix to prepend to the source table in the output (eg, "cm_"). Default: empty string.', + ]); + + // Association tree printer option + $parser->addOption('assoc-tree', [ + 'short' => 'A', + 'help' => 'Print an ASCII tree of associations starting from the given --source-table based on the source XML. Use with -s and -S. Ignores other options and exits.', + 'boolean' => true, + ]); + + $epilog = []; + $epilog[] = 'Examples:'; + $epilog[] = ' bin/cake transmogrify_source_to_target \\\n --source-path /path/to/registry/app/Config/Schema/schema.xml \\\n --target-path /path/to/registry/app/config/transmogrifytables.json \\\n --source-table cm_co_people \\\n --target-table people'; + $epilog[] = ''; + $epilog[] = ' bin/cake transmogrify_source_to_target -s /abs/source.xml -t /abs/target.json -S cousins -T cous -p cm_'; + $epilog[] = ''; + $epilog[] = 'Notes:'; + $epilog[] = ' - Paths must be absolute.'; + $epilog[] = ' - Files may be JSON (.json) or XML (.xml).'; + $epilog[] = ' - The output is a single JSON object similar to __EXAMPLE_TABLE_TEMPLATE__ in tables.json.'; + $epilog[] = ' - Use --source-prefix/-p to control any prefix (like cm_) applied to the source table name in the output.'; + $parser->setEpilog(implode(PHP_EOL, $epilog)); + + return parent::buildOptionParser($parser); + } + + /** + * Execute the command + * + * @param Arguments $args Command arguments + * @param ConsoleIo $io Console IO instance + * @return int Exit code + */ + public function execute(Arguments $args, ConsoleIo $io): int + { + // capture prefix for helper methods + $this->currentPrefix = (string)($args->getOption('source-prefix') ?? ''); + $sourcePath = (string)($args->getOption('source-path') ?? ''); + $targetPath = (string)($args->getOption('target-path') ?? ''); + $sourceTable = (string)($args->getOption('source-table') ?? ''); + $targetTable = (string)($args->getOption('target-table') ?? ''); + $sourcePrefix = (string)($args->getOption('source-prefix') ?? ''); + $assocTree = (bool)($args->getOption('assoc-tree') ?? false); + + // If only association tree requested, we only need source path and source table + if ($assocTree) { + $missing = []; + if ($sourcePath === '') { $missing[] = '--source-path (-s)'; } + if ($sourceTable === '') { $missing[] = '--source-table (-S)'; } + if (!empty($missing)) { + $io->err('Missing required option(s) for --assoc-tree: ' . implode(', ', $missing)); + $io->err('Run with --help to see usage.'); + return BaseCommand::CODE_ERROR; + } + if ($sourcePath === '' || !str_starts_with($sourcePath, DIRECTORY_SEPARATOR)) { + $io->err('SOURCE path must be absolute: ' . $sourcePath); + return BaseCommand::CODE_ERROR; + } + if (!is_readable($sourcePath)) { + $io->err('SOURCE file not readable: ' . $sourcePath); + return BaseCommand::CODE_ERROR; + } + + try { + $srcCfgRaw = $this->configLoaderService->loadGeneric($sourcePath); + } catch (\Throwable $e) { + $io->err('Failed to load source configuration: ' . $e->getMessage()); + return BaseCommand::CODE_ERROR; + } + $this->printAssociationTreeFromSchema($srcCfgRaw, $sourceTable, $io); + return BaseCommand::CODE_SUCCESS; + } + + // Validate required named options are present + $missing = []; + if ($sourcePath === '') { $missing[] = '--source-path (-s)'; } + if ($targetPath === '') { $missing[] = '--target-path (-t)'; } + if ($sourceTable === '') { $missing[] = '--source-table (-S)'; } + if ($targetTable === '') { $missing[] = '--target-table (-T)'; } + if (!empty($missing)) { + $io->err('Missing required option(s): ' . implode(', ', $missing)); + $io->err('Run with --help to see usage.'); + return BaseCommand::CODE_ERROR; + } + + // Validate absolute paths + foreach (['source' => $sourcePath, 'target' => $targetPath] as $label => $path) { + if ($path === '' || !str_starts_with($path, DIRECTORY_SEPARATOR)) { + $io->err(strtoupper($label) . ' path must be absolute: ' . $path); + return BaseCommand::CODE_ERROR; + } + if (!is_readable($path)) { + $io->err(strtoupper($label) . ' file not readable: ' . $path); + return BaseCommand::CODE_ERROR; + } + } + + try { + // Load target config to read __EXAMPLE_TABLE_TEMPLATE__ shape + $tgtCfg = $this->configLoaderService->loadGeneric($targetPath); + // Load source config to extract old schema (eg, from XML) + $srcCfgRaw = $this->configLoaderService->loadGeneric($sourcePath); + } catch (\Throwable $e) { + $io->err('Failed to load configuration: ' . $e->getMessage()); + return BaseCommand::CODE_ERROR; + } + + // Normalize possible schema.xml structures into transmogrify-like configs for source + $srcCfg = $this->normalizeConfig(is_array($srcCfgRaw) ? $srcCfgRaw : []); + + // Extract example template (shape) from target config without filtering out __ keys first + $example = is_array($tgtCfg) && array_key_exists('__EXAMPLE_TABLE_TEMPLATE__', $tgtCfg) + ? (array)$tgtCfg['__EXAMPLE_TABLE_TEMPLATE__'] + : []; + + // Determine the output keys based on the example (ignoring any __* documentation keys) + $exampleKeys = array_filter(array_keys($example), static function($k) { + return !is_string($k) || !str_starts_with($k, '__'); + }); + + // Determine source fields from old schema configuration (if available) + $sourceFields = []; + [$srcKey, $srcTableCfg] = $this->findTableConfig($srcCfg, $sourceTable); + if (is_array($srcTableCfg)) { + if (!empty($srcTableCfg['fieldMap']) && is_array($srcTableCfg['fieldMap'])) { + $sourceFields = array_keys($srcTableCfg['fieldMap']); + } elseif (!empty($srcTableCfg['fields']) && is_array($srcTableCfg['fields'])) { + $sourceFields = array_values($srcTableCfg['fields']); + } + } + $sourceFieldSet = array_flip($sourceFields); + + // Inspect default database to find target table columns + try { + $conn = DBALConnection::factory($io); + $columns = []; + $candidate = $targetTable; + $alts = [ $candidate ]; + // toggle provided prefix to try alternative table naming + $prefix = (string)($sourcePrefix ?? ''); + if ($prefix !== '') { + $alts[] = str_starts_with($candidate, $prefix) ? substr($candidate, strlen($prefix)) : ($prefix . $candidate); + } + $foundTable = null; + foreach (array_unique($alts) as $tbl) { + if ($conn->isMySQL()) { + $db = $conn->fetchOne('SELECT DATABASE()'); + $cols = $conn->fetchAllAssociative('SELECT column_name, data_type, column_type FROM information_schema.columns WHERE table_schema = ? AND table_name = ? ORDER BY ordinal_position', [$db, $tbl]); + if (!empty($cols)) { $columns = $cols; $foundTable = $tbl; break; } + } else { + // PostgreSQL: search in current schema(s) + $cols = $conn->fetchAllAssociative("SELECT column_name, data_type FROM information_schema.columns WHERE table_name = ? ORDER BY ordinal_position", [$tbl]); + if (!empty($cols)) { $columns = $cols; $foundTable = $tbl; break; } + } + } + if ($foundTable === null) { + $io->err('Target table not found in default database: ' . $targetTable); + return BaseCommand::CODE_ERROR; + } + // Build fieldMap using DB columns and source fields set + $fieldMap = []; + $booleanCols = []; + foreach ($columns as $col) { + $colName = $col['column_name']; + $fieldMap[$colName] = array_key_exists($colName, $sourceFieldSet) ? $colName : null; + // Detect booleans by database type + if ($conn->isMySQL()) { + $dt = strtolower((string)($col['data_type'] ?? '')); + $ct = strtolower((string)($col['column_type'] ?? '')); + $isBool = ($dt === 'tinyint' && str_starts_with($ct, 'tinyint(1)')) || ($dt === 'bit' && str_starts_with($ct, 'bit(1)')) || ($dt === 'boolean'); + if ($isBool) { $booleanCols[] = $colName; } + } else { + $dt = strtolower((string)($col['data_type'] ?? '')); + if ($dt === 'boolean' || $dt === 'bool') { $booleanCols[] = $colName; } + } + } + } catch (\Throwable $e) { + $io->err('Failed to inspect database schema: ' . $e->getMessage()); + return BaseCommand::CODE_ERROR; + } + + // Build a template strictly using the example shape. The target config is only used for the example; no values are copied. + // Default values: + // - source: from the provided --source-table option + // - displayField/addChangelog/sqlSelect/pre*/post*: null + // - booleans/cache: [] + // - fieldMap: constructed from DB columns and source schema + $defaults = [ + 'source' => $sourcePrefix . $sourceTable, + 'displayField' => $this->determineDisplayField($sourceFields), + 'addChangelog' => $this->shouldAddChangelog($srcTableCfg['source'] ?? $sourceTable, $sourceFields), + 'booleans' => $booleanCols, + 'cache' => [], + 'sqlSelect' => null, + 'preTable' => null, + 'postTable' => null, + 'preRow' => null, + 'postRow' => null, + 'fieldMap' => $fieldMap ?: new \stdClass(), + ]; + + // If example keys were found, limit output to those keys (plus ensure 'source' exists). + if (!empty($exampleKeys)) { + $template = []; + foreach ($exampleKeys as $k) { + if ($k === 'source') { + $template[$k] = $sourcePrefix . $sourceTable; + continue; + } + $template[$k] = $defaults[$k] ?? null; + } + // Ensure mandatory keys exist even if not in example + foreach (['source','fieldMap'] as $must) { + if (!array_key_exists($must, $template)) { + $template[$must] = $defaults[$must]; + } + } + } else { + // Fallback to standard template keys + $template = $defaults; + } + + // Ensure JSON encoding preserves empty objects properly for fieldMap + $json = json_encode($template, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES); + if ($json === false) { + $io->err('Failed to encode output JSON'); + return BaseCommand::CODE_ERROR; + } + + $io->out($json); + return BaseCommand::CODE_SUCCESS; + } + + /** + * Filter out documentation keys (starting with __) from configuration array + * + * @param array $cfg Configuration array to filter + * @return array Filtered configuration without documentation keys + */ + private function filterDocKeys(array $cfg): array + { + return array_filter($cfg, static function ($value, $key) { + return !(is_string($key) && str_starts_with($key, '__')); + }, ARRAY_FILTER_USE_BOTH); + } + + /** + * Decide if addChangelog should be true based on source configuration fields. + * Requirements: revision, deleted, actor_identifier, and self-referencing FK (eg, cous -> cou_id). + * + * @param string $tableName Name/key of the source table (may be prefixed like cm_) + * @param array $sourceFields List of source column names + */ + private function shouldAddChangelog(string $tableName, array $sourceFields): bool + { + if (empty($sourceFields)) { + return false; + } + $fields = array_map('strtolower', $sourceFields); + $fieldSet = array_flip($fields); + + // Required fixed columns + foreach (['revision','deleted','actor_identifier'] as $req) { + if (!array_key_exists($req, $fieldSet)) { + return false; + } + } + // Determine expected self FK + $name = strtolower($tableName); + $prefix = (string)($this->currentPrefix ?? ''); + if ($prefix !== '' && str_starts_with($name, $prefix)) { + $name = substr($name, strlen($prefix)); + } + // If table name is schema-qualified (eg, public.table), take part after dot + $dot = strrpos($name, '.'); + if ($dot !== false) { + $name = substr($name, $dot + 1); + } + $base = $name; + if (str_ends_with($base, 's')) { + $base = substr($base, 0, -1); + } + $selfFk = $base . '_id'; + + return array_key_exists($selfFk, $fieldSet); + } + + /** + * Normalize a loaded configuration array. + * - If it's a schema-like array (has 'table' nodes), convert to a transmogrify-like map + * keyed by table name with at least 'source' and 'fieldMap'. + */ + private function normalizeConfig(array $cfg): array + { + // Detect schema root + $tablesNode = null; + if (array_key_exists('table', $cfg)) { + $tablesNode = $cfg['table']; + } elseif (isset($cfg['schema']) && is_array($cfg['schema']) && array_key_exists('table', $cfg['schema'])) { + $tablesNode = $cfg['schema']['table']; + } + if ($tablesNode === null) { + // Already config-like + return $cfg; + } + // Ensure list + if (!is_array($tablesNode) || (is_array($tablesNode) && array_keys($tablesNode) !== range(0, count($tablesNode)-1))) { + // Single table object -> wrap + $tables = [$tablesNode]; + } else { + $tables = $tablesNode; + } + $out = []; + foreach ($tables as $t) { + if (!is_array($t)) { continue; } + // Name is typically under '@attributes' => ['name' => ...] + $name = null; + if (isset($t['@attributes']['name'])) { + $name = (string)$t['@attributes']['name']; + } elseif (isset($t['name'])) { + // Fallback if converter placed it differently + $name = is_array($t['name']) && isset($t['name']['@attributes']) ? (string)($t['name']['@attributes']['value'] ?? '') : (string)$t['name']; + } + if ($name === null || $name === '') { continue; } + + // Build simple 1:1 field map using elements, if present + $fieldMap = new \stdClass(); + if (isset($t['field'])) { + $fieldsNode = $t['field']; + // Normalize to list + if (!is_array($fieldsNode) || (is_array($fieldsNode) && array_keys($fieldsNode) !== range(0, count($fieldsNode)-1))) { + $fields = [$fieldsNode]; + } else { + $fields = $fieldsNode; + } + $map = []; + foreach ($fields as $f) { + if (!is_array($f)) { continue; } + $fname = $f['@attributes']['name'] ?? null; + if ($fname) { + $map[$fname] = $fname; + } + } + if (!empty($map)) { + $fieldMap = $map; + } + } + + $out[$name] = [ + 'source' => $name, + 'displayField' => $this->determineDisplayField(is_array($fieldMap) ? array_keys($fieldMap) : []), + 'fieldMap' => $fieldMap + ]; + } + return $out; + } + + /** + * Locate a table configuration by key or by matching the 'source' attribute. + * Also tries matching names with/without a leading provided prefix. + * + * @param array $cfg + * @param string $nameOrSource Either the config key or the source table name + * @return array{0:string|null,1:array|null} [key, config] + */ + private function findTableConfig(array $cfg, string $nameOrSource): array + { + if (array_key_exists($nameOrSource, $cfg) && is_array($cfg[$nameOrSource])) { + return [$nameOrSource, $cfg[$nameOrSource]]; + } + $prefix = (string)($this->currentPrefix ?? ''); + $alt = $nameOrSource; + if ($prefix !== '') { + $alt = str_starts_with($nameOrSource, $prefix) ? substr($nameOrSource, strlen($prefix)) : ($prefix . $nameOrSource); + } + if (array_key_exists($alt, $cfg) && is_array($cfg[$alt])) { + return [$alt, $cfg[$alt]]; + } + foreach ($cfg as $key => $val) { + if (!is_array($val)) { continue; } + $src = $val['source'] ?? null; + if ($src === $nameOrSource || $src === $alt) { + return [$key, $val]; + } + // Also allow key matching to bare table name + if ($key === $nameOrSource || $key === $alt) { + return [$key, $val]; + } + } + return [null, null]; + } + + /** + * Determine displayField from source table fields using provided order: + * name, display_name, username, authenticated_identifier, description, id. + * Returns null if none are present. + */ + private function determineDisplayField(array $sourceFields): ?string + { + if (empty($sourceFields)) { + return null; + } + $fields = array_map('strtolower', $sourceFields); + $set = array_flip($fields); + foreach (['name','display_name','username','authenticated_identifier','description','id'] as $candidate) { + if (array_key_exists($candidate, $set)) { + return $candidate; + } + } + return null; + } + + /** + * Print an ASCII association tree for a given starting table using the parsed XML schema array. + * Supports common schema.xml structures with elements containing @attributes["foreignTable"] + * and nested elements. + */ + private function printAssociationTreeFromSchema(array $schemaArr, string $startTable, ConsoleIo $io): void + { + // Build associative array tree from schema.xml-like array + // We expect tables under ['schema']['table'] or directly under ['table'] + $tablesNode = $schemaArr['schema']['table'] ?? ($schemaArr['table'] ?? []); + // Normalize to list of tables + if (!is_array($tablesNode) || (is_array($tablesNode) && array_keys($tablesNode) !== range(0, count($tablesNode)-1))) { + $tables = [$tablesNode]; + } else { + $tables = $tablesNode; + } + // Build map: tableName => [ children tables ... ] based on column @attributes['constraint'] value 'REFERENCE table(column)' + $children = []; + foreach ($tables as $t) { + if (!is_array($t)) { continue; } + $tbl = $t['@attributes']['name'] ?? ($t['name'] ?? null); + if ($tbl === null) { continue; } + $tbl = (string)$tbl; + if (!isset($children[$tbl])) { $children[$tbl] = []; } + // inspect columns/field definitions; schema.xml may use or + $colsNode = $t['column'] ?? ($t['field'] ?? []); + $cols = []; + if ($colsNode !== [] && $colsNode !== null) { + if (!is_array($colsNode) || (is_array($colsNode) && array_keys($colsNode) !== range(0, count($colsNode)-1))) { + $cols = [$colsNode]; + } else { + $cols = $colsNode; + } + } + foreach ($cols as $idx => $col) { + if (!is_array($col)) { continue; } + $constraint = $col['constraint'] ?? null; + if (!is_string($constraint)) { continue; } + // Expect format: 'REFERENCE table(column)' + if (preg_match('/^REFERENCES\s+([A-Za-z0-9_\.]+)\s*\(([^)]+)\)/', $constraint, $m)) { + $refTable = $m[1]; + // Remove prefix if it was provided and matches + if ($this->currentPrefix !== '' && str_starts_with($refTable, $this->currentPrefix)) { + $refTable = substr($refTable, strlen($this->currentPrefix)); + } + + // current table depends on refTable -> edge from current to parent + // For tree of ancestors starting from $startTable, we build parent map + // but for associative array tree representation, we'll build nested children where parent has child + // i.e., parent -> [ child1, child2 ] based on references found in child + if (!isset($children[$tbl])) { $children[$tbl] = []; } + if (!in_array($refTable, $children[$tbl], true)) { + $children[$tbl][] = $refTable; + } + } + } + } + // Build recursive associative array tree starting at $startTable + $visited = []; + $buildTree = function(string $node) use (&$buildTree, &$children, &$visited) { + if (isset($visited[$node])) { + // cycle: denote specially + return ['__cycle__' => $node]; + } + $visited[$node] = true; + $kids = $children[$node] ?? []; + $tree = []; + foreach ($kids as $k) { + $tree[$k] = $buildTree($k); + } + return $tree; + }; + $tree = [ $startTable => $buildTree($startTable) ]; + // Output the array via Console IO as JSON for readability + $io->out(json_encode($tree, JSON_PRETTY_PRINT)); + } + + private function printAncestorTree(string $node, array $parents, array &$visited, ConsoleIo $io, string $prefix): void + { + if (isset($visited[$node])) { + $io->out($prefix . '↺ ' . $node . ' (cycle)'); + return; + } + $visited[$node] = true; + $edges = $parents[$node] ?? []; + $count = count($edges); + foreach ($edges as $idx => $edge) { + $isLast = ($idx === $count - 1); + $branch = $isLast ? '└── ' : '├── '; + $nextPrefix = $prefix . ($isLast ? ' ' : '│ '); + $label = $edge['to'] ?? '?'; // parent table + $via = $edge['via'] ?? ''; + $io->out($prefix . $branch . $label . ($via !== '' ? (' [' . $via . ']') : '')); + $this->printAncestorTree((string)$label, $parents, $visited, $io, $nextPrefix); + } + } +} diff --git a/app/plugins/Transmogrify/src/Controller/AppController.php b/app/plugins/Transmogrify/src/Controller/AppController.php new file mode 100644 index 000000000..dabc4c59b --- /dev/null +++ b/app/plugins/Transmogrify/src/Controller/AppController.php @@ -0,0 +1,10 @@ +tables[$table]['cache'])) { + return; + } + + // Cache the requested fields. For now, at least, we key on row ID only. + foreach($this->tables[$table]['cache'] as $field) { + if(is_array($field)) { + // This is a list of fields, create a composite key that point to the row ID + + $label = ""; + $key = ""; + + foreach($field as $subfield) { + // eg: co_id+attribute+value+ + $label .= $subfield . "+"; + + // eg: 2+Identifier.type+eppn+ + $key .= $row[$subfield] . "+"; + } + + $this->cache[$table][$label][$key] = $row['id']; + } else { + // If the row has the field then map id to the requested field. + if(array_key_exists($field, $row)) { + $this->cache[$table]['id'][ $row['id'] ][$field] = $row[$field]; + } + } + } + } + + /** + * Find CO ID from related record data + * + * @param array $row Row data containing person_id, external_identity_id, group_id etc + * @return int Mapped CO ID + * @throws \InvalidArgumentException When CO not found + * @since COmanage Registry v5.0.0 + */ + protected function findCoId(array $row): int + { + // By the time we're called, we should have transmogrified the Org Identity + // and CO Person data, so we can just walk the caches + + if (isset($row['co_id'])) { + return (int)$row['co_id']; + } + + // Choose the resolution path by precedence using match(true) + $coId = match (true) { + isset($row['person_id']) => $this->getCoIdFromPersonId((int)$row['person_id']), + + // Map External Identity -> Person -> CO + isset($row['external_identity_id']) => $this->getCoIdFromExternalIdentityId((int)$row['external_identity_id']), + + isset($row['group_id']) => $this->getCoIdFromGroupId((int)$row['group_id']), + + // Legacy/preRow: org_identity_id follows the same External Identity path + isset($row['org_identity_id']) => $this->getCoIdFromExternalIdentityId((int)$row['org_identity_id']), + + isset($row['co_person_id']) => $this->getCoIdFromPersonId((int)$row['co_person_id']), + + default => null, + }; + + if ($coId !== null) { + return $coId; + } + + throw new \InvalidArgumentException('CO not found for record'); + } + + /** + * Resolve a CO ID from a CO Person ID via cache. + * + * @param int $personId + * @return int|null + */ + private function getCoIdFromPersonId(int $personId): ?int + { + if (isset($this->cache['people']['id'][$personId]['co_id'])) { + return (int)$this->cache['people']['id'][$personId]['co_id']; + } + return null; + } + + /** + * Resolve a CO Person ID from an External Identity (or legacy OrgIdentity) ID via cache. + * + * @param int $externalIdentityId + * @return int|null + */ + private function getPersonIdFromExternalIdentity(int $externalIdentityId): ?int + { + $personId = $this->cache['external_identities']['id'][$externalIdentityId]['person_id'] ?? null; + return $personId !== null ? (int)$personId : null; + } + + /** + * Resolve a CO ID from a Group ID via cache. + * + * @param int $groupId + * @return int|null + */ + private function getCoIdFromGroupId(int $groupId): ?int + { + if (isset($this->cache['groups']['id'][$groupId]['co_id'])) { + return (int)$this->cache['groups']['id'][$groupId]['co_id']; + } + return null; + } + + /** + * Resolve a CO ID starting from an External Identity (or legacy OrgIdentity) ID via cache. + * + * @param int $externalIdentityId + * @return int|null + */ + private function getCoIdFromExternalIdentityId(int $externalIdentityId): ?int + { + $personId = $this->getPersonIdFromExternalIdentity($externalIdentityId); + return $personId !== null ? $this->getCoIdFromPersonId($personId) : null; + } +} \ No newline at end of file diff --git a/app/plugins/Transmogrify/src/Lib/Traits/HookRunnersTrait.php b/app/plugins/Transmogrify/src/Lib/Traits/HookRunnersTrait.php new file mode 100644 index 000000000..888976053 --- /dev/null +++ b/app/plugins/Transmogrify/src/Lib/Traits/HookRunnersTrait.php @@ -0,0 +1,123 @@ +tables[$table]['preTable'])) { return; } + $method = $this->tables[$table]['preTable']; + if(!method_exists($this, $method)) { + throw new \RuntimeException("Unknown preTable hook: $method"); + } + $this->{$method}(); + } + + /** + * Run post-table hook if configured for given table + * + * @param string $table Table name + * @return void + * @throws \RuntimeException If hook method doesn't exist + * @since COmanage Registry v5.2.0 + */ + private function runPostTableHook(string $table): void { + if(empty($this->tables[$table]['postTable'])) { return; } + $method = $this->tables[$table]['postTable']; + if(!method_exists($this, $method)) { + throw new \RuntimeException("Unknown postTable hook: $method"); + } + $this->{$method}(); + } + + /** + * Run pre-row hook if configured for given table + * + * @param string $table Table name + * @param array $origRow Original row data by reference + * @param array $row Current row data by reference + * @return void + * @throws \RuntimeException If hook method doesn't exist + * @since COmanage Registry v5.2.0 + */ + private function runPreRowHook(string $table, array &$origRow, array &$row): void { + if(empty($this->tables[$table]['preRow'])) { return; } + $method = $this->tables[$table]['preRow']; + if(!method_exists($this, $method)) { + throw new \RuntimeException("Unknown preRow hook: $method"); + } + $this->{$method}($origRow, $row); + } + + /** + * Run post-row hook if configured for given table + * + * @param string $table Table name + * @param array $origRow Original row data by reference + * @param array $row Current row data by reference + * @return void + * @throws \RuntimeException If hook method doesn't exist + * @since COmanage Registry v5.2.0 + */ + private function runPostRowHook(string $table, array &$origRow, array &$row): void { + if(empty($this->tables[$table]['postRow'])) { return; } + $method = $this->tables[$table]['postRow']; + if(!method_exists($this, $method)) { + throw new \RuntimeException("Unknown postRow hook: $method"); + } + $this->{$method}($origRow, $row); + } + + /** + * Run SQL select hook if configured for given table + * + * @param string $table Table name + * @param string $qualifiedTableName Fully qualified table name + * @return string SQL select statement + * @throws \RuntimeException If hook method doesn't exist + * @since COmanage Registry v5.2.0 + */ + private function runSqlSelectHook(string $table, string $qualifiedTableName): string { + $method = $this->tables[$table]['sqlSelect'] ?? ''; + if($method === '') { + return RawSqlQueries::buildSelectAllOrderedById($qualifiedTableName); + } + if(!method_exists(RawSqlQueries::class, $method)) { + throw new \RuntimeException("Unknown sqlSelect hook: $method"); + } + return RawSqlQueries::{$method}($qualifiedTableName, $this->inconn->isMySQL()); + } +} \ No newline at end of file diff --git a/app/plugins/Transmogrify/src/Lib/Traits/ManageDefaultsTrait.php b/app/plugins/Transmogrify/src/Lib/Traits/ManageDefaultsTrait.php new file mode 100644 index 000000000..2afe257bf --- /dev/null +++ b/app/plugins/Transmogrify/src/Lib/Traits/ManageDefaultsTrait.php @@ -0,0 +1,141 @@ +get('Groups'); + + $iterator = new PaginatedSqlIterator($Groups, []); + + foreach($iterator as $k => $group) { + try { + // Because PaginatedSqlIterator will pick up new Groups as we create them, + // we need to check for any Owners groups (that we just created) and skip them. + if(!$group->isOwners()) { + $ownersGid = $Groups->createOwnersGroup($group); + + // We need to manually populate the cache + $this->cache['groups']['id'][$group->id]['owners_group_id'] = $ownersGid; + } + } + catch(\Exception $e) { + $this->io->error("Failed to create owners group for " + . $group->name . " (" . $group->id . "): " + . $e->getMessage()); + } + } + } + + /** + * Insert default CO Settings for COs that don't have settings. + * + * @since COmanage Registry v5.0.0 + * @return void + */ + protected function insertDefaultSettings(): void + { + // Create a CoSetting for any CO that didn't previously have one. + + $createdSettings = []; + $createdCos = array_keys($this->cache['cos']['id']); + + foreach($this->cache['co_settings']['id'] as $co_setting_id => $cached) { + $createdSettings[] = $cached['co_id']; + } + + $emptySettings = array_values(array_diff($createdCos, $createdSettings)); + + if(!empty($emptySettings)) { + $CoSettings = TableRegistry::getTableLocator()->get('CoSettings'); + + foreach($emptySettings as $coId) { + // Insert a default row into CoSettings for this CO ID + try { + $CoSettings->addDefaults($coId); + } catch (\ConflictException $e) { + // skip + } + } + } + } + + /** + * Insert default Pronoun types for all COs. + * + * @since COmanage Registry v5.0.0 + * @return void + */ + protected function insertPronounTypes(): void + { + // Since the Pronoun MVEA didn't exist in v4, we'll need to create the + // default types for all COs. + + $Types = TableRegistry::getTableLocator()->get('Types'); + + foreach(array_keys($this->cache['cos']['id']) as $coId) { + $Types->addDefault($coId, 'Pronouns.type'); + } + } + + /** + * Set a default value for CO Settings Permitted Telephone Number Fields. + * Returns CANE as the default permitted telephone number field value. + * + * @param array $row Row of table data + * @return string Default value CANE + * @since COmanage Registry v5.0.0 + */ + + protected function populateCoSettingsPhone(array $row): string + { + return PermittedTelephoneNumberFieldsEnum::CANE; + } +} diff --git a/app/plugins/Transmogrify/src/Lib/Traits/RowTransformationTrait.php b/app/plugins/Transmogrify/src/Lib/Traits/RowTransformationTrait.php new file mode 100644 index 000000000..e766b322d --- /dev/null +++ b/app/plugins/Transmogrify/src/Lib/Traits/RowTransformationTrait.php @@ -0,0 +1,388 @@ +cache['groups']['id'][ $origRow['co_group_id'] ]['owners_group_id'])) { + $ownerRow = [ + 'group_id' => $this->cache['groups']['id'][ $origRow['co_group_id'] ]['owners_group_id'], + 'person_id' => $origRow['co_person_id'], + 'created' => $origRow['created'], + 'modified' => $origRow['modified'], + 'group_member_id' => null, + 'revision' => 0, + 'deleted' => 'f', + 'actor_identifier' => $origRow['actor_identifier'] + ]; + + $tableName = 'group_members'; + $qualifiedTableName = $this->outconn->qualifyTableName($tableName); + $this->outconn->insert($qualifiedTableName, $ownerRow); + } else { + $this->io->error("Could not find owners group for CoGroupMember " . $origRow['id']); + } + } + + if(!$row['member'] && !$row['owner']) { + throw new \InvalidArgumentException('member not set on GroupMember'); + } + } + + /** + * Filter Jobs. + * + * @since COmanage Registry v5.0.0 + * @param array $origRow Row of table data (original data) + * @param array $row Row of table data (post fixes) + * @throws InvalidArgumentException + */ + protected function validateJobIsTransmogrifiable(array $origRow, array &$row): void + { + // We don't update any of the attributes, but for rows with unsupported data + // we throw an exception so they don't transmogrify. + + if($row['status'] == 'GO' || $row['status'] == 'Q') { + throw new \InvalidArgumentException("Job is Queued or In Progress"); + } + + if($row['job_type'] == 'EX' || $row['job_type'] == 'OS') { + throw new \InvalidArgumentException("Legacy Job types cannot be transmogrified"); + } + } + + /** + * Translate booleans to string literals to work around DBAL Postgres boolean handling. + * + * @since COmanage Registry v5.0.0 + * @param string $table Table Name + * @param array $row Row of attributes, fixed in place + */ + protected function normalizeBooleanFieldsForDb(string $table, array &$row): void + { + $attrs = ['deleted']; + + // We could introspect this from the schema file... + if(!empty($this->tables[$table]['booleans'])) { + $attrs = array_merge($attrs, $this->tables[$table]['booleans']); + } + + foreach($attrs as $a) { + if(isset($row[$a]) && gettype($row[$a]) == 'boolean') { + // DBAL Postgres boolean handling seems to be somewhat buggy, see history in + // this issue: https://github.com/doctrine/dbal/issues/1847 + // We need to (more generically than this hack) convert from boolean to char + // to avoid errors on insert + if($this->outconn->isMySQL()) { + $row[$a] = ($row[$a] ? '1' : '0'); + } else { + $row[$a] = ($row[$a] ? 't' : 'f'); + } + } + } + } + + /** + * Populate empty Changelog data from legacy records + * + * @since COmanage Registry v5.0.0 + * @param string $table Table Name + * @param array $row Row of attributes, fixed in place + * @param bool $force If true, always create keys + */ + protected function populateChangelogDefaults(string $table, array &$row, bool $force=false): void + { + if ($force || (array_key_exists('deleted', $row) && is_null($row['deleted']))) { + $row['deleted'] = false; + } + + if ($force || (array_key_exists('revision', $row) && is_null($row['revision']))) { + $row['revision'] = 0; + } + + if ($force || (array_key_exists('actor_identifier', $row) && is_null($row['actor_identifier']))) { + $row['actor_identifier'] = 'Transmogrification'; + } + } + + /** + * Map fields that have been renamed from Registry Classic to Registry PE. + * + * @since COmanage Registry v5.0.0 + * @param string $table Table Name + * @param array $row Row of attributes, fixed in place + * @throws InvalidArgumentException + */ + protected function mapLegacyFieldNames(string $table, array &$row): void + { + // oldname => newname, or &newname, which is a function to call. + // Note functions can return more than one mapping + + if(empty($this->tables[$table]['fieldMap'])) { + return; + } + + $fields = $this->tables[$table]['fieldMap']; + + foreach ($fields as $oldname => $newname) { + // Determine the first character only if $newname is a non-empty string + $first = is_string($newname) && $newname !== '' ? $newname[0] : null; + + match (true) { + // No mapping: unset the legacy field + !$newname => $this->performNoMapping($row, $oldname), + + // Function mapping: compute value by helper method named after the &-prefixed token + $first === '&' => $this->performFunctionMapping($row, $oldname, substr((string)$newname, 1), $table), + + // Default value mapping: set only if current value is null + $first === '?' => $this->applyDefaultIfNull($row, $oldname, substr((string)$newname, 1)), + + // Direct rename: copy to new name and unset the old one + default => $this->renameField($row, $oldname, (string)$newname), + }; + } + } + + /** + * Process Extended Attributes by converting them to Ad Hoc Attributes. + * + * @since COmanage Registry v5.0.0 + */ + protected function migrateExtendedAttributesToAdHocAttributes(): void + { + // This is intended to run AFTER AdHocAttributes so that we don't stomp on + // the row identifiers. + + // First, pull the old Extended Attribute configuration. + $extendedAttrs = []; + + $tableName = "cm_co_extended_attributes"; + $qualifiedTableName = $this->inconn->qualifyTableName($tableName); + $insql = RawSqlQueries::buildSelectAllOrderedById($qualifiedTableName); + $stmt = $this->inconn->query($insql); + + while($row = $stmt->fetch()) { + $extendedAttrs[ $row['co_id'] ][] = $row['name']; + } + + if(empty($extendedAttrs)) { + // No need to do anything further if no attributes are configured + return; + } + + foreach(array_keys($extendedAttrs) as $coId) { + $tableName = "cm_co" . $coId . "_person_extended_attributes"; + $qualifiedTableName = $this->inconn->qualifyTableName($tableName); + $insql = RawSqlQueries::buildSelectAll($qualifiedTableName); + $stmt = $this->inconn->query($insql); + + while($eaRow = $stmt->fetch()) { + // If we didn't transmogrify the parent row for some reason then trying + // to insert the ad_hoc_attributes will throw an error. + if(!empty($this->cache['person_roles']['id'][ $eaRow['co_person_role_id'] ])) { + foreach($extendedAttrs[$coId] as $ea) { + $adhocRow = [ + 'person_role_id' => $eaRow['co_person_role_id'], + 'tag' => $ea, + 'value' => $eaRow[$ea], + 'created' => $eaRow['created'], + 'modified' => $eaRow['modified'] + ]; + + // Extended Attributes were not changelog enabled + $this->populateChangelogDefaults('ad_hoc_attributes', $adhocRow, true); + $this->normalizeBooleanFieldsForDb('ad_hoc_attributes', $adhocRow); + + try { + $tableName = 'ad_hoc_attributes'; + $qualifiedTableName = $this->outconn->qualifyTableName($tableName); + $this->outconn->insert($qualifiedTableName, $adhocRow); + } catch (\Doctrine\DBAL\Exception\UniqueConstraintViolationException $e) { + $this->io->warning("record already exists: " . print_r($adhocRow, true)); + } + } + } + } + } + } + + /** + * Split an External Identity into an External Identity Role. + * + * @param array $origRow Row of table data (original data) + * @param array $row Row of table data (post fixes) + * @throws Exception + * @since COmanage Registry v5.0.0 + */ + + protected function mapExternalIdentityToExternalIdentityRole(array $origRow, array $row): void + { + // the original row has the co_id + $roleRow = []; + + // We could set the row ID to be the same as the original parent, but then + // we'd have to reset the sequence after the table is finished migrating. + + foreach([ + // Parent Key + 'id' => 'external_identity_id', + 'o' => 'organization', + 'ou' => 'department', + 'manager_identifier' => 'manager_identifier', + 'sponsor_identifier' => 'sponsor_identifier', + 'status' => 'status', + 'title' => 'title', + 'valid_from' => 'valid_from', + 'valid_through' => 'valid_through', + // Fix up changelog + 'org_identity_id' => 'external_identity_role_id', + 'revision' => 'revision', + 'deleted' => 'deleted', + 'actor_identifier' => 'actor_identifier', + 'created' => 'created', + 'modified' => 'modified' + ] as $oldKey => $newKey) { + $roleRow[$newKey] = $origRow[$oldKey]; + } + + if(!empty($origRow['affiliation'])) { + $row['affiliation'] = $origRow['affiliation']; + $roleRow['affiliation_type_id'] = $this->mapAffiliationType( + row: $row, + coId: $origRow['co_id'] ?? null + ); + } + + $tableName = 'external_identity_roles'; + + // Fix up changelog and booleans prior to insert + $this->populateChangelogDefaults($tableName, $roleRow, true); + $this->normalizeBooleanFieldsForDb($tableName, $roleRow); + + $qualifiedTableName = $this->outconn->qualifyTableName($tableName); + $this->outconn->insert($qualifiedTableName, $roleRow); + } + + /** + * Unset a legacy field when it has no mapping. + * + * @param array &$row Row data to modify + * @param string $oldname Name of field to remove + * @return void + */ + private function performNoMapping(array &$row, string $oldname): void + { + unset($row[$oldname]); + } + + /** + * Compute value for a field via a mapping function name (without the leading &). + * Reuses the original field name in-place. Throws if the mapping yields a falsy value. + * + * @param array &$row Row data to modify + * @param string $oldname Name of field to map + * @param string $funcName Name of mapping function to call + * @param string $table Table name for error reporting + * @return void + * @throws \InvalidArgumentException When mapping returns falsy value + */ + private function performFunctionMapping(array &$row, string $oldname, string $funcName, string $table): void + { + if (!method_exists($this, $funcName)) { + throw new \InvalidArgumentException("Mapping function {$funcName} does not exist for {$table} {$oldname}"); + } + + // We always pass the entire row so the mapping function can implement arbitrary logic + $row[$oldname] = $this->$funcName($row); + + // NOTE: mapAffiliationType can return null since extended types might not exist for deleted COs + if (!$row[$oldname] && $funcName !== 'mapAffiliationType') { + throw new \InvalidArgumentException("Could not find value for {$table} {$oldname}"); + } + } + + /** + * Apply a default value only when the current value is strictly null. + * + * @param array &$row Row data to modify + * @param string $oldname Name of field to check/update + * @param string $default Default value to apply if field is null + * @return void + */ + private function applyDefaultIfNull(array &$row, string $oldname, string $default): void + { + if (array_key_exists($oldname, $row) && $row[$oldname] === null) { + $row[$oldname] = $default; + } + } + + /** + * Rename a field by copying its value to a new key and removing the old key. + * + * @param array &$row Row data to modify + * @param string $oldname Original field name + * @param string $newname New field name + * @return void + */ + private function renameField(array &$row, string $oldname, string $newname): void + { + // Only copy if the old field exists to avoid notices + if (array_key_exists($oldname, $row)) { + $row[$newname] = $row[$oldname]; + unset($row[$oldname]); + } + } +} \ No newline at end of file diff --git a/app/plugins/Transmogrify/src/Lib/Traits/TypeMapperTrait.php b/app/plugins/Transmogrify/src/Lib/Traits/TypeMapperTrait.php new file mode 100644 index 000000000..4c87e65ec --- /dev/null +++ b/app/plugins/Transmogrify/src/Lib/Traits/TypeMapperTrait.php @@ -0,0 +1,324 @@ + + */ + + public const SERVER_TYPE_MAP = [ + 'HT' => 'CoreServer.HttpServers', + 'KA' => 'CoreServer.KafkaServerS', + 'KC' => 'CoreServer.KdcServerS', + 'LD' => 'CoreServer.LdapServerS', + 'MT' => 'CoreServer.MatchServerS', + 'O2' => 'CoreServer.Oauth2ServerS', + 'SQ' => 'CoreServer.SqlServerS', + ]; + + /** + * Map address type to corresponding type ID + * + * @param array $row Row data containing address type + * @return int Mapped type ID + * @since COmanage Registry v5.0.0 + */ + protected function mapAddressType(array $row): int + { + return $this->mapType($row, 'Addresses.type', $this->findCoId($row)); + } + + /** + * Map affiliation type to corresponding type ID + * + * @param array $row Row data containing affiliation + * @param int|null $coId CO Id or null if not known + * @return int|null Mapped type ID + * @since COmanage Registry v5.0.0 + */ + protected function mapAffiliationType(array $row, ?int $coId = null): ?int + { + return $this->mapType( + $row, + 'PersonRoles.affiliation_type', + $coId ?? $this->findCoId($row), + 'affiliation' + ); + } + + /** + * Map email type to corresponding type ID + * + * @param array $row Row data containing email type + * @return int Mapped type ID + * @since COmanage Registry v5.0.0 + */ + protected function mapEmailType(array $row): int + { + return $this->mapType($row, 'EmailAddresses.type', $this->findCoId($row)); + } + + /** + * Map an Extended Type attribute name for model name changes. + * + * @since COmanage Registry v5.0.0 + * @param array $row Row of table data + * @return string Updated attribute name + */ + + protected function mapExtendedType(array $row): string + { + switch($row['attribute']) { + case 'CoDepartment.type': + return 'Departments.type'; + case 'CoPersonRole.affiliation': + return 'PersonRoles.affiliation_type'; + } + + // For everything else, we need to pluralize the model name + $bits = explode('.', $row['attribute'], 2); + + return Inflector::pluralize($bits[0]) . "." . $bits[1]; + } + + /** + * Map identifier type to corresponding type ID + * + * @param array $row Row data containing identifier type + * @return int Mapped type ID + * @since COmanage Registry v5.0.0 + */ + protected function mapIdentifierType(array $row): int + { + return $this->mapType($row, 'Identifiers.type', $this->findCoId($row)); + } + + /** + * Map name type to corresponding type ID + * + * @param array $row Row data containing name type + * @return int Mapped type ID + * @since COmanage Registry v5.0.0 + */ + protected function mapNameType(array $row): int + { + return $this->mapType($row, 'Names.type', $this->findCoId($row)); + } + + /** + * Return a timestamp equivalent to now. + * + * @since COmanage Registry v5.0.0 + * @param array $row Row of table data (ignored) + * @return string Timestamp + */ + + protected function mapNow(array $row) { + if(empty($this->cache['now'])) { + $created = new \Datetime('now'); + $this->cache['now'] = $created->format('Y-m-d H:i:s'); + } + + return $this->cache['now']; + } + + /** + * Map an Org Identity ID to a CO Person ID + * + * @since COmanage Registry v5.0.0 + * @param array $row Row of Org Identity table data + * @return int|null CO Person ID + */ + protected function mapOrgIdentitycoPersonId(array $row): ?int + { + // PE eliminates OrgIdentityLink, so we need to map each Org Identity to + // a Person ID. Historically, an Org Identity could have been relinked and + // had multiple historical mappings. We now fetch only the current row, + // so we no longer need to track or select by revision. + + // Before Transmogrification, we require that Org Identities are unpooled. + // (This is probably how most deployments are set up, but there may be some + // legacy deployments out there.) This ensures whatever CO Person the Org + // Identity currently maps to through CoOrgIdentityLink is in the same CO. + + // There may be multiple mappings if the Org Identity was relinked. Basically + // we're going to lose the multiple mappings, since we can only return one + // value here. (Ideally, we would inject multiple OrgIdentities into the new + // table, but this ends up being rather tricky, since we have to figure out + // what row id to assign, and for the moment we don't have a mechanism to + // do that.) Historical information remains available in history_records, + // and if the deployer keeps an archive of the old database. + + $oid = (int)$row['id']; + + if(!isset($this->cache['org_identities']['co_people'])) { + $this->cache['org_identities']['co_people'] = []; + } + + // Guard 1: If we already have this org identity mapped, return immediately + if (isset($this->cache['org_identities']['co_people'][$oid])) { + return $this->cache['org_identities']['co_people'][$oid]; + } + + // Build cache on first use + $this->io->info('Populating org identity map...'); + + $tableName = 'cm_co_org_identity_links'; + $changelogFK = 'co_org_identity_link_id'; + $qualifiedTableName = $this->inconn->qualifyTableName($tableName); + + // Only fetch current rows (historical/deleted rows are filtered out) + $mapsql = RawSqlQueries::buildSelectAll( + qualifiedTableName: $qualifiedTableName, + onlyCurrent: true, + changelogFK: $changelogFK + ); + + $stmt = $this->inconn->executeQuery($mapsql); + + while ($r = $stmt->fetchAssociative()) { + if (!empty($r['org_identity_id'])) { + $rOid = (int)$r['org_identity_id']; + $cop = isset($r['co_person_id']) ? (int)$r['co_person_id'] : null; + + if ($cop === null) { + // Defensive: skip rows without a co_person_id + $this->io->warning('Org Identity ' . $rOid . ' has no mapped CO Person ID, skipping'); + continue; + } + + if (isset($this->cache['org_identities']['co_people'][$rOid])) { + // Unexpected duplicate "current" mapping, keep the first and warn + $this->io->warning('Found duplicate current CO Person mapping for Org Identity ' . $rOid . ', skipping'); + continue; + } + + $this->cache['org_identities']['co_people'][$rOid] = $cop; + } + } + + // Return the now-cached mapping (or null if not present) + return $this->cache['org_identities']['co_people'][$oid] ?? null; + } + + /** + * Map server type code to corresponding plugin path + * + * @param array $row Row data containing server type + * @return string|null Mapped plugin path or null if not found + * @since COmanage Registry v5.2.0 + */ + protected function mapServerTypeToPlugin(array $row): ?string + { + return (string)self::SERVER_TYPE_MAP[$row['server_type']] ?? null; + } + + /** + * Map telephone type to corresponding type ID + * + * @param array $row Row data containing telephone type + * @return int Mapped type ID + * @since COmanage Registry v5.0.0 + */ + protected function mapTelephoneType(array $row): int + { + return $this->mapType($row, 'TelephoneNumbers.type', $this->findCoId($row)); + } + + /** + * Map a type value to its corresponding ID + * + * @param array $row Row data containing the type value + * @param string $type Type attribute name + * @param int $coId CO ID + * @param string $attr Attribute name in row data + * @return int|null Mapped type ID + * @throws \InvalidArgumentException When type not found + * @since COmanage Registry v5.0.0 + */ + protected function mapType(array $row, string $type, int $coId, string $attr = 'type'): ?int + { + // If we delete a CO, we permanently delete the extended attributes. As a result, the type + // is no longer available because there is no changelog. Types are used by Person Roles and + // External Identity Roles. + if(!$coId) { + throw new \InvalidArgumentException("CO ID not provided for $type " . ($row['id'] ?? '')); + } + $value = $row[$attr] ?? null; + $key = $coId . "+" . $type . "+" . $value . "+"; + if(empty($this->cache['types']['co_id+attribute+value+'][$key])) { + if ( + !isset($this->cache['cos'][$coId]) + || ($this->cache['cos'][$coId]['status'] && $this->cache['cos'][$coId]['status'] == 'TR') + ) { + // This CO has been deleted, so we can't map the type. We will return null + return null; + } + throw new \InvalidArgumentException("Type not found for " . $key); + } + return (int)$this->cache['types']['co_id+attribute+value+'][$key]; + } + + /** + * Map URL type to corresponding type ID + * + * @param array $row Row data containing URL type + * @return int Mapped type ID + * @since COmanage Registry v5.0.0 + */ + protected function mapUrlType(array $row): int + { + return $this->mapType($row, 'Urls.type', $this->findCoId($row)); + } +} diff --git a/app/plugins/Transmogrify/src/Lib/Util/CommanLinePrinter.php b/app/plugins/Transmogrify/src/Lib/Util/CommanLinePrinter.php new file mode 100644 index 000000000..0df0a8fb8 --- /dev/null +++ b/app/plugins/Transmogrify/src/Lib/Util/CommanLinePrinter.php @@ -0,0 +1,184 @@ +io = $io; + $this->barColor = in_array($barColor, ['blue', 'green'], true) ? $barColor : 'blue'; + $this->barWidth = max(10, $barWidth); + $this->useColors = $useColors; + } + + /** Initialize the two-area layout and draw the 0% bar */ + public function start(int $total): void + { + $this->total = max(0, $total); + $this->current = 0; + $this->messageLines = 0; + + // Draw initial progress bar line (without trailing newline) + $this->rawWrite("\r" . $this->formatBar(0)); + + // Save cursor position at the end of the progress bar line + $this->rawWrite("\033[s"); + + // Move cursor back down ONLY if messages exist + if ($this->messageLines > 0) { + $this->rawWrite("\033[" . $this->messageLines . "B\r"); + } + } + + /** Update the progress bar (0..total). Call as work advances. */ + public function update(int $current): void + { + $this->current = min(max(0, $current), $this->total); + + // Restore to saved position (progress bar line), redraw, save again + $this->rawWrite("\033[u"); // restore saved cursor (bar line) + $this->rawWrite("\r" . $this->formatBar($this->current)); + $this->rawWrite("\033[s"); // re-save position at end of bar + + // Move cursor back down to where messages continue + if ($this->messageLines > 0) { + $this->rawWrite("\033[" . $this->messageLines . "B\r"); + } else { + // exactly one line below the bar is the first message line + $this->rawWrite("\033[1B\r"); + } + } + + /** Finish: force bar to 100% and add a newline separating it from any further output */ + public function finish(): void + { + $this->update($this->total); + // Move cursor to end of messages (already there), and ensure a newline after the last message block + $this->rawWrite(PHP_EOL); + } + + // Convenience wrappers for messages under the bar + public function info(string $message): void { $this->message($message, 'info'); } + public function warn(string $message): void { $this->message($message, 'warn'); } + public function error(string $message): void { $this->message($message, 'error'); } + public function debug(string $message): void { $this->message($message, 'debug'); } + + /** Print a message under the bar. Handles multi-line strings. */ + public function message(string $message, string $level = 'info'): void + { + $lines = $this->colorizeLevel($level, $message); + // Ensure the message ends with a newline so we can count line advances + if ($lines === '' || substr($lines, -1) !== "\n") { + $lines .= "\n"; + } + + // If this is the first message, create the message area by moving to the next line + if ($this->messageLines === 0) { + $this->rawWrite(PHP_EOL); + } + + $this->rawWrite($lines); + + // Count how many terminal lines were added based on newlines seen. + // Note: This doesn’t account for soft-wrapped lines; keep messages reasonably short. + $this->messageLines += substr_count($lines, "\n"); + } + + private function colorizeLevel(string $level, string $message): string + { + $level = strtolower($level); + $prefix = ''; + $color = null; + switch ($level) { + case 'warn': + case 'warning': + $prefix = '[WARN] '; + $color = 'yellow'; + break; + case 'error': + $prefix = '[ERROR] '; + $color = 'red'; + break; + case 'debug': + $prefix = '[DEBUG] '; + $color = 'cyan'; + break; + default: + $prefix = '[INFO] '; + $color = null; // default terminal color + } + + $text = $prefix . $message; + if ($this->useColors && $color) { + return $this->wrapColor($text, $color); + } + return $text; + } + + private function formatBar(int $current): string + { + $total = max(1, $this->total); + $pct = (int) floor(($current / $total) * 100); + + $width = $this->barWidth; + $filled = (int) floor(($pct / 100) * $width); + $remaining = max(0, $width - $filled); + + $filledStr = str_repeat('=', max(0, $filled - 1)) . ($filled > 0 ? '>' : ''); + $emptyStr = str_repeat('.', $remaining); + + $bar = sprintf('[%s%s] %3d%% (%d/%d)', $filledStr, $emptyStr, $pct, $current, $this->total); + + // Clear the line to the right to avoid remnants on shorter redraws + $bar .= "\033[K"; + + if ($this->useColors) { + $color = $this->barColor === 'green' ? 'green' : 'blue'; + $bar = $this->wrapColor($bar, $color); + } + return $bar; + } + + private function wrapColor(string $text, string $color): string + { + $map = [ + 'red' => '0;31', + 'green' => '0;32', + 'yellow' => '0;33', + 'blue' => '0;34', + 'magenta'=> '0;35', + 'cyan' => '0;36', + ]; + $code = $map[$color] ?? null; + if (!$code) { return $text; } + return "\033[{$code}m{$text}\033[0m"; + } + + private function rawWrite(string $str): void + { + if ($this->io) { + // ConsoleIo::out() defaults to a trailing newline; we want raw text + $this->io->out($str, 0); + } else { + // Fallback to STDOUT + echo $str; + } + } +} diff --git a/app/plugins/Transmogrify/src/Lib/Util/CommandLinePrinter.php b/app/plugins/Transmogrify/src/Lib/Util/CommandLinePrinter.php new file mode 100644 index 000000000..91f58f3fd --- /dev/null +++ b/app/plugins/Transmogrify/src/Lib/Util/CommandLinePrinter.php @@ -0,0 +1,203 @@ +io = $io; + $this->barColor = in_array($barColor, ['blue', 'green'], true) ? $barColor : 'blue'; + $this->barWidth = max(10, $barWidth); + $this->useColors = $useColors; + } + + /** + * Initialize or reinitialize the singleton and start the progress display. + */ + public static function initialize(?ConsoleIo $io, int $total, string $barColor = 'blue', int $barWidth = 50, bool $useColors = true): self + { + self::$instance = new self($io, $barColor, $barWidth, $useColors); + self::$instance->start($total); + return self::$instance; + } + + /** + * Get the current singleton instance. If not initialized, creates a default one. + */ + public static function get(): self + { + if (!self::$instance) { + self::$instance = new self(null, 'blue', 50, true); + } + return self::$instance; + } + + /** Initialize the two-area layout and draw the 0% bar */ + public function start(int $total): void + { + $this->total = max(0, $total); + $this->current = 0; + $this->messageLines = 0; + + // Draw initial progress bar line (without trailing newline) + $this->rawWrite("\r" . $this->formatBar(0)); + + // Save cursor position at the end of the progress bar line + $this->rawWrite("\033[s"); + + // Move to the line below where messages begin + $this->rawWrite(PHP_EOL); + } + + /** Update the progress bar (0..total). Call as work advances. */ + public function update(int $current): void + { + $this->current = min(max(0, $current), $this->total); + + // Restore to saved position (progress bar line), redraw, save again + $this->rawWrite("\033[u"); // restore saved cursor (bar line) + $this->rawWrite("\r" . $this->formatBar($this->current)); + $this->rawWrite("\033[s"); // re-save position at end of bar + + // Move cursor back down to where messages continue + if ($this->messageLines > 0) { + $this->rawWrite("\033[" . $this->messageLines . "B\r"); + } else { + // exactly one line below the bar is the first message line + $this->rawWrite("\033[1B\r"); + } + } + + /** Finish: force bar to 100% and add a newline separating it from any further output */ + public function finish(): void + { + $this->update($this->total); + // Move cursor to end of messages (already there), and ensure a newline after the last message block + $this->rawWrite(PHP_EOL); + } + + // Convenience wrappers for messages under the bar + public function info(string $message): void { $this->message($message, 'info'); } + public function warn(string $message): void { $this->message($message, 'warn'); } + public function error(string $message): void { $this->message($message, 'error'); } + public function debug(string $message): void { $this->message($message, 'debug'); } + + /** Print a message under the bar. Handles multi-line strings. */ + public function message(string $message, string $level = 'info'): void + { + $lines = $this->colorizeLevel($level, $message); + // Ensure the message ends with a newline so we can count line advances + if ($lines === '' || substr($lines, -1) !== "\n") { + $lines .= "\n"; + } + + $this->rawWrite($lines); + + // Count how many terminal lines were added based on newlines seen. + // Note: This doesn’t account for soft-wrapped lines; keep messages reasonably short. + $this->messageLines += substr_count($lines, "\n"); + } + + private function colorizeLevel(string $level, string $message): string + { + $level = strtolower($level); + $prefix = ''; + $color = null; + switch ($level) { + case 'warn': + case 'warning': + $prefix = '[WARN] '; + $color = 'yellow'; + break; + case 'error': + $prefix = '[ERROR] '; + $color = 'red'; + break; + case 'debug': + $prefix = '[DEBUG] '; + $color = 'cyan'; + break; + default: + $prefix = '[INFO] '; + $color = null; // default terminal color + } + + $text = $prefix . $message; + if ($this->useColors && $color) { + return $this->wrapColor($text, $color); + } + return $text; + } + + private function formatBar(int $current): string + { + $total = max(1, $this->total); + $pct = (int) floor(($current / $total) * 100); + + $width = $this->barWidth; + $filled = (int) floor(($pct / 100) * $width); + $remaining = max(0, $width - $filled); + + $filledStr = str_repeat('=', max(0, $filled - 1)) . ($filled > 0 ? '>' : ''); + $emptyStr = str_repeat('.', $remaining); + + $bar = sprintf('[%s%s] %3d%% (%d/%d)', $filledStr, $emptyStr, $pct, $current, $this->total); + + // Clear the line to the right to avoid remnants on shorter redraws + $bar .= "\033[K"; + + if ($this->useColors) { + $color = $this->barColor === 'green' ? 'green' : 'blue'; + $bar = $this->wrapColor($bar, $color); + } + return $bar; + } + + private function wrapColor(string $text, string $color): string + { + $map = [ + 'red' => '0;31', + 'green' => '0;32', + 'yellow' => '0;33', + 'blue' => '0;34', + 'magenta'=> '0;35', + 'cyan' => '0;36', + ]; + $code = $map[$color] ?? null; + if (!$code) { return $text; } + return "\033[{$code}m{$text}\033[0m"; + } + + private function rawWrite(string $str): void + { + if ($this->io) { + // ConsoleIo::out() defaults to a trailing newline; we want raw text + $this->io->out($str, 0); + } else { + // Fallback to STDOUT + echo $str; + } + } +} diff --git a/app/plugins/Transmogrify/src/Lib/Util/DbInfoPrinter.php b/app/plugins/Transmogrify/src/Lib/Util/DbInfoPrinter.php new file mode 100644 index 000000000..da6616291 --- /dev/null +++ b/app/plugins/Transmogrify/src/Lib/Util/DbInfoPrinter.php @@ -0,0 +1,246 @@ +io = $io; + $this->service = $service; + $this->pluginRoot = dirname(__DIR__, 3); + } + + /** + * Initializes the singleton instance + * @param ConsoleIo $io Console IO instance for output + * @param DbInfoService $service Database info service + * @return self The singleton instance + */ + public static function initialize(ConsoleIo $io, DbInfoService $service): self + { + if (self::$instance === null) { + self::$instance = new self($io, $service); + } + return self::$instance; + } + + + /** + * Prints database connection and schema information + * @param bool $asJson Whether to output as JSON + * @param bool $withPing Whether to test database connectivity + * @param bool $withSchema Whether to include schema information + * @param string|null $schemaRole Limit schema info to specific role + */ + public function print(bool $asJson, bool $withPing, bool $withSchema = false, ?string $schemaRole = null): void + { + $aliases = [ 'source' => 'transmogrify', 'target' => 'default' ]; + $data = []; + + foreach ($aliases as $role => $alias) { + // Base connection info + $info = $this->service->getConnectionInfo($role); + + if ($withPing) { + $info['status'] = $this->service->ping($alias); + } + + if ($withSchema && ($schemaRole === null || $schemaRole === $role)) { + try { + + $info['schema'] = $this->service->loadSchemaInfo( + $alias, + $this->pluginRoot . DS . Transmogrify::TABLES_JSON_PATH); + } catch (\Throwable $e) { + $info['schema'] = [ 'error' => $e->getMessage() ]; + } + } + + $data[$role] = $info; + } + + if ($asJson) { + $this->io->out(json_encode($data, JSON_PRETTY_PRINT)); + return; + } + + foreach (['source', 'target'] as $role) { + $i = $data[$role] ?? []; + $header = ucfirst($role) . ':'; + $this->io->out(self::STYLE_BOLD . $header . self::COLOR_RESET); + $this->io->out(str_repeat('-', strlen($header))); + $this->io->out(' alias: ' . ($i['alias'] ?? '')); + $this->io->out(' configured:' . ((($i['configured'] ?? false)) ? ' yes' : ' no')); + if (!empty($i['driver'])) $this->io->out(' driver: ' . $i['driver']); + if (!empty($i['host'])) $this->io->out(' host: ' . $i['host']); + if (!empty($i['port'])) $this->io->out(' port: ' . $i['port']); + if (!empty($i['database'])) $this->io->out(' database: ' . $i['database']); + if (!empty($i['username'])) $this->io->out(' username: ' . $i['username']); + if (!empty($i['dsn'])) $this->io->out(' dsn: ' . $i['dsn']); + if (!empty($i['schema'])) { + // Determine a schema name (for PostgreSQL) and strip schema prefixes for sample tables display + $schemaName = null; + $bareSample = []; + $s = $i['schema']; + if (!empty($s['sample_tables'])) { + foreach ($s['sample_tables'] as $t) { + $dot = strrpos($t, '.'); + if ($dot !== false) { + $schemaName = $schemaName ?? substr($t, 0, $dot); + $bareSample[] = substr($t, $dot + 1); + } else { + $bareSample[] = $t; + } + } + } + $s = $i['schema']; + $schemaHeader = 'Schema:'; + $this->io->out(' ' . self::STYLE_BOLD . $schemaHeader . self::COLOR_RESET); + $this->io->out(' ' . str_repeat('-', strlen($schemaHeader))); + if (!empty($schemaName)) { + $this->io->out(' name: ' . $schemaName); + } + $this->io->out(' empty: ' . (($s['empty'] ?? false) ? 'yes' : 'no')); + $this->io->out(' tables: ' . ($s['table_count'] ?? 0)); + if (!empty($s['sample_tables'])) { + $sampleHeader = 'Sample tables:'; + $this->io->out(' ' . self::STYLE_BOLD . $sampleHeader . self::COLOR_RESET); + $this->io->out(' ' . str_repeat('-', strlen($sampleHeader))); + $list = !empty($bareSample) ? $bareSample : $s['sample_tables']; + foreach ($list as $t) { $this->io->out(' - ' . $t); } + } + if (!empty($s['tables_compare'])) { + $cmp = $s['tables_compare']; + // Add a blank line then print comparison headers at top level (aligned with SOURCE/TARGET) + $this->io->out(''); + $header = 'Tables present (json ∧ db):'; + $this->io->out(self::STYLE_BOLD . $header . self::COLOR_RESET); + $this->io->out(str_repeat('-', strlen($header))); + // Tables present in both JSON and DB + foreach (($cmp['both'] ?? []) as $t) { + $this->io->out(' ' . self::COLOR_GREEN . '✔' . self::COLOR_RESET . ' ' . $t); + } + // Only declared in tables.json + if (!empty($cmp['only_in_json'])) { + $onlyJsonHeader = 'Only in tables.json:'; + $this->io->out(self::STYLE_BOLD . $onlyJsonHeader . self::COLOR_RESET); + $this->io->out(str_repeat('-', strlen($onlyJsonHeader))); + foreach ($cmp['only_in_json'] as $t) { $this->io->out(' - ' . $t); } + } + // Present only in the database (render in 3 columns) + if (!empty($cmp['only_in_db'])) { + $onlyDbHeader = 'Only in database:'; + $this->io->out(self::STYLE_BOLD . $onlyDbHeader . self::COLOR_RESET); + $this->io->out(str_repeat('-', strlen($onlyDbHeader))); + $list = array_values($cmp['only_in_db']); + $cols = 3; + $rows = (int)ceil(count($list) / $cols); + // Determine max width per column + $widths = array_fill(0, $cols, 0); + for ($c = 0; $c < $cols; $c++) { + for ($r = 0; $r < $rows; $r++) { + $idx = $r + $rows * $c; + if ($idx < count($list)) { + $len = strlen((string)$list[$idx]); + if ($len > $widths[$c]) { $widths[$c] = $len; } + } + } + } + // Print rows + for ($r = 0; $r < $rows; $r++) { + $line = ' '; + for ($c = 0; $c < $cols; $c++) { + $idx = $r + $rows * $c; + if ($idx < count($list)) { + $cell = (string)$list[$idx]; + // No padding after last printed column + if ($c === $cols - 1 || ($r + $rows * ($c + 1)) >= count($list)) { + $line .= $cell; + } else { + $line .= str_pad($cell, $widths[$c]) . ' '; + } + } + } + $this->io->out($line); + } + } + } + } + + if (isset($i['status'])) { + $st = $i['status']; + + $this->io->out(' connectivity: ' . ($st['ok'] + ? self::COLOR_GREEN . '✔ OK' . self::COLOR_RESET + : self::COLOR_RED . '✘ ERROR' . self::COLOR_RESET)); + if (!empty($st['server'])) $this->io->out(' server: ' . $st['server']); + if (!$st['ok'] && !empty($st['error'])) $this->io->out(' error: ' . $st['error']); + } + $this->io->out(''); + } + } + +} diff --git a/app/plugins/Transmogrify/src/Lib/Util/RawSqlQueries.php b/app/plugins/Transmogrify/src/Lib/Util/RawSqlQueries.php new file mode 100644 index 000000000..6ff0bffdd --- /dev/null +++ b/app/plugins/Transmogrify/src/Lib/Util/RawSqlQueries.php @@ -0,0 +1,421 @@ +qualifyTableName($sourceTable); + $maxId = $inconn->fetchOne(self::buildSelectMaxId($qualifiedTableName)); + $maxId = ((int)($maxId ?? 0)) + 1; + + $qualifiedTableName = $outconn->qualifyTableName($targetTable); + $cio->info("Resetting primary key sequence for $qualifiedTableName to $maxId"); + + // Strictly speaking we should use prepared statements, but we control the + // data here, and also we're executing a maintenance operation (so query + // optimization is less important). + $outsql = RawSqlQueries::buildSequenceReset($qualifiedTableName, $maxId, $outconn->isMySQL()); + try { + $outconn->executeQuery($outsql); + } catch (\Exception $e) { + return false; + } + + return true; + } + + /** + * Return SQL used to select COUs from inbound database. + * + * @since COmanage Registry v5.0.0 + * @param string $tableName Name of the SQL table + * @param bool $isMySQL Whether the database is MySQL + * @return string SQL string to select rows from inbound database + */ + + public static function couSqlSelect(string $tableName, bool $isMySQL): string { + if($isMySQL) { + $sqlTemplate = RawSqlQueries::COU_SQL_SELECT_TEMPLATE_MYSQL; + } else { + $sqlTemplate = RawSqlQueries::COU_SQL_SELECT_TEMPLATE_POSTGRESQL; + } + + return str_replace('{table}', $tableName, $sqlTemplate); + } + + /** + * Return SQL used to select COUs from inbound database. + * + * @since COmanage Registry v5.0.0 + * @param string $tableName Name of the SQL table + * @param bool $isMySQL Whether the database is MySQL + * @return string SQL string to select rows from inbound database + */ + + public static function roleSqlSelect(string $tableName, bool $isMySQL): string { + return RawSqlQueries::ROLE_SQL_SELECT; + } + + // Any COU at any time can be made the child of another COU + // and so during transmogrification we cannot simply select + // the rows of the COU table by ascending id because it leads + // to foreign key constraints errors since a parent with a larger + // value for id may not be in the outbound table when a child COU + // is processed. + // + // Instead we need to order the rows for the COU inbound table by + // generation starting with generation 0 which has no parents. + // To do this we use a Common Table Expression (CTE), specifically + // WITH RECURSIVE. See https://www.postgresql.org/docs/current/queries-with.html#QUERIES-WITH-RECURSIVE + // for PostgreSQL and https://dev.mysql.com/doc/refman/8.0/en/with.html#common-table-expressions-recursive + // for MySQL. This is now a standard technique for sorting hierarchical or tree-structured + // data. + // + // Our need is more complicated than the standard example because in addition to the + // column parent_id our table also has the column cou_id used by ChangelogBehavior + // as a foreign key back to id. Because of this any row may appear more than once + // in the final intermediate table computed during recursion. We handle this by using + // GROUP BY id in the final SELECT and then using aggregate functions for all columns except for + // id. + // + // Unfortunately PostgreSQL and MySQL do not define the same aggregate functions so we need a unique + // SQL template for each below. + + /** + * MySQL template for recursive CTE query to select COU records ordered by generation + * Uses GROUP_CONCAT for string aggregation + */ + final const COU_SQL_SELECT_TEMPLATE_MYSQL = <<plugin( + 'Transmogrify', + ['path' => '/transmogrify'], + function (RouteBuilder $builder) { + // Add custom routes here + + $builder->fallbacks(); + } + ); + parent::routes($routes); + } + + /** + * Add middleware for the plugin. + * + * @param \Cake\Http\MiddlewareQueue $middlewareQueue The middleware queue to update. + * @return \Cake\Http\MiddlewareQueue + */ + public function middleware(MiddlewareQueue $middlewareQueue): MiddlewareQueue + { + // Add your middlewares here + // remove this method hook if you don't need it + + return $middlewareQueue; + } + + /** + * Add commands for the plugin. + * + * @param \Cake\Console\CommandCollection $commands The command collection to update. + * @return \Cake\Console\CommandCollection + */ + public function console(CommandCollection $commands): CommandCollection + { + $commands->add('transmogrify', TransmogrifyCommand::class); + $commands->add('transmogrify source-to-target', TransmogrifySourceToTargetCommand::class); + + return parent::console($commands); + } + + /** + * Register application container services. + * + * @param \Cake\Core\ContainerInterface $container The Container to update. + * @return void + * @link https://book.cakephp.org/5/en/development/dependency-injection.html#dependency-injection + */ + public function services(ContainerInterface $container): void + { + // Add your services here + // remove this method hook if you don't need it + $container->add(TransmogrifyCommand::class) + ->addArguments([DbInfoService::class, ConfigLoaderService::class]); + $container->add(TransmogrifySourceToTargetCommand::class) + ->addArgument(ConfigLoaderService::class); + } +} diff --git a/app/plugins/Transmogrify/webroot/.gitkeep b/app/plugins/Transmogrify/webroot/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/app/src/Application.php b/app/src/Application.php index fbed5ede5..1e57c5454 100644 --- a/app/src/Application.php +++ b/app/src/Application.php @@ -16,7 +16,10 @@ */ namespace App; +use App\Service\ConfigLoaderService; +use App\Service\DbInfoService; use Cake\Core\Configure; +use Cake\Core\ContainerInterface; use Cake\Core\Exception\MissingPluginException; use Cake\Error\Middleware\ErrorHandlerMiddleware; use Cake\Http\BaseApplication; @@ -124,4 +127,20 @@ protected function bootstrapCli(): void // Load more plugins here } + + + /** + * Register application container services. + * + * @param \Cake\Core\ContainerInterface $container The Container to update + * @return void + */ + public function services(ContainerInterface $container): void + { + parent::services($container); + + // Register services so the container can resolve them (constructor autowiring) + $container->add(DbInfoService::class); + $container->add(ConfigLoaderService::class); + } } diff --git a/app/src/Command/TransmogrifyCommand.php b/app/src/Command/TransmogrifyCommand.php deleted file mode 100644 index a3d24f886..000000000 --- a/app/src/Command/TransmogrifyCommand.php +++ /dev/null @@ -1,1388 +0,0 @@ - [ - 'source' => 'cm_cos', - 'displayField' => 'name', - 'addChangelog' => true, - // We don't really need status, but we need something cached for co_settings - 'cache' => [ 'status' ] - ], - 'types' => [ - 'source' => 'cm_co_extended_types', - 'displayField' => 'display_name', - 'postTable' => 'insertPronounTypes', - 'fieldMap' => [ - 'attribute' => '&map_extended_type', - 'name' => 'value', - // For some reason, cm_co_extended_types never had created/modified metadata - 'created' => '&map_now', - 'modified' => '&map_now' - ], - 'cache' => [ [ 'co_id', 'attribute', 'value' ] ] - ], - 'co_settings' => [ - 'source' => 'cm_co_settings', - 'displayField' => 'co_id', - 'addChangelog' => true, - 'booleans' => [], - 'postTable' => 'insertDefaultSettings', - 'cache' => [ 'co_id' ], - 'fieldMap' => [ - 'global_search_limit' => 'search_global_limit', - 'required_fields_addr' => 'required_fields_address', - 'permitted_fields_telephone_number' => '&populate_co_settings_phone', - // XXX CFM-80 these fields are not yet migrated - // be sure to add appropriate fields to 'booleans' - 'enable_nsf_demo' => null, // CFM-123 - 'disable_expiration' => null, - 'disable_ois_sync' => null, - 'group_validity_sync_window' => null, - 'garbage_collection_interval' => null, - 'enable_normalization' => null, - 'enable_empty_cou' => null, - 'invitation_validity' => null, - 't_and_c_login_mode' => null, - 'sponsor_eligibility' => null, - 'sponsor_co_group_id' => null, - 'theme_stacking' => null, - 'default_co_pipeline_id' => null, // XXX was this ever used? - 'elect_strategy_primary_name' => null, - 'co_dashboard_id' => null, - 'co_theme_id' => null, - 'person_picker_email_type' => null, - 'person_picker_identifier_type' => null, - 'person_picker_display_types' => null, - // No longer supported in PE, see CFM-316 - 'group_create_admin_only' => null - ] - ], - 'authentication_events' => [ - 'source' => 'cm_authentication_events', - 'displayField' => 'authenticated_identifier' - ], - 'api_users' => [ - 'source' => 'cm_api_users', - 'displayField' => 'username', - 'booleans' => [ 'privileged' ], - 'fieldMap' => [ - 'password' => 'api_key' - ] - ], - 'cous' => [ - 'source' => 'cm_cous', - 'displayField' => 'name', - 'sqlSelect' => 'couSqlSelect' - ], - //'dashboards' => [ 'source' => 'cm_co_dashboards' ] - 'people' => [ - 'source' => 'cm_co_people', - 'displayField' => 'id', - 'cache' => [ 'co_id' ], - 'fieldMap' => [ - // Rename the changelog key - 'co_person_id' => 'person_id' - ] - ], - 'person_roles' => [ - 'source' => 'cm_co_person_roles', - 'sqlSelect' => 'roleSqlSelect', - 'displayField' => 'id', - // We don't currently need status specifically, just that the role exists - 'cache' => [ 'status' ], - 'fieldMap' => [ - 'co_person_id' => 'person_id', - // Rename the changelog key - 'co_person_role_id' => 'person_role_id', - // We need to map affiliation_type_id before we null out affiliation - 'affiliation_type_id' => '&map_affiliation_type', - 'affiliation' => null, - 'manager_co_person_id' => 'manager_person_id', - 'sponsor_co_person_id' => 'sponsor_person_id', - 'o' => 'organization', - 'ou' => 'department', -// XXX temporary until tables are migrated - 'source_org_identity_id' => null - ] - ], - 'external_identities' => [ - 'source' => 'cm_org_identities', - 'displayField' => 'id', - 'fieldMap' => [ - 'co_id' => null, - 'person_id' => '&map_org_identity_co_person_id', - // Rename the changelog key - 'org_identity_id' => 'external_identity_id', - // These fields are migrated to external_identity_roles by split_external_identity() - 'title' => null, - 'o' => null, - 'ou' => null, - 'affiliation' => null, - 'manager_identifier' => null, - 'sponsor_identifier' => null, - 'valid_from' => null, - 'valid_through' => null - ], - 'postRow' => 'split_external_identity', - 'cache' => [ 'person_id' ] - ], - 'groups' => [ - 'source' => 'cm_co_groups', - 'displayField' => 'name', - 'cache' => [ 'co_id', 'owners_group_id' ], - 'booleans' => [ 'nesting_mode_all', 'open' ], - 'fieldMap' => [ - // auto is implied by group_type - 'auto' => null, - // Rename the changelog key - 'co_group_id' => 'group_id', - // Make sure group_type is populated if not already set - 'group_type' => '?S' - ], - 'postTable' => 'createOwnersGroups' - ], - 'group_nestings' => [ - 'source' => 'cm_co_group_nestings', - 'displayField' => 'id', - 'booleans' => [ 'negate' ], - 'fieldMap' => [ - 'co_group_id' => 'group_id', - 'target_co_group_id' => 'target_group_id', - // Rename the changelog key - 'co_group_nesting_id' => 'group_nesting_id' - ] - ], - 'group_members' => [ - 'source' => 'cm_co_group_members', - 'displayField' => 'id', - 'booleans' => [ 'member', 'owner' ], - 'fieldMap' => [ - 'co_group_id' => 'group_id', - 'co_person_id' => 'person_id', - 'member' => null, - 'owner' => null, - 'co_group_nesting_id' => 'group_nesting_id', - // Rename the changelog key - 'co_group_member_id' => 'group_member_id', - // Temporary until implemented - 'source_org_identity_id' => null - ], - 'preRow' => 'check_group_membership' - ], - 'names' => [ - 'source' => 'cm_names', - 'displayField' => 'id', - 'booleans' => [ 'primary_name' ], - 'fieldMap' => [ - 'co_person_id' => 'person_id', - 'org_identity_id' => 'external_identity_id', - // We need to map type_id before we null out type - 'type_id' => '&map_name_type', - 'type' => null - ] - ], - 'ad_hoc_attributes' => [ - 'source' => 'cm_ad_hoc_attributes', - 'displayField' => 'id', - 'fieldMap' => [ - 'co_person_role_id' => 'person_role_id', - 'org_identity_id' => 'external_identity_id', -// XXX temporary until tables are migrated - 'co_department_id' => null, - 'organization_id' => null - ], - 'postTable' => 'processExtendedAttributes' - ], - 'addresses' => [ - 'source' => 'cm_addresses', - 'displayField' => 'id', - 'fieldMap' => [ - 'co_person_role_id' => 'person_role_id', - 'org_identity_id' => 'external_identity_id', - 'type_id' => '&map_address_type', - 'type' => null, -// XXX temporary until tables are migrated - 'co_department_id' => null, - 'organization_id' => null - ] - ], - 'email_addresses' => [ - 'source' => 'cm_email_addresses', - 'displayField' => 'id', - 'booleans' => [ 'verified' ], - 'fieldMap' => [ - 'co_person_id' => 'person_id', - 'org_identity_id' => 'external_identity_id', - 'type_id' => '&map_email_type', - 'type' => null, -// XXX temporary until tables are migrated - 'co_department_id' => null, - 'organization_id' => null - ] - ], - 'identifiers' => [ - 'source' => 'cm_identifiers', - 'displayField' => 'id', - 'booleans' => [ 'login' ], - 'fieldMap' => [ - 'co_group_id' => 'group_id', - 'co_person_id' => 'person_id', - 'org_identity_id' => 'external_identity_id', - 'type_id' => '&map_identifier_type', - 'type' => null, -// XXX temporary until tables are migrated - 'co_department_id' => null, - 'co_provisioning_target_id' => null, - 'organization_id' => null - ], - 'preRow' => 'map_login_identifiers' - ], - 'telephone_numbers' => [ - 'source' => 'cm_telephone_numbers', - 'displayField' => 'id', - 'fieldMap' => [ - 'co_person_role_id' => 'person_role_id', - 'org_identity_id' => 'external_identity_id', - 'type_id' => '&map_telephone_type', - 'type' => null, -// XXX temporary until tables are migrated - 'co_department_id' => null, - 'organization_id' => null - ] - ], - 'urls' => [ - 'source' => 'cm_urls', - 'displayField' => 'id', - 'fieldMap' => [ - 'co_person_id' => 'person_id', - 'org_identity_id' => 'external_identity_id', - 'type_id' => '&map_url_type', - 'type' => null, -// XXX temporary until tables are migrated - 'co_department_id' => null, - 'organization_id' => null - ] - ], - 'history_records' => [ - 'source' => 'cm_history_records', - 'displayField' => 'id', - 'fieldMap' => [ - 'actor_co_person_id' => 'actor_person_id', - 'co_person_id' => 'person_id', - 'co_person_role_id' => 'person_role_id', - 'co_group_id' => 'group_id', - 'org_identity_id' => 'external_identity_id', -// XXX temporary until tables are migrated - 'co_email_list_id' => null, - 'co_service_id' => null - ] - ], - 'jobs' => [ - 'source' => 'cm_co_jobs', - 'displayField' => 'id', - 'fieldMap' => [ - 'job_type' => 'plugin', - 'job_mode' => null, - 'queue_time' => 'register_time', - 'complete_time' => 'finish_time', - 'job_type_fk' => null, - 'job_params' => 'parameters', - 'requeued_from_co_job_id' => 'requeued_from_job_id', - // XXX CFM-246 not yet supported - 'max_retry' => null, - 'max_retry_count' => null - ], - 'preRow' => 'filterJobs' - ], - 'job_history_records' => [ - 'source' => 'cm_co_job_history_records', - 'displayField' => 'id', - 'fieldMap' => [ - 'co_job_id' => 'job_id', - 'co_person_id' => 'person_id', - 'org_identity_id' => 'external_identity_id' - ] - ] - ]; - - // Table specific field mapping cache - protected $cache = []; - - // Make some objects more easily accessible - protected $inconn = null; - protected $outconn = null; - - // Shell arguments, for easier access - protected $args = null; - protected $io = null; - - /** - * Build an Option Parser. - * - * @since COmanage Registry v5.0.0 - * @param ConsoleOptionParser $parser ConsoleOptionParser - * @return ConsoleOptionParser ConsoleOptionParser - */ - - protected function buildOptionParser(ConsoleOptionParser $parser): ConsoleOptionParser { - $parser->addOption('login-identifier-copy', [ - 'help' => __d('command', 'tm.login-identifier-copy'), - 'boolean' => true - ]); - - $parser->addOption('login-identifier-type', [ - 'help' => __d('command', 'tm.login-identifier-type') - ]); - - $parser->setEpilog(__d('command', 'tm.epilog')); - - return $parser; - } - - /** - * Cache results as configured for the specified table. - * - * @since COmanage Registry v5.0.0 - * @param string $table Table to cache - * @param array $row Row of table data - */ - - protected function cacheResults(string $table, array $row) { - if(!empty($this->tables[$table]['cache'])) { - // Cache the requested fields. For now, at least, we key on row ID only. - foreach($this->tables[$table]['cache'] as $field) { - if(is_array($field)) { - // This is a list of fields, create a composite key that point to the row ID - - $label = ""; - $key = ""; - - foreach($field as $subfield) { - // eg: co_id+attribute+value+ - $label .= $subfield . "+"; - - // eg: 2+Identifier.type+eppn+ - $key .= $row[$subfield] . "+"; - } - - $this->cache[$table][$label][$key] = $row['id']; - } else { - // If the row has the field then map id to the requested field. - if(array_key_exists($field, $row)) { - $this->cache[$table]['id'][ $row['id'] ][$field] = $row[$field]; - } - } - } - } - } - - /** - * Check if a group membership is actually asserted, and reassign ownerships. - * - * @since COmanage Registry v5.0.0 - * @param array $origRow Row of table data (original data) - * @param array $row Row of table data (post fixes) - * @throws InvalidArgumentException - */ - - protected function check_group_membership(array $origRow, array $row) { - // We need to handle the various member+owner scenarios, but basically - // (1) If 'owner' is set, manually create a Group Membership in the appropriate - // Owners Group (we need to be called via preRow to do this) - // (2) If 'member' is NOT set, throw an exception so we don't create - // in invalid membership - // (3) Otherwise just return so the Membership gets created - - if($origRow['owner'] && !$origRow['deleted'] && !$origRow['co_group_member_id']) { - // Create a membership in the appropriate owners group, but not - // on changelog entries - - if(!empty($this->cache['groups']['id'][ $origRow['co_group_id'] ]['owners_group_id'])) { - $ownerRow = [ - 'group_id' => $this->cache['groups']['id'][ $origRow['co_group_id'] ]['owners_group_id'], - 'person_id' => $origRow['co_person_id'], - 'created' => $origRow['created'], - 'modified' => $origRow['modified'], - 'group_member_id' => null, - 'revision' => 0, - 'deleted' => 'f', - 'actor_identifier' => $origRow['actor_identifier'] - ]; - - $tableName = 'group_members'; - $qualifiedTableName = $this->outconn->qualifyTableName($tableName); - $this->outconn->insert($qualifiedTableName, $ownerRow); - } else { - $this->io->error("Could not find owners group for CoGroupMember " . $origRow['id']); - } - } - - if(!$row['member'] && !$row['owner']) { - throw new \InvalidArgumentException('member not set on GroupMember'); - } - } - - /** - * Return SQL used to select COUs from inbound database. - * - * @since COmanage Registry v5.0.0 - * @param string $tableName Name of the SQL table - * @return string SQL string to select rows from inbound database - */ - - protected function couSqlSelect(string $tableName): string { - if($this->inconn->isMySQL()) { - $sqlTemplate = TransmogrifyUtilities::COU_SQL_SELECT_TEMPLATE_MYSQL; - } else { - $sqlTemplate = TransmogrifyUtilities::COU_SQL_SELECT_TEMPLATE_POSTGRESQL; - } - - $sql = str_replace('{table}', $tableName, $sqlTemplate); - - return $sql; - } - - /** - * Create an Owners Group for an existing Group. - * - * @since COmanage Registry v5.0.0 - * @param array $origRow Row of table data (original data) - * @param array $row Row of table data (post fixes) - */ - - protected function createOwnersGroups() { - // Pull all Groups and create Owners Group for them. Deployments generally - // don't have so many Groups that we need PaginatedSqlIterator, but we'll - // use it here anyway just in case. - - // By doing this once for the table we avoid having to sort through - // changelog metadata to figure out which rows to actually create owners - // groups for. - - $Groups = TableRegistry::getTableLocator()->get('Groups'); - - $iterator = new PaginatedSqlIterator($Groups, []); - - foreach($iterator as $k => $group) { - try { - // Because PaginatedSqlIterator will pick up new Groups as we create them, - // we need to check for any Owners groups (that we just created) and skip them. - if(!$group->isOwners()) { - $ownersGid = $Groups->createOwnersGroup($group); - - // We need to manually populate the cache - $this->cache['groups']['id'][$group->id]['owners_group_id'] = $ownersGid; - } - } - catch(\Exception $e) { - $this->io->error("Failed to create owners group for " - . $group->name . " (" . $group->id . "): " - . $e->getMessage()); - } - } - } - - /** - * Execute the Transmogrify Command. - * - * @since COmanage Registry v5.0.0 - * @param Arguments $args Command Arguments - * @param ConsoleIo $io Console IO - */ - - public function execute(Arguments $args, ConsoleIo $io) { - $this->args = $args; - $this->io = $io; - - // Load data from the inbound "transmogrify" database to a newly created - // (and empty) v5 database. The schema should already be applied to the - // new database. - - // First, open connections to both old and new databases. - $this->inconn = DBALConnection::factory($io, 'transmogrify'); - $this->outconn = DBALConnection::factory($io, 'default'); - - // We accept a list of table names, mostly for testing purposes - $atables = $args->getArguments(); - - // Register the current version for future upgrade purposes - - $metaTable = $this->getTableLocator()->get('Meta'); - $metaTable->setUpgradeVersion(); - - foreach(array_keys($this->tables) as $t) { - // If the command line args include a list of tables skip this table - // if it is not in that list. - if(!empty($atables) && !in_array($t, $atables)) - continue; - - $io->info(message: sprintf("Transmogrifying table %s(%s)", Inflector::classify($t), $t)); - - // Find the maximum id in the inbound table and reset the sequence for the outbound - // table to be that value plus one so that as we process the rows any entirely new rows - // inserted (entirely new rows have no existing id and will be given the id by the - // auto sequence) do not have a primary key id that conflicts with an existing row - // from the inbound table. - $qualifiedTableName = $this->inconn->qualifyTableName($this->tables[$t]['source']); - $maxId = $this->inconn->fetchOne('SELECT MAX(id) FROM ' . $qualifiedTableName); - $maxId++; - - $qualifiedTableName = $this->outconn->qualifyTableName($t); - $this->io->info("Resetting sequence for $qualifiedTableName to $maxId"); - - // Strictly speaking we should use prepared statements, but we control the - // data here, and also we're executing a maintenance operation (so query - // optimization is less important). - if($this->outconn->isMySQL()) { - $outsql = "ALTER TABLE $qualifiedTableName AUTO_INCREMENT = " . $maxId; - } else { - $outsql = "ALTER SEQUENCE " . $qualifiedTableName . "_id_seq RESTART WITH " . $maxId; - } - $this->outconn->executeQuery($outsql); - - // Run any preprocessing functions for the table. - if(!empty($this->tables[$t]['preTable'])) { - $p = $this->tables[$t]['preTable']; - - $this->$p(); - } - - // Check if the outbound table contains any rows and skip the table if it does. - $Model = $this->getTableLocator()->get($t); - if($Model->find()->count() > 0) { - $io->warning("Skipping Transmogrification. Table (" . $t . ") is not empty. Drop the database (or truncate) and start over."); - continue; - } - - // Count the rows in the inbound table so that we can log the percentage processed. - $qualifiedTableName = $this->inconn->qualifyTableName($this->tables[$t]['source']); - $count = $this->inconn->fetchOne("SELECT COUNT(*) FROM " . $qualifiedTableName); - - // Select all the rows from the inbound table. - if(!empty($this->tables[$t]['sqlSelect'])) { - $p = $this->tables[$t]['sqlSelect']; - $insql = $this->$p($qualifiedTableName); - } else { - $insql = "SELECT * FROM " . $qualifiedTableName . " ORDER BY id ASC"; - } - $stmt = $this->inconn->executeQuery($insql); - - $tally = 0; - $warns = 0; - $err = 0; - - // Loop over each row from the inbound table. - while($row = $stmt->fetchAssociative()) { - if(!empty($row[ $this->tables[$t]['displayField'] ])) { - $io->verbose("$t " . $row[ $this->tables[$t]['displayField'] ]); - } - - try { - // Make a copy of the original data for any postprocessing followups. - $origRow = $row; - - // Run any preprocessing functions for the row. - - if(!empty($this->tables[$t]['preRow'])) { - $p = $this->tables[$t]['preRow']; - - $this->$p($origRow, $row); - } - - // Do this before fixBooleans since we'll insert some - $this->fixChangelog($t, $row, isset($this->tables[$t]['addChangelog']) && $this->tables[$t]['addChangelog']); - - $this->fixBooleans($t, $row); - - $this->mapFields($t, $row); - - $qualifiedTableName = $this->outconn->qualifyTableName($t); - $this->outconn->insert($qualifiedTableName, $row); - - $this->cacheResults($t, $row); - - // Run any post processing functions for the row. - - if(!empty($this->tables[$t]['postRow'])) { - $p = $this->tables[$t]['postRow']; - - $this->$p($origRow, $row); - } - } - catch(ForeignKeyConstraintViolationException $e) { - // A foreign key associated with this record did not load, so we can't - // load this record. This can happen, eg, because the source_field_id - // did not load, perhaps because it was associated with an Org Identity - // not linked to a CO Person that was not migrated. - $warns++; - $io->warning("Skipping $t record " . $row['id'] . " due to invalid foreign key: " . $e->getMessage()); - } - catch(\InvalidArgumentException $e) { - // If we can't find a value for mapping we skip the record - // (ie: mapFields basically requires a successful mapping) - $warns++; - $io->warning("Skipping $t record " . $row['id'] . ": " . $e->getMessage()); - } - catch(\Exception $e) { - $err++; - $io->error("$t record " . $row['id'] . ": " . $e->getMessage()); - } - - $tally++; - - if(!$this->args->getOption('quiet') && !$this->args->getOption('verbose')) { - // We don't output the progress bar for quiet for obvious reasons, - // or for verbose so we don't interfere with the extra output - $this->cliLogPercentage($tally, $count); - } - } - - // Log warning and error count. - $io->out("(Warnings: " . $warns . ")"); - $io->out("(Errors: " . $err . ")"); - - - // Run any post processing functions for the table. - if(!empty($this->tables[$t]['postTable'])) { - $p = $this->tables[$t]['postTable']; - - $this->$p(); - } - } - } - - /** - * Filter Jobs. - * - * @since COmanage Registry v5.0.0 - * @param array $origRow Row of table data (original data) - * @param array $row Row of table data (post fixes) - * @throws InvalidArgumentException - */ - - protected function filterJobs(array $origRow, array $row) { - // We don't update any of the attributes, but for rows with unsupported data - // we throw an exception so they don't transmogrify. - - if($row['status'] == 'GO' || $row['status'] == 'Q') { - throw new \InvalidArgumentException("Job is Queued or In Progress"); - } - - if($row['job_type'] == 'EX' || $row['job_type'] == 'OS') { - throw new \InvalidArgumentException("Legacy Job types cannot be transmogrified"); - } - } - - /** - * Find the CO for a row of table data, based on a foreign key. - * - * @since COmanage Registry v5.0.0 - * @param array $row Row of table data - * @return int CO ID - * @throws InvalidArgumentException - */ - - protected function findCoId(array $row) { - // By the time we're called, we should have transmogrified the Org Identity - // and CO Person data, so we can just walk the caches - - if(!empty($row['person_id'])) { - if(isset($this->cache['people']['id'][ $row['person_id'] ]['co_id'])) { - return $this->cache['people']['id'][ $row['person_id'] ]['co_id']; - } - } elseif(!empty($row['external_identity_id'])) { - // Map the OrgIdentity to a CO Person, then to the CO - if(!empty($this->cache['external_identities']['id'][ $row['external_identity_id'] ]['person_id'])) { - $personId = $this->cache['external_identities']['id'][ $row['external_identity_id'] ]['person_id']; - - if(isset($this->cache['people']['id'][ $personId ]['co_id'])) { - return $this->cache['people']['id'][ $personId ]['co_id']; - } - } - } elseif(!empty($row['group_id'])) { - if(isset($this->cache['groups']['id'][ $row['group_id'] ]['co_id'])) { - return $this->cache['groups']['id'][ $row['group_id'] ]['co_id']; - } - } - // We also support being called using the old keys for use in the preRow context - elseif(!empty($row['org_identity_id'])) { - // Map the OrgIdentity to a CO Person, then to the CO - if(!empty($this->cache['external_identities']['id'][ $row['org_identity_id'] ]['person_id'])) { - $personId = $this->cache['external_identities']['id'][ $row['org_identity_id'] ]['person_id']; - - if(isset($this->cache['people']['id'][ $personId ]['co_id'])) { - return $this->cache['people']['id'][ $personId ]['co_id']; - } - } - } elseif(!empty($row['co_person_id'])) { - if(isset($this->cache['people']['id'][ $row['co_person_id'] ]['co_id'])) { - return $this->cache['people']['id'][ $row['co_person_id'] ]['co_id']; - } - } - - throw new \InvalidArgumentException('CO not found for record'); - } - - /** - * Translate booleans to string literals to work around DBAL Postgres boolean handling. - * - * @since COmanage Registry v5.0.0 - * @param string $table Table Name - * @param array $row Row of attributes, fixed in place - */ - - protected function fixBooleans(string $table, array &$row) { - $attrs = ['deleted']; - - // We could introspect this from the schema file... - if(!empty($this->tables[$table]['booleans'])) { - $attrs = array_merge($attrs, $this->tables[$table]['booleans']); - } - - foreach($attrs as $a) { - if(isset($row[$a]) && gettype($row[$a]) == 'boolean') { - // DBAL Postgres boolean handling seems to be somewhat buggy, see history in - // this issue: https://github.com/doctrine/dbal/issues/1847 - // We need to (more generically than this hack) convert from boolean to char - // to avoid errors on insert - if($this->outconn->isMySQL()) { - $row[$a] = ($row[$a] ? '1' : '0'); - } else { - $row[$a] = ($row[$a] ? 't' : 'f'); - } - } - } - } - - /** - * Populate empty Changelog data from legacy records - * - * @since COmanage Registry v5.0.0 - * @param string $table Table Name - * @param array $row Row of attributes, fixed in place - * @param bool $force If true, always create keys - */ - - protected function fixChangelog(string $table, array &$row, bool $force=false) { - if($force || (array_key_exists('deleted', $row) && is_null($row['deleted']))) { - $row['deleted'] = false; - } - - if($force || (array_key_exists('revision', $row) && is_null($row['revision']))) { - $row['revision'] = 0; - } - - if($force || (array_key_exists('actor_identifier', $row) && is_null($row['actor_identifier']))) { - $row['actor_identifier'] = 'Transmogrification'; - } - - // The parent FK should remain NULL since this is the original record. - /* - // If the table was renamed, we need to rename the changelog key as well. - // NOTE: We don't actually do this here because it creates issues with the - // order of field processing. Instead, each key must be renamed - // manually in the fieldMap. - // eg: cm_org_identities -> org_identity_id - $oldfk = Inflector::singularize(substr($this->tables[$table]['source'], 3)) . "_id"; - // eg: external_identities -> external_identity_id - $newfk = Inflector::singularize($table) . "_id"; - - if($oldfk != $newfk && array_key_exists($oldfk, $row)) { - $row[$newfk] = $row[$oldfk]; - unset($row[$oldfk]); - }*/ - } - - /** - * Insert default CO Settings. - * - * @since COmanage Registry v5.0.0 - */ - - protected function insertDefaultSettings() { - // Create a CoSetting for any CO that didn't previously have one. - - $createdSettings = []; - $createdCos = array_keys($this->cache['cos']['id']); - - foreach($this->cache['co_settings']['id'] as $co_setting_id => $cached) { - $createdSettings[] = $cached['co_id']; - } - - $emptySettings = array_values(array_diff($createdCos, $createdSettings)); - - if(!empty($emptySettings)) { - $CoSettings = TableRegistry::getTableLocator()->get('CoSettings'); - - foreach($emptySettings as $coId) { - // Insert a default row into CoSettings for this CO ID - try { - $CoSettings->addDefaults($coId); - } catch (\ConflictException $e) { - // skip - } - } - } - } - - /** - * Insert default Pronoun types. - * - * @since COmanage Registry v5.0.0 - */ - - protected function insertPronounTypes() { - // Since the Pronoun MVEA didn't exist in v4, we'll need to create the - // default types for all COs. - - $Types = TableRegistry::getTableLocator()->get('Types'); - - foreach(array_keys($this->cache['cos']['id']) as $coId) { - $Types->addDefault($coId, 'Pronouns.type'); - } - } - - /** - * Map fields that have been renamed from Registry Classic to Registry PE. - * - * @since COmanage Registry v5.0.0 - * @param string $table Table Name - * @param array $row Row of attributes, fixed in place - * @throws InvalidArgumentException - */ - - protected function mapFields(string $table, array &$row) { - // oldname => newname, or &newname, which is a function to call. - // Note functions can returns more than one mapping - $fields = []; - - if(!empty($this->tables[$table]['fieldMap'])) { - $fields = $this->tables[$table]['fieldMap']; - } - - foreach($fields as $oldname => $newname) { - if(!$newname) { - // This attribute doesn't map, so simply unset it - unset($row[$oldname]); - } elseif($newname[0] == '&') { - // This is a function to map the field, in which case we reuse the old name - $f = substr($newname, 1); - - // We always pass the entire row so the mapping function can implement - // whatever logic it needs - $row[$oldname] = $this->$f($row); - - if(!$row[$oldname]) { - throw new \InvalidArgumentException("Could not find value for $table $oldname"); - } - } elseif($newname[0] == '?') { - // This is a default value to populate if the current value is null - $v = substr($newname, 1); - - if($row[$oldname] === null) { - $row[$oldname] = $v; - } - } else { - // Copy the value to the new name, then unset the old name - $row[$newname] = $row[$oldname]; - unset($row[$oldname]); - } - } - } - - /** - * Map an address type string to a foreign key. - * - * @since COmanage Registry v5.0.0 - * @param array $row Row of table data - * @return int type_id - */ - - protected function map_address_type(array $row) { - return $this->map_type($row, 'Addresses.type', $this->findCoId($row)); - } - - /** - * Map an affiliation type string to a foreign key. - * - * @since COmanage Registry v5.0.0 - * @param array $row Row of table data - * @return int type_id - */ - - protected function map_affiliation_type(array $row) { - return $this->map_type($row, 'PersonRoles.affiliation_type', $this->findCoId($row), 'affiliation'); - } - - /** - * Map an email type string to a foreign key. - * - * @since COmanage Registry v5.0.0 - * @param array $row Row of table data - * @return int type_id - */ - - protected function map_email_type(array $row) { - return $this->map_type($row, 'EmailAddresses.type', $this->findCoId($row)); - } - - /** - * Map an Extended Type attribute name for model name changes. - * - * @since COmanage Registry v5.0.0 - * @param array $row Row of table data - * @return string Updated attribute name - */ - - protected function map_extended_type(array $row) { - switch($row['attribute']) { - case 'CoDepartment.type': - return 'Departments.type'; - case 'CoPersonRole.affiliation': - return 'PersonRoles.affiliation_type'; - } - - // For everything else, we need to pluralize the model name - $bits = explode('.', $row['attribute'], 2); - - return Inflector::pluralize($bits[0]) . "." . $bits[1]; - } - - /** - * Map login identifiers, in accordance with the configuration. - * - * @since COmanage Registry v5.0.0 - * @param array $origRow Row of table data (original data) - * @param array $row Row of table data (post fixes) - * @throws InvalidArgumentException - */ - - protected function map_login_identifiers(array $origRow, array $row) { - // There might be multiple reasons to copy the row, but we only want to - // copy it once. - $copyRow = false; - - if(!empty($origRow['org_identity_id'])) { - if($this->args->getOption('login-identifier-copy') - && $origRow['login']) { - $copyRow = true; - } - - // Note the argument here is the old v4 string (eg "eppn") and not the - // PE foreign key - if($this->args->getOption('login-identifier-type') - && $origRow['type'] == $this->args->getOption('login-identifier-type')) { - $copyRow = true; - } - - // Identifiers attached to External Identities do not have login flags in PE - $row['login'] = false; - } - - if($copyRow) { - // Find the Person ID associated with this External Identity ID - - if(!empty($this->cache['external_identities']['id'][ $origRow['org_identity_id'] ]['person_id'])) { - // Insert a new row attached to the Person, leave the original record - // (ie: $row) untouched - - $copiedRow = [ - 'person_id' => $this->map_org_identity_co_person_id(['id' => $origRow['org_identity_id']]), - 'identifier' => $origRow['identifier'], - 'type_id' => $this->map_identifier_type($origRow), - 'status' => $origRow['status'], - 'login' => true, - 'created' => $origRow['created'], - 'modified' => $origRow['modified'] - ]; - - // Set up changelog and fix booleans - $this->fixChangelog('identifiers', $copiedRow, true); - $this->fixBooleans('identifiers', $copiedRow); - - try { - $tableName = 'identifiers'; - $qualifiedTableName = $this->outconn->qualifyTableName($tableName); - $this->outconn->insert($qualifiedTableName, $copiedRow); - } catch (\Doctrine\DBAL\Exception\UniqueConstraintViolationException $e) { - $this->io->warning("record already exists: " . print_r($copiedRow, true)); - } - } - } - } - - /** - * Map an identifier type string to a foreign key. - * - * @since COmanage Registry v5.0.0 - * @param array $row Row of table data (ignored) - * @return int type_id - */ - - protected function map_identifier_type(array $row) { - return $this->map_type($row, 'Identifiers.type', $this->findCoId($row)); - } - - /** - * Map a name type string to a foreign key. - * - * @since COmanage Registry v5.0.0 - * @param array $row Row of table data (ignored) - * @return int type_id - */ - - protected function map_name_type(array $row) { - return $this->map_type($row, 'Names.type', $this->findCoId($row)); - } - - /** - * Return a timestamp equivalent to now. - * - * @since COmanage Registry v5.0.0 - * @param array $row Row of table data (ignored) - * @return string Timestamp - */ - - protected function map_now(array $row) { - if(empty($this->cache['now'])) { - $created = new \Datetime('now'); - $this->cache['now'] = $created->format('Y-m-d H:i:s'); - } - - return $this->cache['now']; - } - - /** - * Map an Org Identity ID to a CO Person ID - * - * @since COmanage Registry v5.0.0 - * @param array $row Row of Org Identity table data - * @return int CO Person ID - */ - - protected function map_org_identity_co_person_id(array $row) { - // PE eliminates OrgIdentityLink, so we need to map each Org Identity to - // a Person ID. This is a bit trickier than it sounds, since an Org Identity - // could have been relinked. - - // Before Transmogrification, we require that Org Identities are unpooled. - // (This is probably how most deployments are set up, but there may be some - // legacy deployments out there.) This ensures whatever CO Person the Org - // Identity currently maps to through CoOrgIdentityLink is in the same CO. - - // There may be multiple mappings if the Org Identity was relinked. Basically - // we're going to lose the multiple mappings, since we can only return one - // value here. (Ideally, we would inject multiple OrgIdentities into the new - // table, but this ends up being rather tricky, since we have to figure out - // what row id to assign, and for the moment we don't have a mechanism to - // do that.) Historical information remains available in history_records, - // and if the deployer keeps an archive of the old database. - - // To figure out which person_id to use, we pull the record with the - // highest revision number. Note we might be transmogrifying a deleted row, - // so we can't ignore deleted rows here. - - if(empty($this->cache['org_identities']['co_people'])) { - $this->io->info('Populating org identity map...'); - - // We pull deleted rows because we might be migrating deleted rows - $tableName = "cm_co_org_identity_links"; - $qualifiedTableName = $this->inconn->qualifyTableName($tableName); - $mapsql = "SELECT * FROM $qualifiedTableName"; - $stmt = $this->inconn->query($mapsql); - - while($r = $stmt->fetch()) { - if(!empty($r['org_identity_id'])) { - if(isset($this->cache['org_identities']['co_people'][ $r['org_identity_id'] ][ $r['revision'] ])) { - // If for some reason we already have a record, it's probably due to - // improper unpooling from a legacy deployment. We'll accept only the - // first record and throw warnings on the others. - - $this->io->warning("Found existing CO Person for Org Identity " . $r['org_identity_id'] . ", skipping"); - } else { - $this->cache['org_identities']['co_people'][ $r['org_identity_id'] ][ $r['revision'] ] = $r['co_person_id']; - } - } - } - } - - if(!empty($this->cache['org_identities']['co_people'][ $row['id'] ])) { - // Return the record with the highest revision number - $rev = max(array_keys($this->cache['org_identities']['co_people'][ $row['id'] ])); - - return $this->cache['org_identities']['co_people'][ $row['id'] ][$rev]; - } - - return null; - } - - /** - * Map a telephone type string to a foreign key. - * - * @since COmanage Registry v5.0.0 - * @param array $row Row of table data - * @return int type_id - */ - - protected function map_telephone_type(array $row) { - return $this->map_type($row, 'TelephoneNumbers.type', $this->findCoId($row)); - } - - /** - * Map a type string to a foreign key. - * - * @since COmanage Registry v5.0.0 - * @param array $row Row of table data - * @param string $type Type to map (types:attribute) - * @param int $coId CO ID - * @param string $attr Row column to use for type value - * @return int type_id - * @throws InvalidArgumentException - */ - - protected function map_type(array $row, string $type, $coId, string $attr="type") { - if(!$coId) { - throw new \InvalidArgumentException("CO ID not provided for $type " . $row['id']); - } - - $key = $coId . "+" . $type . "+" . $row[$attr] . "+"; - - if(empty($this->cache['types']['co_id+attribute+value+'][$key])) { - throw new \InvalidArgumentException("Type not found for " . $key); - } - - return $this->cache['types']['co_id+attribute+value+'][$key]; - } - - /** - * Map a URL type string to a foreign key. - * - * @since COmanage Registry v5.0.0 - * @param array $row Row of table data - * @return int type_id - */ - - protected function map_url_type(array $row) { - return $this->map_type($row, 'Urls.type', $this->findCoId($row)); - } - - /** - * Set a default value for CO Settings Permitted Telephone Number Fields. - * - * @since COmanage Registry v5.0.0 - * @param array $row Row of table data - * @return string Default value - */ - - protected function populate_co_settings_phone(array $row) { - return \App\Lib\Enum\PermittedTelephoneNumberFieldsEnum::CANE; - } - - /** - * Process Extended Attributes by converting them to Ad Hoc Attributes. - * - * @since COmanage Registry v5.0.0 - */ - - protected function processExtendedAttributes() { - // This is intended to run AFTER AdHocAttributes so that we don't stomp on - // the row identifiers. - - // First, pull the old Extended Attribute configuration. - $extendedAttrs = []; - - $tableName = "cm_co_extended_attributes"; - $qualifiedTableName = $this->inconn->qualifyTableName($tableName); - $insql = "SELECT * FROM $qualifiedTableName ORDER BY id ASC"; - $stmt = $this->inconn->query($insql); - - while($row = $stmt->fetch()) { - $extendedAttrs[ $row['co_id'] ][] = $row['name']; - } - - if(empty($extendedAttrs)) { - // No need to do anything further if no attributes are configured - return; - } - - foreach(array_keys($extendedAttrs) as $coId) { - $tableName = "cm_co" . $coId . "_person_extended_attributes"; - $qualifiedTableName = $this->inconn->qualifyTableName($tableName); - $insql = "SELECT * FROM $qualifiedTableName"; - $stmt = $this->inconn->query($insql); - - while($eaRow = $stmt->fetch()) { - // If we didn't transmogrify the parent row for some reason then trying - // to insert the ad_hoc_attributes will throw an error. - if(!empty($this->cache['person_roles']['id'][ $eaRow['co_person_role_id'] ])) { - foreach($extendedAttrs[$coId] as $ea) { - $adhocRow = [ - 'person_role_id' => $eaRow['co_person_role_id'], - 'tag' => $ea, - 'value' => $eaRow[$ea], - 'created' => $eaRow['created'], - 'modified' => $eaRow['modified'] - ]; - - // Extended Attributes were not changelog enabled - $this->fixChangelog('ad_hoc_attributes', $adhocRow, true); - $this->fixBooleans('ad_hoc_attributes', $adhocRow); - - try { - $tableName = 'ad_hoc_attributes'; - $qualifiedTableName = $this->outconn->qualifyTableName($tableName); - $this->outconn->insert($qualifiedTableName, $adhocRow); - } catch (\Doctrine\DBAL\Exception\UniqueConstraintViolationException $e) { - $this->io->warning("record already exists: " . print_r($adhocRow, true)); - } - } - } - } - } - } - - /** - * Return SQL used to select CO Person Roles from inbound database. - * - * @since COmanage Registry v5.0.0 - * @param string $tableName Name of the SQL table - * @return string SQL string to select rows from inbound database - */ - - protected function roleSqlSelect(string $tableName): string { - // Cast the affiliation value to 'member' when NULL or empty string. - define("ROLE_SQL_SELECT", << 'external_identity_id', - 'o' => 'organization', - 'ou' => 'department', - 'manager_identifier' => 'manager_identifier', - 'sponsor_identifier' => 'sponsor_identifier', - 'status' => 'status', - 'title' => 'title', - 'valid_from' => 'valid_from', - 'valid_through' => 'valid_through', - // Fix up changelog - 'org_identity_id' => 'external_identity_role_id', - 'revision' => 'revision', - 'deleted' => 'deleted', - 'actor_identifier' => 'actor_identifier', - 'created' => 'created', - 'modified' => 'modified' - ] as $oldKey => $newKey) { - $roleRow[$newKey] = $origRow[$oldKey]; - } - - // Affiliation requires special handling. We need to use the post-fixed $row - // because map_affiliation_type calls findCoId which uses the foreign key to - // lookup the CO ID in the cache, however by the time we've been called - // affiliation has been null'd out (since we're moving it to the role row). - // So shove it back in before calling map_affiliation_type. - if(!empty($origRow['affiliation'])) { - $row['affiliation'] = $origRow['affiliation']; - $roleRow['affiliation_type_id'] = $this->map_affiliation_type($row); - } - - // Fix up changelog - // Since we're creating a new row, we have to manually fix up booleans - $roleRow['deleted'] = ($roleRow['deleted'] ? 't' : 'f'); - - $tableName = 'external_identity_roles'; - $qualifiedTableName = $this->outconn->qualifyTableName($tableName); - $this->outconn->insert($qualifiedTableName, $roleRow); - } -} \ No newline at end of file diff --git a/app/src/Lib/Util/TransmogrifyUtilities.php b/app/src/Lib/Util/TransmogrifyUtilities.php deleted file mode 100644 index 65be538f8..000000000 --- a/app/src/Lib/Util/TransmogrifyUtilities.php +++ /dev/null @@ -1,179 +0,0 @@ -loadGeneric($path); + // Filter out documentation or comment keys (eg, keys starting with "__") + $cfg = array_filter($cfg, static function ($value, $key) { + return !(is_string($key) && str_starts_with($key, '__')); + }, ARRAY_FILTER_USE_BOTH); + return $cfg; + } + + /** + * Generic config loader that supports JSON (.json) and XML (.xml) files. + * Returns associative array representation of the root object. + * + * @param string $path Absolute or project-relative path + * @return array + */ + public function loadGeneric(string $path): array + { + // Resolve path if relative + if (!is_readable($path)) { + if (defined('ROOT')) { + $alt = ROOT . DIRECTORY_SEPARATOR . ltrim($path, DIRECTORY_SEPARATOR); + if (is_readable($alt)) { + $path = $alt; + } + } + } + if (!is_readable($path)) { + throw new \RuntimeException('Config not readable: ' . $path); + } + + $ext = strtolower(pathinfo($path, PATHINFO_EXTENSION)); + $raw = file_get_contents($path); + if ($raw === false) { + throw new \RuntimeException('Failed to read config: ' . $path); + } + + if ($ext === 'json') { + // Decode JSON and detect corruption with detailed error messages + $data = json_decode($raw, true); + if ($data === null && json_last_error() !== JSON_ERROR_NONE) { + $err = function_exists('json_last_error_msg') ? json_last_error_msg() : ('code ' . json_last_error()); + throw new \RuntimeException('Invalid JSON config: ' . $path . ' Details: ' . $err); + } + if (!is_array($data)) { + // Enforce that root must be an object/array for config purposes + throw new \RuntimeException('Invalid JSON config: ' . $path . ' Details: Expected root object or array'); + } + return $data; + } + + if ($ext === 'xml') { + // Use libxml internal error handling instead of deprecated @ suppression + $prev = libxml_use_internal_errors(true); + $xml = simplexml_load_string($raw, 'SimpleXMLElement', LIBXML_NONET | LIBXML_NOERROR | LIBXML_NOWARNING); + if ($xml === false) { + $errors = libxml_get_errors(); + libxml_clear_errors(); + libxml_use_internal_errors($prev); + $messages = array_map(static function ($e) { + return trim($e->message ?? '') . ' at line ' . ($e->line ?? ''); + }, $errors ?: []); + $detail = $messages ? (' Details: ' . implode(' | ', $messages)) : ''; + throw new \RuntimeException('Invalid XML config: ' . $path . $detail); + } + // Clear any accumulated libxml errors and restore previous setting + libxml_clear_errors(); + libxml_use_internal_errors($prev); + + $json = json_encode($xml); + $arr = json_decode($json, true); + if (!is_array($arr)) { + throw new \RuntimeException('Failed to convert XML to array: ' . $path); + } + return $arr; + } + + throw new \RuntimeException('Unsupported config extension (expected .json or .xml): ' . $path); + } +} diff --git a/app/src/Service/DbInfoService.php b/app/src/Service/DbInfoService.php new file mode 100644 index 000000000..bae30d07b --- /dev/null +++ b/app/src/Service/DbInfoService.php @@ -0,0 +1,184 @@ + 'transmogrify', 'target' => 'default' ]; + $role = $roleOrAlias; + $alias = $map[$roleOrAlias] ?? $roleOrAlias; + + $cfg = null; + try { + $cfg = ConnectionManager::getConfig($alias); + } catch (\Throwable $e) { + $cfg = null; + } + + return [ + 'alias' => $alias, + 'role' => ($alias === ($map['source'] ?? 'transmogrify')) ? 'source' : (($alias === ($map['target'] ?? 'default')) ? 'target' : $role), + 'configured' => $cfg !== null, + 'driver' => $cfg['driver'] ?? null, + 'host' => $cfg['host'] ?? ($cfg['hostname'] ?? null), + 'port' => $cfg['port'] ?? null, + 'database' => $cfg['database'] ?? ($cfg['dbname'] ?? null), + 'username' => $cfg['username'] ?? ($cfg['user'] ?? null), + 'password' => null, + 'dsn' => $cfg['url'] ?? null, + ]; + } + + /** + * Ping a connection alias and return status info. + * @param string $alias Cake connection alias + * @return array{ok:bool,server:?(string),error:?(string)} + */ + public function ping(string $alias): array + { + $status = [ 'ok' => false, 'error' => null, 'server' => null ]; + try { + $conn = DBALConnection::factory(connection: $alias); + if ($conn->isMySQL()) { + $ver = $conn->fetchOne('SELECT VERSION()'); + } else { + $ver = $conn->fetchOne('SHOW server_version'); + if (!$ver) { $ver = $conn->fetchOne('SELECT version()'); } + } + $status['ok'] = true; + $status['server'] = $ver; + } catch (\Throwable $e) { + $status['ok'] = false; + $status['error'] = $e->getMessage(); + } + return $status; + } + + /** + * Load schema information for a given alias. + * @param string $alias Cake connection alias + * @param string|null $tablesJsonPath Path to the tables.json file + * @return array{table_count:int,empty:bool,sample_tables:array,tables_compare:array,loaded_schema:?(string)} + */ + public function loadSchemaInfo(string $alias, ?string $tablesJsonPath = null): array + { + if ($tablesJsonPath === null) { + return []; + } + $conn = DBALConnection::factory(connection: $alias); + return $this->loadSchemaInfoFromConnection($conn, $tablesJsonPath); + } + + /** + * Exposed for reuse when a DBALConnection already exists. + * @param DBALConnection $conn The database connection + * @param string $tablesJsonPath Path to the tables.json file + * @return array Schema information array + * + * todo: The new version should render all the tables. And if i pass a the parameter transmogrify then + * it should render the ones that have been transmogrified. + */ + public function loadSchemaInfoFromConnection(DBALConnection $conn, string $tablesJsonPath): array + { + // Load declared tables from tables.json + $declared = []; + $loadedSchemaName = basename($tablesJsonPath); + try { + + if (is_readable($tablesJsonPath)) { + $json = file_get_contents($tablesJsonPath); + $cfg = json_decode($json, true); + if (is_array($cfg)) { + // Filter out documentation keys (eg, keys starting with "__") to avoid printing template entries + $cfg = array_filter($cfg, static function ($value, $key) { + return !(is_string($key) && str_starts_with($key, '__')); + }, ARRAY_FILTER_USE_BOTH); + $declared = array_keys($cfg); + } + } + } catch (\Throwable $e) { + // ignore; we'll fall back to empty list + } + + // Gather list of non-system tables + $tables = []; + if ($conn->isMySQL()) { + $db = $conn->fetchOne('SELECT DATABASE()'); + $rows = $conn->fetchAllAssociative('SELECT table_name FROM information_schema.tables WHERE table_schema = ? AND table_type = ? ORDER BY table_name ASC', [$db, 'BASE TABLE']); + $tables = array_map(fn($r) => $r['table_name'], $rows); + } else { + // PostgreSQL + $rows = $conn->fetchAllAssociative("SELECT schemaname, tablename FROM pg_catalog.pg_tables WHERE schemaname NOT IN ('pg_catalog','information_schema') ORDER BY schemaname, tablename"); + foreach ($rows as $r) { $tables[] = ($r['schemaname'] . '.' . $r['tablename']); } + } + + // Check if DB is empty (no tables) or all tables empty (zero rows) + $info = [ + 'table_count' => count($tables), + 'sample_tables' => array_slice($tables, 0, self::TABLE_SAMPLE_LIMIT), + 'empty' => false, + 'loaded_schema' => $loadedSchemaName, + ]; + + $info['empty'] = (count($tables) === 0); + if (!$info['empty']) { + // If there are tables, check if any table has rows; if none, still empty of data + $hasData = false; + foreach ($info['sample_tables'] as $t) { + try { + // For qualified names, don't double-quote + $count = (int)$conn->fetchOne('SELECT COUNT(*) FROM ' . $t); + if ($count > 0) { $hasData = true; break; } + } catch (\Throwable $e) { + // ignore per-table errors + } + } + $info['empty'] = !$hasData; + } + + // Build comparison lists between tables.json and actual DB tables, ignoring schema names + $normalize = function(string $t): string { + // Strip any schema prefix such as public.table or mysch.table + $p = strrpos($t, '.'); + return $p === false ? $t : substr($t, $p + 1); + }; + $actualBare = array_map($normalize, $tables); + $declaredBare = array_map($normalize, $declared); + // Now compute sets on bare names + $actual = $actualBare; + $declared = $declaredBare; + sort($actual); + sort($declared); + $both = array_values(array_unique(array_intersect($declared, $actual))); + $onlyJson = array_values(array_unique(array_diff($declared, $actual))); + $onlyDb = array_values(array_unique(array_diff($actual, $declared))); + + $info['tables_compare'] = [ + 'both' => $both, + 'only_in_json' => $onlyJson, + 'only_in_db' => $onlyDb, + ]; + + return $info; + } +} diff --git a/app/vendor/cakephp-plugins.php b/app/vendor/cakephp-plugins.php index 79b785483..c161ef702 100644 --- a/app/vendor/cakephp-plugins.php +++ b/app/vendor/cakephp-plugins.php @@ -16,5 +16,6 @@ 'Migrations' => $baseDir . '/vendor/cakephp/migrations/', 'OrcidSource' => $baseDir . '/plugins/OrcidSource/', 'SshKeyAuthenticator' => $baseDir . '/plugins/SshKeyAuthenticator/', + 'Transmogrify' => $baseDir . '/plugins/Transmogrify/', ], ]; From 00208f3153c259af743950f6952566c58d064b5c Mon Sep 17 00:00:00 2001 From: Ioannis Igoumenos Date: Fri, 3 Oct 2025 15:50:23 +0300 Subject: [PATCH 02/15] Fixes --- .../src/Command/TransmogrifyCommand.php | 6 +- .../src/Lib/Traits/TypeMapperTrait.php | 86 ++--- .../src/Lib/Util/CommanLinePrinter.php | 184 ---------- .../src/Lib/Util/CommandLinePrinter.php | 329 +++++++++--------- 4 files changed, 202 insertions(+), 403 deletions(-) delete mode 100644 app/plugins/Transmogrify/src/Lib/Util/CommanLinePrinter.php diff --git a/app/plugins/Transmogrify/src/Command/TransmogrifyCommand.php b/app/plugins/Transmogrify/src/Command/TransmogrifyCommand.php index 17aecffe4..591f855b0 100644 --- a/app/plugins/Transmogrify/src/Command/TransmogrifyCommand.php +++ b/app/plugins/Transmogrify/src/Command/TransmogrifyCommand.php @@ -48,7 +48,7 @@ use Transmogrify\Lib\Traits\ManageDefaultsTrait; use Transmogrify\Lib\Traits\RowTransformationTrait; use Transmogrify\Lib\Traits\TypeMapperTrait; -use Transmogrify\Lib\Util\CommanLinePrinter; +use Transmogrify\Lib\Util\CommandLinePrinter; use Transmogrify\Lib\Util\DbInfoPrinter; use Transmogrify\Lib\Util\RawSqlQueries; @@ -282,7 +282,7 @@ public function execute(Arguments $args, ConsoleIo $io): int }; $stmt = $this->inconn->executeQuery($insql); - $progress = new CommanLinePrinter($io, 'green', 50, true); + $progress = new CommandLinePrinter($io, 'green', 50, true); $progress->start($count); $tally = 0; $warns = 0; @@ -357,7 +357,7 @@ public function execute(Arguments $args, ConsoleIo $io): int $io->error(sprintf('Errors: %d', $err)); // Step 11: Execute any post-processing hooks for the table - if ($modeltableEmpty && !$notSelected) { + if ($modeltableEmpty && $notSelected) { $this->runPostTableHook($t); } diff --git a/app/plugins/Transmogrify/src/Lib/Traits/TypeMapperTrait.php b/app/plugins/Transmogrify/src/Lib/Traits/TypeMapperTrait.php index 4c87e65ec..333c45895 100644 --- a/app/plugins/Transmogrify/src/Lib/Traits/TypeMapperTrait.php +++ b/app/plugins/Transmogrify/src/Lib/Traits/TypeMapperTrait.php @@ -202,54 +202,56 @@ protected function mapOrgIdentitycoPersonId(array $row): ?int $oid = (int)$row['id']; - if(!isset($this->cache['org_identities']['co_people'])) { + if(empty($this->cache['org_identities']['co_people'])) { $this->cache['org_identities']['co_people'] = []; - } - - // Guard 1: If we already have this org identity mapped, return immediately - if (isset($this->cache['org_identities']['co_people'][$oid])) { - return $this->cache['org_identities']['co_people'][$oid]; - } - - // Build cache on first use - $this->io->info('Populating org identity map...'); - - $tableName = 'cm_co_org_identity_links'; - $changelogFK = 'co_org_identity_link_id'; - $qualifiedTableName = $this->inconn->qualifyTableName($tableName); - - // Only fetch current rows (historical/deleted rows are filtered out) - $mapsql = RawSqlQueries::buildSelectAll( - qualifiedTableName: $qualifiedTableName, - onlyCurrent: true, - changelogFK: $changelogFK - ); - $stmt = $this->inconn->executeQuery($mapsql); - - while ($r = $stmt->fetchAssociative()) { - if (!empty($r['org_identity_id'])) { - $rOid = (int)$r['org_identity_id']; - $cop = isset($r['co_person_id']) ? (int)$r['co_person_id'] : null; - - if ($cop === null) { - // Defensive: skip rows without a co_person_id - $this->io->warning('Org Identity ' . $rOid . ' has no mapped CO Person ID, skipping'); - continue; - } - - if (isset($this->cache['org_identities']['co_people'][$rOid])) { - // Unexpected duplicate "current" mapping, keep the first and warn - $this->io->warning('Found duplicate current CO Person mapping for Org Identity ' . $rOid . ', skipping'); - continue; + // Build cache on first use + $this->io->info('Populating org identity map...'); + + $tableName = 'cm_co_org_identity_links'; + $changelogFK = 'co_org_identity_link_id'; + $qualifiedTableName = $this->inconn->qualifyTableName($tableName); + + // Only fetch current rows (historical/deleted rows are filtered out) + $mapsql = RawSqlQueries::buildSelectAll( + qualifiedTableName: $qualifiedTableName, + onlyCurrent: true, + changelogFK: $changelogFK + ); + + $stmt = $this->inconn->executeQuery($mapsql); + + while ($r = $stmt->fetchAssociative()) { + if (!empty($r['org_identity_id'])) { + $rOid = (int)$r['org_identity_id']; + $rOrevision = (int)$r['revision']; + $cop = isset($r['co_person_id']) ? (int)$r['co_person_id'] : null; + + if ($cop === null) { + $this->io->warning('Org Identity ' . $rOid . ' has no mapped CO Person ID, skipping'); + continue; + } + + if (isset($this->cache['org_identities']['co_people'][$rOid][$rOrevision])) { + // If for some reason we already have a record, it's probably due to + // improper unpooling from a legacy deployment. We'll accept only the + // first record and throw warnings on the others. + $this->io->verbose('Found duplicate current CO Person mapping for Org Identity ' . $rOid . ', skipping'); + continue; + } + + $this->cache['org_identities']['co_people'][$rOid][$rOrevision] = $cop; } - - $this->cache['org_identities']['co_people'][$rOid] = $cop; } } - // Return the now-cached mapping (or null if not present) - return $this->cache['org_identities']['co_people'][$oid] ?? null; + if (!empty($this->cache['org_identities']['co_people'][$oid])) { + // Return the record with the highest revision number + $rev = max(array_keys($this->cache['org_identities']['co_people'][ $oid ])); + return $this->cache['org_identities']['co_people'][$oid][$rev]; + } + + return null; } /** diff --git a/app/plugins/Transmogrify/src/Lib/Util/CommanLinePrinter.php b/app/plugins/Transmogrify/src/Lib/Util/CommanLinePrinter.php deleted file mode 100644 index 0df0a8fb8..000000000 --- a/app/plugins/Transmogrify/src/Lib/Util/CommanLinePrinter.php +++ /dev/null @@ -1,184 +0,0 @@ -io = $io; - $this->barColor = in_array($barColor, ['blue', 'green'], true) ? $barColor : 'blue'; - $this->barWidth = max(10, $barWidth); - $this->useColors = $useColors; - } - - /** Initialize the two-area layout and draw the 0% bar */ - public function start(int $total): void - { - $this->total = max(0, $total); - $this->current = 0; - $this->messageLines = 0; - - // Draw initial progress bar line (without trailing newline) - $this->rawWrite("\r" . $this->formatBar(0)); - - // Save cursor position at the end of the progress bar line - $this->rawWrite("\033[s"); - - // Move cursor back down ONLY if messages exist - if ($this->messageLines > 0) { - $this->rawWrite("\033[" . $this->messageLines . "B\r"); - } - } - - /** Update the progress bar (0..total). Call as work advances. */ - public function update(int $current): void - { - $this->current = min(max(0, $current), $this->total); - - // Restore to saved position (progress bar line), redraw, save again - $this->rawWrite("\033[u"); // restore saved cursor (bar line) - $this->rawWrite("\r" . $this->formatBar($this->current)); - $this->rawWrite("\033[s"); // re-save position at end of bar - - // Move cursor back down to where messages continue - if ($this->messageLines > 0) { - $this->rawWrite("\033[" . $this->messageLines . "B\r"); - } else { - // exactly one line below the bar is the first message line - $this->rawWrite("\033[1B\r"); - } - } - - /** Finish: force bar to 100% and add a newline separating it from any further output */ - public function finish(): void - { - $this->update($this->total); - // Move cursor to end of messages (already there), and ensure a newline after the last message block - $this->rawWrite(PHP_EOL); - } - - // Convenience wrappers for messages under the bar - public function info(string $message): void { $this->message($message, 'info'); } - public function warn(string $message): void { $this->message($message, 'warn'); } - public function error(string $message): void { $this->message($message, 'error'); } - public function debug(string $message): void { $this->message($message, 'debug'); } - - /** Print a message under the bar. Handles multi-line strings. */ - public function message(string $message, string $level = 'info'): void - { - $lines = $this->colorizeLevel($level, $message); - // Ensure the message ends with a newline so we can count line advances - if ($lines === '' || substr($lines, -1) !== "\n") { - $lines .= "\n"; - } - - // If this is the first message, create the message area by moving to the next line - if ($this->messageLines === 0) { - $this->rawWrite(PHP_EOL); - } - - $this->rawWrite($lines); - - // Count how many terminal lines were added based on newlines seen. - // Note: This doesn’t account for soft-wrapped lines; keep messages reasonably short. - $this->messageLines += substr_count($lines, "\n"); - } - - private function colorizeLevel(string $level, string $message): string - { - $level = strtolower($level); - $prefix = ''; - $color = null; - switch ($level) { - case 'warn': - case 'warning': - $prefix = '[WARN] '; - $color = 'yellow'; - break; - case 'error': - $prefix = '[ERROR] '; - $color = 'red'; - break; - case 'debug': - $prefix = '[DEBUG] '; - $color = 'cyan'; - break; - default: - $prefix = '[INFO] '; - $color = null; // default terminal color - } - - $text = $prefix . $message; - if ($this->useColors && $color) { - return $this->wrapColor($text, $color); - } - return $text; - } - - private function formatBar(int $current): string - { - $total = max(1, $this->total); - $pct = (int) floor(($current / $total) * 100); - - $width = $this->barWidth; - $filled = (int) floor(($pct / 100) * $width); - $remaining = max(0, $width - $filled); - - $filledStr = str_repeat('=', max(0, $filled - 1)) . ($filled > 0 ? '>' : ''); - $emptyStr = str_repeat('.', $remaining); - - $bar = sprintf('[%s%s] %3d%% (%d/%d)', $filledStr, $emptyStr, $pct, $current, $this->total); - - // Clear the line to the right to avoid remnants on shorter redraws - $bar .= "\033[K"; - - if ($this->useColors) { - $color = $this->barColor === 'green' ? 'green' : 'blue'; - $bar = $this->wrapColor($bar, $color); - } - return $bar; - } - - private function wrapColor(string $text, string $color): string - { - $map = [ - 'red' => '0;31', - 'green' => '0;32', - 'yellow' => '0;33', - 'blue' => '0;34', - 'magenta'=> '0;35', - 'cyan' => '0;36', - ]; - $code = $map[$color] ?? null; - if (!$code) { return $text; } - return "\033[{$code}m{$text}\033[0m"; - } - - private function rawWrite(string $str): void - { - if ($this->io) { - // ConsoleIo::out() defaults to a trailing newline; we want raw text - $this->io->out($str, 0); - } else { - // Fallback to STDOUT - echo $str; - } - } -} diff --git a/app/plugins/Transmogrify/src/Lib/Util/CommandLinePrinter.php b/app/plugins/Transmogrify/src/Lib/Util/CommandLinePrinter.php index 91f58f3fd..41352174b 100644 --- a/app/plugins/Transmogrify/src/Lib/Util/CommandLinePrinter.php +++ b/app/plugins/Transmogrify/src/Lib/Util/CommandLinePrinter.php @@ -5,199 +5,180 @@ use Cake\Console\ConsoleIo; /** - * CommandLinePrinter (Dual-area progress + message logger) + * DualAreaProgress * - * Singleton providing a progress bar fixed on the top line while printing - * messages (info/warn/error/debug) underneath. Uses ANSI escape codes only. + * Keeps a progress bar on the top line and logs messages beneath it. + * Works with plain STDOUT or Cake's ConsoleIo. Uses ANSI escape codes. */ class CommandLinePrinter { - public static ?CommandLinePrinter $instance = null; - - private ?ConsoleIo $io; - private int $total = 0; - private int $current = 0; - private int $messageLines = 0; // number of newline-terminated lines printed under the bar - private string $barColor; // 'blue' or 'green' - private int $barWidth; - private bool $useColors; - - /** - * Private constructor to enforce singleton usage. Use initialize(). - */ - private function __construct(?ConsoleIo $io = null, string $barColor = 'blue', int $barWidth = 50, bool $useColors = true) - { - $this->io = $io; - $this->barColor = in_array($barColor, ['blue', 'green'], true) ? $barColor : 'blue'; - $this->barWidth = max(10, $barWidth); - $this->useColors = $useColors; + private ?ConsoleIo $io; + private int $total = 0; + private int $current = 0; + private int $messageLines = 0; // how many newline-terminated lines printed under the bar + private string $barColor; // 'blue' or 'green' + private int $barWidth; + private bool $useColors; + + public function __construct(?ConsoleIo $io = null, string $barColor = 'blue', int $barWidth = 50, bool $useColors = true) + { + $this->io = $io; + $this->barColor = in_array($barColor, ['blue', 'green'], true) ? $barColor : 'blue'; + $this->barWidth = max(10, $barWidth); + $this->useColors = $useColors; + } + + /** Initialize the two-area layout and draw the 0% bar */ + public function start(int $total): void + { + $this->total = max(0, $total); + $this->current = 0; + $this->messageLines = 0; + + // Draw initial progress bar line (without trailing newline) + $this->rawWrite("\r" . $this->formatBar(0)); + + // Save cursor position at the end of the progress bar line + $this->rawWrite("\033[s"); + + // Move cursor back down ONLY if messages exist + if ($this->messageLines > 0) { + $this->rawWrite("\033[" . $this->messageLines . "B\r"); } - - /** - * Initialize or reinitialize the singleton and start the progress display. - */ - public static function initialize(?ConsoleIo $io, int $total, string $barColor = 'blue', int $barWidth = 50, bool $useColors = true): self - { - self::$instance = new self($io, $barColor, $barWidth, $useColors); - self::$instance->start($total); - return self::$instance; - } - - /** - * Get the current singleton instance. If not initialized, creates a default one. - */ - public static function get(): self - { - if (!self::$instance) { - self::$instance = new self(null, 'blue', 50, true); - } - return self::$instance; - } - - /** Initialize the two-area layout and draw the 0% bar */ - public function start(int $total): void - { - $this->total = max(0, $total); - $this->current = 0; - $this->messageLines = 0; - - // Draw initial progress bar line (without trailing newline) - $this->rawWrite("\r" . $this->formatBar(0)); - - // Save cursor position at the end of the progress bar line - $this->rawWrite("\033[s"); - - // Move to the line below where messages begin - $this->rawWrite(PHP_EOL); + } + + /** Update the progress bar (0..total). Call as work advances. */ + public function update(int $current): void + { + $this->current = min(max(0, $current), $this->total); + + // Restore to saved position (progress bar line), redraw, save again + $this->rawWrite("\033[u"); // restore saved cursor (bar line) + $this->rawWrite("\r" . $this->formatBar($this->current)); + $this->rawWrite("\033[s"); // re-save position at end of bar + + // Move cursor back down to where messages continue + if ($this->messageLines > 0) { + $this->rawWrite("\033[" . $this->messageLines . "B\r"); + } else { + // exactly one line below the bar is the first message line + $this->rawWrite("\033[1B\r"); } - - /** Update the progress bar (0..total). Call as work advances. */ - public function update(int $current): void - { - $this->current = min(max(0, $current), $this->total); - - // Restore to saved position (progress bar line), redraw, save again - $this->rawWrite("\033[u"); // restore saved cursor (bar line) - $this->rawWrite("\r" . $this->formatBar($this->current)); - $this->rawWrite("\033[s"); // re-save position at end of bar - - // Move cursor back down to where messages continue - if ($this->messageLines > 0) { - $this->rawWrite("\033[" . $this->messageLines . "B\r"); - } else { - // exactly one line below the bar is the first message line - $this->rawWrite("\033[1B\r"); - } + } + + /** Finish: force bar to 100% and add a newline separating it from any further output */ + public function finish(): void + { + $this->update($this->total); + // Move cursor to end of messages (already there), and ensure a newline after the last message block + $this->rawWrite(PHP_EOL); + } + + // Convenience wrappers for messages under the bar + public function info(string $message): void { $this->message($message, 'info'); } + public function warn(string $message): void { $this->message($message, 'warn'); } + public function error(string $message): void { $this->message($message, 'error'); } + public function debug(string $message): void { $this->message($message, 'debug'); } + + /** Print a message under the bar. Handles multi-line strings. */ + public function message(string $message, string $level = 'info'): void + { + $lines = $this->colorizeLevel($level, $message); + // Ensure the message ends with a newline so we can count line advances + if ($lines === '' || substr($lines, -1) !== "\n") { + $lines .= "\n"; } - /** Finish: force bar to 100% and add a newline separating it from any further output */ - public function finish(): void - { - $this->update($this->total); - // Move cursor to end of messages (already there), and ensure a newline after the last message block - $this->rawWrite(PHP_EOL); + // If this is the first message, create the message area by moving to the next line + if ($this->messageLines === 0) { + $this->rawWrite(PHP_EOL); } - // Convenience wrappers for messages under the bar - public function info(string $message): void { $this->message($message, 'info'); } - public function warn(string $message): void { $this->message($message, 'warn'); } - public function error(string $message): void { $this->message($message, 'error'); } - public function debug(string $message): void { $this->message($message, 'debug'); } - - /** Print a message under the bar. Handles multi-line strings. */ - public function message(string $message, string $level = 'info'): void - { - $lines = $this->colorizeLevel($level, $message); - // Ensure the message ends with a newline so we can count line advances - if ($lines === '' || substr($lines, -1) !== "\n") { - $lines .= "\n"; - } - - $this->rawWrite($lines); - - // Count how many terminal lines were added based on newlines seen. - // Note: This doesn’t account for soft-wrapped lines; keep messages reasonably short. - $this->messageLines += substr_count($lines, "\n"); + $this->rawWrite($lines); + + // Count how many terminal lines were added based on newlines seen. + // Note: This doesn’t account for soft-wrapped lines; keep messages reasonably short. + $this->messageLines += substr_count($lines, "\n"); + } + + private function colorizeLevel(string $level, string $message): string + { + $level = strtolower($level); + $prefix = ''; + $color = null; + switch ($level) { + case 'warn': + case 'warning': + $prefix = '[WARN] '; + $color = 'yellow'; + break; + case 'error': + $prefix = '[ERROR] '; + $color = 'red'; + break; + case 'debug': + $prefix = '[DEBUG] '; + $color = 'cyan'; + break; + default: + $prefix = '[INFO] '; + $color = null; // default terminal color } - private function colorizeLevel(string $level, string $message): string - { - $level = strtolower($level); - $prefix = ''; - $color = null; - switch ($level) { - case 'warn': - case 'warning': - $prefix = '[WARN] '; - $color = 'yellow'; - break; - case 'error': - $prefix = '[ERROR] '; - $color = 'red'; - break; - case 'debug': - $prefix = '[DEBUG] '; - $color = 'cyan'; - break; - default: - $prefix = '[INFO] '; - $color = null; // default terminal color - } - - $text = $prefix . $message; - if ($this->useColors && $color) { - return $this->wrapColor($text, $color); - } - return $text; + $text = $prefix . $message; + if ($this->useColors && $color) { + return $this->wrapColor($text, $color); } + return $text; + } - private function formatBar(int $current): string - { - $total = max(1, $this->total); - $pct = (int) floor(($current / $total) * 100); + private function formatBar(int $current): string + { + $total = max(1, $this->total); + $pct = (int) floor(($current / $total) * 100); - $width = $this->barWidth; - $filled = (int) floor(($pct / 100) * $width); - $remaining = max(0, $width - $filled); + $width = $this->barWidth; + $filled = (int) floor(($pct / 100) * $width); + $remaining = max(0, $width - $filled); - $filledStr = str_repeat('=', max(0, $filled - 1)) . ($filled > 0 ? '>' : ''); - $emptyStr = str_repeat('.', $remaining); + $filledStr = str_repeat('=', max(0, $filled - 1)) . ($filled > 0 ? '>' : ''); + $emptyStr = str_repeat('.', $remaining); - $bar = sprintf('[%s%s] %3d%% (%d/%d)', $filledStr, $emptyStr, $pct, $current, $this->total); + $bar = sprintf('[%s%s] %3d%% (%d/%d)', $filledStr, $emptyStr, $pct, $current, $this->total); - // Clear the line to the right to avoid remnants on shorter redraws - $bar .= "\033[K"; + // Clear the line to the right to avoid remnants on shorter redraws + $bar .= "\033[K"; - if ($this->useColors) { - $color = $this->barColor === 'green' ? 'green' : 'blue'; - $bar = $this->wrapColor($bar, $color); - } - return $bar; + if ($this->useColors) { + $color = $this->barColor === 'green' ? 'green' : 'blue'; + $bar = $this->wrapColor($bar, $color); } - - private function wrapColor(string $text, string $color): string - { - $map = [ - 'red' => '0;31', - 'green' => '0;32', - 'yellow' => '0;33', - 'blue' => '0;34', - 'magenta'=> '0;35', - 'cyan' => '0;36', - ]; - $code = $map[$color] ?? null; - if (!$code) { return $text; } - return "\033[{$code}m{$text}\033[0m"; - } - - private function rawWrite(string $str): void - { - if ($this->io) { - // ConsoleIo::out() defaults to a trailing newline; we want raw text - $this->io->out($str, 0); - } else { - // Fallback to STDOUT - echo $str; - } + return $bar; + } + + private function wrapColor(string $text, string $color): string + { + $map = [ + 'red' => '0;31', + 'green' => '0;32', + 'yellow' => '0;33', + 'blue' => '0;34', + 'magenta'=> '0;35', + 'cyan' => '0;36', + ]; + $code = $map[$color] ?? null; + if (!$code) { return $text; } + return "\033[{$code}m{$text}\033[0m"; + } + + private function rawWrite(string $str): void + { + if ($this->io) { + // ConsoleIo::out() defaults to a trailing newline; we want raw text + $this->io->out($str, 0); + } else { + // Fallback to STDOUT + echo $str; } + } } From 8f771f5bf7318f7a20a47f3e26d73941cbb79edf Mon Sep 17 00:00:00 2001 From: Ioannis Igoumenos Date: Fri, 10 Oct 2025 20:30:18 +0300 Subject: [PATCH 03/15] Improvements.Fixed org identities transmogrify. --- .../Transmogrify/config/schema/tables.json | 16 +++--- .../src/Command/TransmogrifyCommand.php | 7 ++- .../src/Lib/Traits/CacheTrait.php | 1 + .../src/Lib/Traits/HookRunnersTrait.php | 6 +++ .../src/Lib/Traits/RowTransformationTrait.php | 45 ++++++++++++++--- .../src/Lib/Traits/TypeMapperTrait.php | 49 +++++++++---------- .../src/Lib/Util/RawSqlQueries.php | 25 +++++----- 7 files changed, 93 insertions(+), 56 deletions(-) diff --git a/app/plugins/Transmogrify/config/schema/tables.json b/app/plugins/Transmogrify/config/schema/tables.json index 656737920..4e84bef37 100644 --- a/app/plugins/Transmogrify/config/schema/tables.json +++ b/app/plugins/Transmogrify/config/schema/tables.json @@ -128,6 +128,8 @@ "external_identities": { "source": "cm_org_identities", "displayField": "id", + "cache": ["person_id"], + "postRow": "mapExternalIdentityToExternalIdentityRole", "fieldMap": { "co_id": null, "person_id": "&mapOrgIdentitycoPersonId", @@ -140,9 +142,7 @@ "sponsor_identifier": null, "valid_from": null, "valid_through": null - }, - "postRow": "mapExternalIdentityToExternalIdentityRole", - "cache": ["person_id"] + } }, "groups": { "source": "cm_co_groups", @@ -152,7 +152,8 @@ "fieldMap": { "auto": null, "co_group_id": "group_id", - "group_type": "?S" + "group_type": "?S", + "introduction": null }, "postTable": "createOwnersGroups" }, @@ -310,12 +311,7 @@ "source": "cm_servers", "displayField": "description", "addChangelog": false, - "cache": ["status"], - "sqlSelect": null, - "preTable": null, - "postTable": null, - "preRow": null, - "postRow": null, + "cache": ["co_id"], "fieldMap": { "plugin": "&mapServerTypeToPlugin" } diff --git a/app/plugins/Transmogrify/src/Command/TransmogrifyCommand.php b/app/plugins/Transmogrify/src/Command/TransmogrifyCommand.php index 591f855b0..0793576ea 100644 --- a/app/plugins/Transmogrify/src/Command/TransmogrifyCommand.php +++ b/app/plugins/Transmogrify/src/Command/TransmogrifyCommand.php @@ -332,16 +332,19 @@ public function execute(Arguments $args, ConsoleIo $io): int // not linked to a CO Person that was not migrated. $warns++; $progress->warn("Skipping $t record " . $row['id'] . " due to invalid foreign key: " . $e->getMessage()); + $io->ask('Press to continue...'); } catch(\InvalidArgumentException $e) { // If we can't find a value for mapping we skip the record // (ie: mapLegacyFieldNames basically requires a successful mapping) $warns++; $progress->warn("Skipping $t record " . $row['id'] . ": " . $e->getMessage()); + $io->ask('Press to continue...'); } catch(\Exception $e) { $err++; $progress->error("$t record " . $row['id'] . ": " . $e->getMessage()); + $io->ask('Press to continue...'); } $tally++; @@ -357,7 +360,7 @@ public function execute(Arguments $args, ConsoleIo $io): int $io->error(sprintf('Errors: %d', $err)); // Step 11: Execute any post-processing hooks for the table - if ($modeltableEmpty && $notSelected) { + if ($modeltableEmpty && !$notSelected) { $this->runPostTableHook($t); } @@ -370,7 +373,7 @@ public function execute(Arguments $args, ConsoleIo $io): int $io->info("This is the last table to process."); } - $io->ask('Press to continue...'); + $io->ask('Press to continue...'); } return BaseCommand::CODE_SUCCESS; diff --git a/app/plugins/Transmogrify/src/Lib/Traits/CacheTrait.php b/app/plugins/Transmogrify/src/Lib/Traits/CacheTrait.php index 6c382e9d5..84f3beebe 100644 --- a/app/plugins/Transmogrify/src/Lib/Traits/CacheTrait.php +++ b/app/plugins/Transmogrify/src/Lib/Traits/CacheTrait.php @@ -37,6 +37,7 @@ trait CacheTrait * @var array */ protected array $cache = []; + /** * Cache results as configured for the specified table. * diff --git a/app/plugins/Transmogrify/src/Lib/Traits/HookRunnersTrait.php b/app/plugins/Transmogrify/src/Lib/Traits/HookRunnersTrait.php index 888976053..9e139c230 100644 --- a/app/plugins/Transmogrify/src/Lib/Traits/HookRunnersTrait.php +++ b/app/plugins/Transmogrify/src/Lib/Traits/HookRunnersTrait.php @@ -43,6 +43,8 @@ private function runPreTableHook(string $table): void { if(!method_exists($this, $method)) { throw new \RuntimeException("Unknown preTable hook: $method"); } + + $this->io->info('Running pre-table hook: ' . ucfirst(preg_replace('/([A-Z])/', ' $1', $method))); $this->{$method}(); } @@ -60,6 +62,7 @@ private function runPostTableHook(string $table): void { if(!method_exists($this, $method)) { throw new \RuntimeException("Unknown postTable hook: $method"); } + $this->io->info('Running post-table hook: ' . ucfirst(preg_replace('/([A-Z])/', ' $1', $method))); $this->{$method}(); } @@ -79,6 +82,7 @@ private function runPreRowHook(string $table, array &$origRow, array &$row): voi if(!method_exists($this, $method)) { throw new \RuntimeException("Unknown preRow hook: $method"); } + $this->io->info('Running pre-row hook: ' . ucfirst(preg_replace('/([A-Z])/', ' $1', $method))); $this->{$method}($origRow, $row); } @@ -98,6 +102,7 @@ private function runPostRowHook(string $table, array &$origRow, array &$row): vo if(!method_exists($this, $method)) { throw new \RuntimeException("Unknown postRow hook: $method"); } + $this->io->info('Running post-row hook: ' . ucfirst(preg_replace('/([A-Z])/', ' $1', $method))); $this->{$method}($origRow, $row); } @@ -118,6 +123,7 @@ private function runSqlSelectHook(string $table, string $qualifiedTableName): st if(!method_exists(RawSqlQueries::class, $method)) { throw new \RuntimeException("Unknown sqlSelect hook: $method"); } + $this->io->info('Running SQL select hook: ' . ucfirst(preg_replace('/([A-Z])/', ' $1', $method))); return RawSqlQueries::{$method}($qualifiedTableName, $this->inconn->isMySQL()); } } \ No newline at end of file diff --git a/app/plugins/Transmogrify/src/Lib/Traits/RowTransformationTrait.php b/app/plugins/Transmogrify/src/Lib/Traits/RowTransformationTrait.php index e766b322d..35cb34b9a 100644 --- a/app/plugins/Transmogrify/src/Lib/Traits/RowTransformationTrait.php +++ b/app/plugins/Transmogrify/src/Lib/Traits/RowTransformationTrait.php @@ -298,12 +298,37 @@ protected function mapExternalIdentityToExternalIdentityRole(array $origRow, arr $roleRow[$newKey] = $origRow[$oldKey]; } - if(!empty($origRow['affiliation'])) { - $row['affiliation'] = $origRow['affiliation']; - $roleRow['affiliation_type_id'] = $this->mapAffiliationType( - row: $row, - coId: $origRow['co_id'] ?? null - ); + // Rationale: mapOrgIdentitycoPersonId accepts only the first mapping it finds + // (later mappings are treated as legacy/unpooled anomalies and ignored). + // Therefore, each ExternalIdentity produces at most one ExternalIdentityRole. + // To avoid inserting a bogus self-referential changelog link, do not carry any + // legacy key into external_identity_role_id. Start a fresh changelog chain. + $roleRow['external_identity_role_id'] = null; + $roleRow['revision'] = 0; + + + try { + if (!empty($origRow['affiliation'])) { + $row['affiliation'] = $origRow['affiliation']; + $roleRow['affiliation_type_id'] = $this->mapAffiliationType( + row: $row, + coId: $origRow['co_id'] ?? null + ); + } + } catch (\Exception $e) { + $this->io->warning("Failed to map affiliation type: " . $e->getMessage()); + if ( + isset($origRow['co_id']) + && (!isset($this->cache['cos'][$origRow['co_id']]) + || ($this->cache['cos'][$origRow['co_id']]['status']) + && $this->cache['cos'][$origRow['co_id']]['status'] == 'TR') + ) { + // This CO has been deleted, so we can't map the type. We will return null + $roleRow['affiliation_type_id'] = null; + } else { + // Rethrow the exception + throw $e; + } } $tableName = 'external_identity_roles'; @@ -313,7 +338,13 @@ protected function mapExternalIdentityToExternalIdentityRole(array $origRow, arr $this->normalizeBooleanFieldsForDb($tableName, $roleRow); $qualifiedTableName = $this->outconn->qualifyTableName($tableName); - $this->outconn->insert($qualifiedTableName, $roleRow); + + + try { + $this->outconn->insert($qualifiedTableName, $roleRow); + } catch (\Exception $e) { + $this->io->warning("record already exists: " . print_r($roleRow, true)); + } } /** diff --git a/app/plugins/Transmogrify/src/Lib/Traits/TypeMapperTrait.php b/app/plugins/Transmogrify/src/Lib/Traits/TypeMapperTrait.php index 333c45895..c3a706411 100644 --- a/app/plugins/Transmogrify/src/Lib/Traits/TypeMapperTrait.php +++ b/app/plugins/Transmogrify/src/Lib/Traits/TypeMapperTrait.php @@ -29,7 +29,7 @@ namespace Transmogrify\Lib\Traits; -use App\Lib\Enum\StatusEnum; +use Doctrine\DBAL\Exception; use Transmogrify\Lib\Util\RawSqlQueries; use Cake\Utility\Inflector; use Transmogrify\Lib\Traits\CacheTrait; @@ -176,9 +176,10 @@ protected function mapNow(array $row) { /** * Map an Org Identity ID to a CO Person ID * - * @since COmanage Registry v5.0.0 - * @param array $row Row of Org Identity table data + * @param array $row Row of Org Identity table data * @return int|null CO Person ID + * @throws Exception + * @since COmanage Registry v5.0.0 */ protected function mapOrgIdentitycoPersonId(array $row): ?int { @@ -200,7 +201,7 @@ protected function mapOrgIdentitycoPersonId(array $row): ?int // do that.) Historical information remains available in history_records, // and if the deployer keeps an archive of the old database. - $oid = (int)$row['id']; + $rowId = (int)$row['id']; if(empty($this->cache['org_identities']['co_people'])) { $this->cache['org_identities']['co_people'] = []; @@ -213,42 +214,40 @@ protected function mapOrgIdentitycoPersonId(array $row): ?int $qualifiedTableName = $this->inconn->qualifyTableName($tableName); // Only fetch current rows (historical/deleted rows are filtered out) - $mapsql = RawSqlQueries::buildSelectAll( + $mapsql = RawSqlQueries::buildSelectAllWithNoChangelong( qualifiedTableName: $qualifiedTableName, - onlyCurrent: true, changelogFK: $changelogFK ); $stmt = $this->inconn->executeQuery($mapsql); - while ($r = $stmt->fetchAssociative()) { - if (!empty($r['org_identity_id'])) { - $rOid = (int)$r['org_identity_id']; - $rOrevision = (int)$r['revision']; - $cop = isset($r['co_person_id']) ? (int)$r['co_person_id'] : null; - - if ($cop === null) { - $this->io->warning('Org Identity ' . $rOid . ' has no mapped CO Person ID, skipping'); - continue; - } + while($r = $stmt->fetchAssociative()) { + $oid = $r['org_identity_id'] ?? null; - if (isset($this->cache['org_identities']['co_people'][$rOid][$rOrevision])) { + if(!empty($oid)) { + $rowRev = $r['revision']; + if(isset($this->cache['org_identities']['co_people'][ $oid ][ $rowRev ])) { // If for some reason we already have a record, it's probably due to // improper unpooling from a legacy deployment. We'll accept only the // first record and throw warnings on the others. - $this->io->verbose('Found duplicate current CO Person mapping for Org Identity ' . $rOid . ', skipping'); - continue; - } - $this->cache['org_identities']['co_people'][$rOid][$rOrevision] = $cop; + $this->io->verbose("Found existing CO Person for Org Identity " . $oid . ", skipping"); + } else { + $this->cache['org_identities']['co_people'][ $oid ][ $rowRev ] = $r['co_person_id']; + } } } + + // Sort the array + sort($this->cache['org_identities']['co_people']); } - if (!empty($this->cache['org_identities']['co_people'][$oid])) { + if(!empty($this->cache['org_identities']['co_people'][ $rowId ])) { + // XXX OrgIdentities with no org identity link are not supported in v5 // Return the record with the highest revision number - $rev = max(array_keys($this->cache['org_identities']['co_people'][ $oid ])); - return $this->cache['org_identities']['co_people'][$oid][$rev]; + $rev = max(array_keys($this->cache['org_identities']['co_people'][ $rowId ])); + + return $this->cache['org_identities']['co_people'][ $rowId ][$rev]; } return null; @@ -302,7 +301,7 @@ protected function mapType(array $row, string $type, int $coId, string $attr = ' if(empty($this->cache['types']['co_id+attribute+value+'][$key])) { if ( !isset($this->cache['cos'][$coId]) - || ($this->cache['cos'][$coId]['status'] && $this->cache['cos'][$coId]['status'] == 'TR') + || ($this->cache['cos'][$coId]['status'] && in_array($this->cache['cos'][$coId]['status'], ['TR'])) ) { // This CO has been deleted, so we can't map the type. We will return null return null; diff --git a/app/plugins/Transmogrify/src/Lib/Util/RawSqlQueries.php b/app/plugins/Transmogrify/src/Lib/Util/RawSqlQueries.php index 6ff0bffdd..2c622ce5d 100644 --- a/app/plugins/Transmogrify/src/Lib/Util/RawSqlQueries.php +++ b/app/plugins/Transmogrify/src/Lib/Util/RawSqlQueries.php @@ -69,22 +69,23 @@ public static function buildSelectAllOrderedById(string $qualifiedTableName): st /** * Builds SQL query to select all rows, optionally filtering changelog records * @param string $qualifiedTableName Fully qualified table name - * @param bool $onlyCurrent Whether to include changelog records (default true) - * @param string|null $changelogFK Name of the changelog foreign key column * @return string SQL query string */ - public static function buildSelectAll( + public static function buildSelectAll(string $qualifiedTableName): string { + return "SELECT * FROM $qualifiedTableName"; + } + + /** + * Builds SQL query to select all rows, filtering changelog records + * @param string $qualifiedTableName Fully qualified table name + * @param string $changelogFK Changelog Foreign Key + * @return string SQL query string + */ + public static function buildSelectAllWithNoChangelong( string $qualifiedTableName, - bool $onlyCurrent = false, - string $changelogFK = null + string $changelogFK ): string { - if ($onlyCurrent) { - if (empty($changelogFK)) { - throw new \InvalidArgumentException('changelogFK is required when onlyCurrent is true'); - } - return "SELECT * FROM $qualifiedTableName WHERE $changelogFK IS NULL"; - } - return "SELECT * FROM $qualifiedTableName"; + return "SELECT * FROM $qualifiedTableName WHERE $changelogFK IS NULL"; } /** From e5a2025bdbe9ed61ea22eb8d47c24b465d6f31b6 Mon Sep 17 00:00:00 2001 From: Ioannis Igoumenos Date: Sat, 11 Oct 2025 20:54:02 +0300 Subject: [PATCH 04/15] OrgIdentities eligibility to transmogrify --- app/plugins/Transmogrify/README.md | 3 + .../Transmogrify/config/schema/tables.json | 1 + .../src/Command/TransmogrifyCommand.php | 21 ++- .../src/Lib/Traits/RowTransformationTrait.php | 7 +- .../src/Lib/Traits/TypeMapperTrait.php | 26 ++- .../src/Lib/Util/OrgIdentitiesHealth.php | 141 +++++++++++++++ .../src/Lib/Util/RawSqlQueries.php | 165 +++++++++++++++++- 7 files changed, 346 insertions(+), 18 deletions(-) create mode 100644 app/plugins/Transmogrify/src/Lib/Util/OrgIdentitiesHealth.php diff --git a/app/plugins/Transmogrify/README.md b/app/plugins/Transmogrify/README.md index 05c513bb5..5a3163fbf 100644 --- a/app/plugins/Transmogrify/README.md +++ b/app/plugins/Transmogrify/README.md @@ -60,6 +60,9 @@ These options come directly from TransmogrifyCommand::buildOptionParser. - --login-identifier-type TYPE - Identifier type value to use for login identifiers when --login-identifier-copy is set. +- --orgidentities-health + - Run Org Identities health check (eligibility/exclusion breakdown based on non-historical links and person existence) and print a transmogrification readiness report, then exit. + ## Typical usage diff --git a/app/plugins/Transmogrify/config/schema/tables.json b/app/plugins/Transmogrify/config/schema/tables.json index 4e84bef37..8e732ec5c 100644 --- a/app/plugins/Transmogrify/config/schema/tables.json +++ b/app/plugins/Transmogrify/config/schema/tables.json @@ -129,6 +129,7 @@ "source": "cm_org_identities", "displayField": "id", "cache": ["person_id"], + "sqlSelect": "orgidentitiesSqlSelect", "postRow": "mapExternalIdentityToExternalIdentityRole", "fieldMap": { "co_id": null, diff --git a/app/plugins/Transmogrify/src/Command/TransmogrifyCommand.php b/app/plugins/Transmogrify/src/Command/TransmogrifyCommand.php index 0793576ea..ff3ae1535 100644 --- a/app/plugins/Transmogrify/src/Command/TransmogrifyCommand.php +++ b/app/plugins/Transmogrify/src/Command/TransmogrifyCommand.php @@ -51,6 +51,7 @@ use Transmogrify\Lib\Util\CommandLinePrinter; use Transmogrify\Lib\Util\DbInfoPrinter; use Transmogrify\Lib\Util\RawSqlQueries; +use Transmogrify\Lib\Util\OrgIdentitiesHealth; class TransmogrifyCommand extends BaseCommand { use CacheTrait; @@ -166,6 +167,12 @@ protected function buildOptionParser(ConsoleOptionParser $parser): ConsoleOption 'help' => __d('command', 'tm.login-identifier-type') ]); + // Health report option (Org Identities readiness) + $parser->addOption('orgidentities-health', [ + 'help' => 'Run Org Identities health check (eligibility/exclusion breakdown) and exit', + 'boolean' => true + ]); + $parser->setEpilog(__d('command', 'tm.epilog')); return $parser; @@ -197,6 +204,12 @@ public function execute(Arguments $args, ConsoleIo $io): int return $code; } + // Health report: run and exit + if ($this->args->getOption('orgidentities-health')) { + OrgIdentitiesHealth::run($this->inconn, $this->io); + return BaseCommand::CODE_SUCCESS; + } + // Load tables configuration (from JSON) and extend it with schema data $this->loadTablesConfig(); @@ -273,7 +286,6 @@ public function execute(Arguments $args, ConsoleIo $io): int // Step 7: Get total count of source records for progress tracking $qualifiedTableName = $this->inconn->qualifyTableName($this->tables[$t]['source']); - $count = $this->inconn->fetchOne(RawSqlQueries::buildCountAll($qualifiedTableName)); // Step 8: Build and execute query to fetch all source records $insql = match(true) { @@ -283,6 +295,13 @@ public function execute(Arguments $args, ConsoleIo $io): int $stmt = $this->inconn->executeQuery($insql); $progress = new CommandLinePrinter($io, 'green', 50, true); + // If a custom SELECT is used, count the exact result set; otherwise count the whole table + if (!empty($this->tables[$t]['sqlSelect'])) { + $countSql = RawSqlQueries::buildCountFromSelect($insql); + $count = (int)$this->inconn->fetchOne($countSql); + } else { + $count = (int)$this->inconn->fetchOne(RawSqlQueries::buildCountAll($qualifiedTableName)); + } $progress->start($count); $tally = 0; $warns = 0; diff --git a/app/plugins/Transmogrify/src/Lib/Traits/RowTransformationTrait.php b/app/plugins/Transmogrify/src/Lib/Traits/RowTransformationTrait.php index 35cb34b9a..104376f3b 100644 --- a/app/plugins/Transmogrify/src/Lib/Traits/RowTransformationTrait.php +++ b/app/plugins/Transmogrify/src/Lib/Traits/RowTransformationTrait.php @@ -339,12 +339,7 @@ protected function mapExternalIdentityToExternalIdentityRole(array $origRow, arr $qualifiedTableName = $this->outconn->qualifyTableName($tableName); - - try { - $this->outconn->insert($qualifiedTableName, $roleRow); - } catch (\Exception $e) { - $this->io->warning("record already exists: " . print_r($roleRow, true)); - } + $this->outconn->insert($qualifiedTableName, $roleRow); } /** diff --git a/app/plugins/Transmogrify/src/Lib/Traits/TypeMapperTrait.php b/app/plugins/Transmogrify/src/Lib/Traits/TypeMapperTrait.php index c3a706411..b33945b55 100644 --- a/app/plugins/Transmogrify/src/Lib/Traits/TypeMapperTrait.php +++ b/app/plugins/Transmogrify/src/Lib/Traits/TypeMapperTrait.php @@ -203,7 +203,7 @@ protected function mapOrgIdentitycoPersonId(array $row): ?int $rowId = (int)$row['id']; - if(empty($this->cache['org_identities']['co_people'])) { + if (empty($this->cache['org_identities']['co_people'])) { $this->cache['org_identities']['co_people'] = []; // Build cache on first use @@ -213,7 +213,7 @@ protected function mapOrgIdentitycoPersonId(array $row): ?int $changelogFK = 'co_org_identity_link_id'; $qualifiedTableName = $this->inconn->qualifyTableName($tableName); - // Only fetch current rows (historical/deleted rows are filtered out) + // Only fetch current rows (historical/changelog rows are filtered out) $mapsql = RawSqlQueries::buildSelectAllWithNoChangelong( qualifiedTableName: $qualifiedTableName, changelogFK: $changelogFK @@ -221,7 +221,7 @@ protected function mapOrgIdentitycoPersonId(array $row): ?int $stmt = $this->inconn->executeQuery($mapsql); - while($r = $stmt->fetchAssociative()) { + while ($r = $stmt->fetchAssociative()) { $oid = $r['org_identity_id'] ?? null; if(!empty($oid)) { @@ -233,23 +233,31 @@ protected function mapOrgIdentitycoPersonId(array $row): ?int $this->io->verbose("Found existing CO Person for Org Identity " . $oid . ", skipping"); } else { - $this->cache['org_identities']['co_people'][ $oid ][ $rowRev ] = $r['co_person_id']; + // Store as-is; we'll resolve the latest revision on lookup + $this->cache['org_identities']['co_people'][ $oid ][ $rowRev ] = (int)$r['co_person_id']; } } } - // Sort the array - sort($this->cache['org_identities']['co_people']); + // Preserve keys while providing the deterministic order of revisions + foreach ($this->cache['org_identities']['co_people'] as &$revisions) { + if (is_array($revisions)) { + ksort($revisions, SORT_NUMERIC); + } + } + unset($revisions); } - if(!empty($this->cache['org_identities']['co_people'][ $rowId ])) { + if (!empty($this->cache['org_identities']['co_people'][$rowId])) { // XXX OrgIdentities with no org identity link are not supported in v5 // Return the record with the highest revision number - $rev = max(array_keys($this->cache['org_identities']['co_people'][ $rowId ])); + $revisions = $this->cache['org_identities']['co_people'][$rowId]; + $rev = max(array_keys($revisions)); - return $this->cache['org_identities']['co_people'][ $rowId ][$rev]; + return (int)$revisions[$rev]; } + // No current mapping found for this Org Identity return null; } diff --git a/app/plugins/Transmogrify/src/Lib/Util/OrgIdentitiesHealth.php b/app/plugins/Transmogrify/src/Lib/Util/OrgIdentitiesHealth.php new file mode 100644 index 000000000..b585c071f --- /dev/null +++ b/app/plugins/Transmogrify/src/Lib/Util/OrgIdentitiesHealth.php @@ -0,0 +1,141 @@ +out('Running Org Identities health check...'); + $sql = RawSqlQueries::ORGIDENTITIES_HEALTH_SQL_QUERY; + + try { + $rows = $inconn->fetchAllAssociative($sql); + } catch (\Throwable $e) { + $io->err('Org Identities health check failed: ' . $e->getMessage()); + return; + } + + if (empty($rows)) { + $io->out('No results.'); + return; + } + + // Detect available columns + $first = $rows[0]; + $hasIncluded = array_key_exists('included_count', $first); + $hasExcluded = array_key_exists('excluded_count', $first); + $hasIndicator = array_key_exists('indicator', $first); + $hasCount = array_key_exists('count', $first); + + // Prepare headers based on detected columns + if ($hasIncluded || $hasExcluded) { + $headers = ['Reason', 'Included', 'Excluded']; + if ($hasIndicator) { + $headers[] = 'Indicator'; + } + } else { + // Fallback to simple reason + count (and indicator if present) + $headers = ['Reason', 'Count']; + if ($hasIndicator) { + $headers[] = 'Indicator'; + } + } + + // Compute column widths + $widths = array_fill(0, count($headers), 0); + $reasonIdx = 0; + $incIdx = array_search('Included', $headers, true); + $excIdx = array_search('Excluded', $headers, true); + $cntIdx = array_search('Count', $headers, true); + $indIdx = array_search('Indicator', $headers, true); + + // Initialize with header widths + foreach ($headers as $i => $h) { + $widths[$i] = max($widths[$i], mb_strlen($h)); + } + + // Measure data + foreach ($rows as $r) { + $reasonLen = mb_strlen((string)($r['reason'] ?? '')); + $widths[$reasonIdx] = max($widths[$reasonIdx], $reasonLen); + + if ($incIdx !== false) { + $widths[$incIdx] = max($widths[$incIdx], mb_strlen((string)($r['included_count'] ?? ''))); + } + if ($excIdx !== false) { + $widths[$excIdx] = max($widths[$excIdx], mb_strlen((string)($r['excluded_count'] ?? ''))); + } + if ($cntIdx !== false) { + $widths[$cntIdx] = max($widths[$cntIdx], mb_strlen((string)($r['count'] ?? ''))); + } + if ($indIdx !== false) { + $widths[$indIdx] = max($widths[$indIdx], mb_strlen((string)($r['indicator'] ?? ''))); + } + } + + // Helper to pad a cell + $pad = static function (string $s, int $w): string { + $len = mb_strlen($s); + if ($len >= $w) { + return $s; + } + return $s . str_repeat(' ', $w - $len); + }; + + // Print header + $lineParts = []; + foreach ($headers as $i => $h) { + $lineParts[] = $pad($h, $widths[$i]); + } + $io->out(implode(' | ', $lineParts)); + + // Print separator + $sepParts = array_map(static fn($w) => str_repeat('-', $w), $widths); + $io->out(implode('--+--', $sepParts)); + + // Print rows + foreach ($rows as $r) { + $rowParts = []; + $rowParts[] = $pad((string)($r['reason'] ?? ''), $widths[$reasonIdx]); + + if ($incIdx !== false) { + $rowParts[] = $pad((string)($r['included_count'] ?? ''), $widths[$incIdx]); + } + if ($excIdx !== false) { + $rowParts[] = $pad((string)($r['excluded_count'] ?? ''), $widths[$excIdx]); + } + if ($cntIdx !== false) { + $rowParts[] = $pad((string)($r['count'] ?? ''), $widths[$cntIdx]); + } + if ($indIdx !== false) { + $indRaw = (string)($r['indicator'] ?? ''); + $cell = $pad($indRaw, $widths[$indIdx]); + + // Colorize first visible char, leave padding spaces uncolored to preserve alignment + if ($indRaw === 'x' || $indRaw === '✓') { + $color = ($indRaw === 'x') ? "\033[31m" : "\033[32m"; // red for x, green for ✓ + $reset = "\033[0m"; + $first = mb_substr($cell, 0, 1); + $rest = mb_substr($cell, 1); + $cell = $color . $first . $reset . $rest; + } + + $rowParts[] = $cell; + } + + $io->out(implode(' | ', $rowParts)); + } + } +} diff --git a/app/plugins/Transmogrify/src/Lib/Util/RawSqlQueries.php b/app/plugins/Transmogrify/src/Lib/Util/RawSqlQueries.php index 2c622ce5d..7eccfec7e 100644 --- a/app/plugins/Transmogrify/src/Lib/Util/RawSqlQueries.php +++ b/app/plugins/Transmogrify/src/Lib/Util/RawSqlQueries.php @@ -42,6 +42,7 @@ class RawSqlQueries { * Builds SQL query to get maximum ID from a table * @param string $qualifiedTableName Fully qualified table name * @return string SQL query string + * @since COmanage Registry v5.2.0 */ public static function buildSelectMaxId(string $qualifiedTableName): string { return 'SELECT MAX(id) FROM ' . $qualifiedTableName; @@ -51,15 +52,33 @@ public static function buildSelectMaxId(string $qualifiedTableName): string { * Builds SQL query to count all rows in a table * @param string $qualifiedTableName Fully qualified table name * @return string SQL query string + * @since COmanage Registry v5.2.0 */ public static function buildCountAll(string $qualifiedTableName): string { return 'SELECT COUNT(*) FROM ' . $qualifiedTableName; } + /** + * Build a portable COUNT(*) wrapper around an arbitrary SELECT statement. + * Strips a trailing ORDER BY to satisfy engines that disallow ORDER BY in subqueries. + * + * @param string $selectSql Arbitrary SELECT SQL + * @return string SQL that returns a single COUNT(*) + * @since COmanage Registry v5.2.0 + */ + public static function buildCountFromSelect(string $selectSql): string { + // Remove trailing ORDER BY ... (simple heuristic, works for our generated queries) + $sql = preg_replace('/\s+ORDER\s+BY\s+[\s\S]*$/i', '', $selectSql); + + return "SELECT COUNT(*) FROM ($sql) subq"; + } + + /** * Builds SQL query to select all rows ordered by ID ascending * @param string $qualifiedTableName Fully qualified table name * @return string SQL query string + * @since COmanage Registry v5.2.0 */ public static function buildSelectAllOrderedById(string $qualifiedTableName): string { return 'SELECT * FROM ' . $qualifiedTableName . ' ORDER BY id ASC'; @@ -70,6 +89,7 @@ public static function buildSelectAllOrderedById(string $qualifiedTableName): st * Builds SQL query to select all rows, optionally filtering changelog records * @param string $qualifiedTableName Fully qualified table name * @return string SQL query string + * @since COmanage Registry v5.2.0 */ public static function buildSelectAll(string $qualifiedTableName): string { return "SELECT * FROM $qualifiedTableName"; @@ -80,6 +100,7 @@ public static function buildSelectAll(string $qualifiedTableName): string { * @param string $qualifiedTableName Fully qualified table name * @param string $changelogFK Changelog Foreign Key * @return string SQL query string + * @since COmanage Registry v5.2.0 */ public static function buildSelectAllWithNoChangelong( string $qualifiedTableName, @@ -94,6 +115,7 @@ public static function buildSelectAllWithNoChangelong( * @param int $nextId Next ID value to set * @param bool $isMySQL Whether target database is MySQL * @return string SQL query string + * @since COmanage Registry v5.2.0 */ public static function buildSequenceReset(string $qualifiedTableName, int $nextId, bool $isMySQL): string { if($isMySQL) { @@ -173,6 +195,19 @@ public static function roleSqlSelect(string $tableName, bool $isMySQL): string { return RawSqlQueries::ROLE_SQL_SELECT; } + + /** + * Return SQL used to select Organization Identities from inbound database. + * + * @param string $tableName Name of the SQL table + * @param bool $isMySQL Whether the database is MySQL + * @return string SQL string to select rows from inbound database + * @since COmanage Registry v5.2.0 + */ + public static function orgidentitiesSqlSelect(string $tableName, bool $isMySQL): string { + return RawSqlQueries::ORG_IDENTITIES_SQL_SELECT; + } + // Any COU at any time can be made the child of another COU // and so during transmogrification we cannot simply select // the rows of the COU table by ascending id because it leads @@ -328,10 +363,32 @@ public static function roleSqlSelect(string $tableName, bool $isMySQL): string { ORDER BY MAX(generation_number) ASC; SQL; + /** + * SQL template for selecting organization identities that have at least one org identity link + */ + final const ORG_IDENTITIES_SQL_SELECT = << Date: Sat, 11 Oct 2025 21:07:43 +0300 Subject: [PATCH 05/15] Update Readme.md --- app/plugins/Transmogrify/README.md | 42 ++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/app/plugins/Transmogrify/README.md b/app/plugins/Transmogrify/README.md index 5a3163fbf..0c51ce724 100644 --- a/app/plugins/Transmogrify/README.md +++ b/app/plugins/Transmogrify/README.md @@ -126,3 +126,45 @@ Hints: - src/Command/TransmogrifyCommand.php - config/schema/tables.json - src/Lib/Traits/* (type mapping, caching, row transformations, hooks) + +## Org Identities → External Identities: Readiness and Migration Behavior + +This plugin migrates Organizational Identities to the new model introduced in COmanage Registry v5: + +- OrgIdentity was split into ExternalIdentity and ExternalIdentityRole. +- ExternalIdentity directly relates to Person (a Person may have multiple External Identities, but each External Identity belongs to a single Person). +- The CoOrgIdentityLink crosswalk was eliminated and Organizational Identity Pooling was dropped. +- affiliation was replaced by affiliation_type_id. +- o was renamed to organization. +- ou was renamed to department. +- External Identities no longer have Primary Names. +- External Identities do not carry the login flag on Identifiers. As of v5.1.0, the Identifier Mapper Pipeline Plugin can be used to set this flag. + +Eligibility (reasoning) is based on “non‑historical” links in cm_co_org_identity_links (co_org_identity_link_id IS NULL): +- A) No non‑historical link: excluded (x) +- B) Has non‑historical link(s) but all co_person_id are NULL: excluded (x) +- C) Has at least one non‑historical link with a non‑NULL co_person_id: included (✓) + +Included (✓) Org Identities are migrated as: +- One ExternalIdentity linked directly to the Person (person_id from the link) +- One ExternalIdentityRole for role‑like attributes with v5 field changes applied: + - affiliation → affiliation_type_id (type‑mapped) + - o → organization + - ou → department +- External Identities do not have Primary Names; identifier “login” flags are not set by this migration (use the Identifier Mapper Pipeline if needed) + +### Recommended preflight: Org Identities Health command + +Run this before migrating to verify which Org Identities will be included vs excluded, using the same non‑historical link reasoning: + +```bash +bin/cake transmogrify --orgidentities-health +``` + + +You’ll see a fixed‑width table with Reason, Included/Excluded counts, and an Indicator (✓ included, x excluded). Reasons are: +- A) No non‑historical link (excluded) +- B) Has non‑historical link(s) but all co_person_id are NULL (excluded) +- C) Has at least one non‑historical link with a non‑NULL co_person_id (included) + +Totals summarize overall readiness. Use this report to address data conditions (eg, missing person links) so that important Org Identities are eligible for migration. From af48ba518cd8c2a4b3c80df7812ff2cdb140893b Mon Sep 17 00:00:00 2001 From: Ioannis Igoumenos Date: Sun, 12 Oct 2025 12:38:48 +0300 Subject: [PATCH 06/15] Fix groups transmogrify according to AR-G-9 --- app/plugins/Transmogrify/README.md | 43 +++++ .../Transmogrify/config/schema/tables.json | 1 + .../src/Command/TransmogrifyCommand.php | 160 +++++++++++------- .../src/Lib/Traits/RowTransformationTrait.php | 56 +++++- .../src/Lib/Traits/TypeMapperTrait.php | 12 +- .../src/Lib/Util/GroupsHealth.php | 141 +++++++++++++++ .../src/Lib/Util/RawSqlQueries.php | 80 +++++++-- 7 files changed, 409 insertions(+), 84 deletions(-) create mode 100644 app/plugins/Transmogrify/src/Lib/Util/GroupsHealth.php diff --git a/app/plugins/Transmogrify/README.md b/app/plugins/Transmogrify/README.md index 0c51ce724..c5a13dbbe 100644 --- a/app/plugins/Transmogrify/README.md +++ b/app/plugins/Transmogrify/README.md @@ -63,6 +63,12 @@ These options come directly from TransmogrifyCommand::buildOptionParser. - --orgidentities-health - Run Org Identities health check (eligibility/exclusion breakdown based on non-historical links and person existence) and print a transmogrification readiness report, then exit. +- --groups-health + - Run Groups health check (AR‑Group‑9: invalid Standard group names) and print a transmogrification readiness report, then exit. + +- --groups-colon-replacement STRING + - Optional: replace ":" with STRING in Standard group names during migration (opt‑in). Use with care; the name "CO" remains invalid and will not be auto‑renamed. + ## Typical usage @@ -168,3 +174,40 @@ You’ll see a fixed‑width table with Reason, Included/Excluded counts, and an - C) Has at least one non‑historical link with a non‑NULL co_person_id (included) Totals summarize overall readiness. Use this report to address data conditions (eg, missing person links) so that important Org Identities are eligible for migration. + +### Recommended preflight: Groups Health command (Standard naming rule) + +Transmogrify enforces a naming rule for Standard groups: a Standard group is considered invalid if its name contains a colon (:) or equals “CO” (case‑insensitive, trimmed). Invalid Standard groups will not be migrated by default and require admin action (eg, rename) before proceeding. + +Run this health check to see how many groups are affected and where action is needed: + +```bash +bin/cake transmogrify --groups-health +``` + +You’ll see a fixed‑width table with Reason, Included/Excluded counts, and an Indicator (✓ valid/eligible, x invalid/action required). Reasons are: +- Invalid: Standard group name contains “:” (excluded) + - Standard groups (type S) whose name includes a colon are considered invalid by default and require renaming before migration. + +- Invalid: Standard group name equals “CO” (excluded) + - Standard groups (type S) named exactly “CO” (case‑insensitive, trimmed) are considered invalid and require renaming. + +- Valid: Does not violate the naming rule (included) + - All other groups: either not Standard type, or Standard whose name does not contain “:” and is not exactly “CO”. + +Totals summarize overall readiness: +- Invalid (total): total number of invalid Standard groups (sum of the invalid reasons). +- Valid (total): total number of groups eligible to migrate without renaming. +- Total Groups: grand total of groups evaluated (valid + invalid). + +Use this report to identify Standard groups that must be renamed to comply with the rule. After remediation, re‑run the health check and verify that Invalid (total) is 0 and all groups you intend to migrate appear under Valid. + +Optional remediation helper (opt‑in): colon replacement +- By default, Transmogrify does not change group names and will error on invalid Standard names. +- You can opt in to replace “:” in Standard group names with a safer character or string during migration. The special name “CO” remains invalid and is not auto‑renamed. + +Example (replace ":" with "-"): + +```bash +bin/cake transmogrify --groups-colon-replacement '-' +``` diff --git a/app/plugins/Transmogrify/config/schema/tables.json b/app/plugins/Transmogrify/config/schema/tables.json index 8e732ec5c..d35bbfac6 100644 --- a/app/plugins/Transmogrify/config/schema/tables.json +++ b/app/plugins/Transmogrify/config/schema/tables.json @@ -150,6 +150,7 @@ "displayField": "name", "cache": ["co_id", "owners_group_id"], "booleans": ["nesting_mode_all", "open"], + "preRow": "applyCheckGroupNameARRule", "fieldMap": { "auto": null, "co_group_id": "group_id", diff --git a/app/plugins/Transmogrify/src/Command/TransmogrifyCommand.php b/app/plugins/Transmogrify/src/Command/TransmogrifyCommand.php index ff3ae1535..27519b25f 100644 --- a/app/plugins/Transmogrify/src/Command/TransmogrifyCommand.php +++ b/app/plugins/Transmogrify/src/Command/TransmogrifyCommand.php @@ -18,7 +18,7 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * + * * @link https://www.internet2.edu/comanage COmanage Project * @package registry * @since COmanage Registry v5.0.0 @@ -52,6 +52,7 @@ use Transmogrify\Lib\Util\DbInfoPrinter; use Transmogrify\Lib\Util\RawSqlQueries; use Transmogrify\Lib\Util\OrgIdentitiesHealth; +use Transmogrify\Lib\Util\GroupsHealth; class TransmogrifyCommand extends BaseCommand { use CacheTrait; @@ -116,66 +117,82 @@ public function run(array $argv, ConsoleIo $io): int * @param ConsoleOptionParser $parser ConsoleOptionParser * @return ConsoleOptionParser ConsoleOptionParser */ - + protected function buildOptionParser(ConsoleOptionParser $parser): ConsoleOptionParser { - // Allow overriding the tables config path - $parser->addOption('tables-config', [ - 'help' => 'Path to transmogrify tables JSON config', - 'default' => TransmogrifyEnum::TABLES_JSON_PATH - ]); - $parser->addOption('dump-tables-config', [ - 'help' => 'Output the effective tables configuration (after schema extension) and exit', - 'boolean' => true - ]); - // Specify a table (or repeat option) to migrate only a subset - $parser->addOption('table', [ - 'help' => 'Migrate only the specified table. Repeat the option to migrate multiple tables', - 'multiple' => true - ]); - // List available target tables and exit - $parser->addOption('list-tables', [ - 'help' => 'List available target tables from the transmogrify config and exit', - 'boolean' => true - ]); - // Info options integrated into TransmogrifyCommand - $parser->addOption('info', [ - 'help' => 'Print source and target database configuration and exit', - 'boolean' => true - ]); - $parser->addOption('info-json', [ - 'help' => 'Output info in JSON (use with --info)', - 'boolean' => true - ]); - $parser->addOption('info-ping', [ - 'help' => 'Ping connections and include connectivity + server version (use with --info or --info-schema)', - 'boolean' => true - ]); - $parser->addOption('info-schema', [ - 'help' => 'Print schema information and whether the database is empty (defaults to target). Use --info-schema-role to select source/target', - 'boolean' => true - ]); - $parser->addOption('info-schema-role', [ - 'help' => 'When using --info-schema, which database to inspect: source or target (default: target)' - ]); - $parser->addOption('login-identifier-copy', [ - 'help' => __d('command', 'tm.login-identifier-copy'), - 'boolean' => true - ]); - - $parser->addOption('login-identifier-type', [ - 'help' => __d('command', 'tm.login-identifier-type') - ]); - - // Health report option (Org Identities readiness) - $parser->addOption('orgidentities-health', [ - 'help' => 'Run Org Identities health check (eligibility/exclusion breakdown) and exit', - 'boolean' => true - ]); - - $parser->setEpilog(__d('command', 'tm.epilog')); - - return $parser; + // Allow overriding the tables config path + $parser->addOption('tables-config', [ + 'help' => 'Path to transmogrify tables JSON config', + 'default' => TransmogrifyEnum::TABLES_JSON_PATH + ]); + $parser->addOption('dump-tables-config', [ + 'help' => 'Output the effective tables configuration (after schema extension) and exit', + 'boolean' => true + ]); + // Specify a table (or repeat option) to migrate only a subset + $parser->addOption('table', [ + 'help' => 'Migrate only the specified table. Repeat the option to migrate multiple tables', + 'multiple' => true + ]); + // List available target tables and exit + $parser->addOption('list-tables', [ + 'help' => 'List available target tables from the transmogrify config and exit', + 'boolean' => true + ]); + // Info options integrated into TransmogrifyCommand + $parser->addOption('info', [ + 'help' => 'Print source and target database configuration and exit', + 'boolean' => true + ]); + $parser->addOption('info-json', [ + 'help' => 'Output info in JSON (use with --info)', + 'boolean' => true + ]); + $parser->addOption('info-ping', [ + 'help' => 'Ping connections and include connectivity + server version (use with --info or --info-schema)', + 'boolean' => true + ]); + $parser->addOption('info-schema', [ + 'help' => 'Print schema information and whether the database is empty (defaults to target). Use --info-schema-role to select source/target', + 'boolean' => true + ]); + $parser->addOption('info-schema-role', [ + 'help' => 'When using --info-schema, which database to inspect: source or target (default: target)' + ]); + $parser->addOption('login-identifier-copy', [ + 'help' => __d('command', 'tm.login-identifier-copy'), + 'boolean' => true + ]); + + $parser->addOption('login-identifier-type', [ + 'help' => __d('command', 'tm.login-identifier-type') + ]); + + // Health report option (Org Identities readiness) + $parser->addOption('orgidentities-health', [ + 'help' => 'Run Org Identities health check (eligibility/exclusion breakdown) and exit', + 'boolean' => true + ]); + // Health report option (Groups naming rule readiness) + $parser->addOption('groups-health', [ + 'help' => 'Run Groups health check (AR-Group-9: invalid Standard names) and exit', + 'boolean' => true + ]); + // Optional: replace colons in Standard group names during migration (opt-in, off by default) + $parser->addOption('groups-colon-replacement', [ + 'help' => 'If set, replace ":" with this value in Standard group names during migration. WARNING: name "CO" remains invalid and is not auto-renamed.' + ]); + // Convenience flag for using a literal dash as replacement + $parser->addOption('groups-colon-replacement-dash', [ + 'help' => 'Use "-" as the replacement for ":" in Standard group names (shorthand when passing a lone "-" is problematic)', + 'boolean' => true + ]); + + + + $parser->setEpilog(__d('command', 'tm.epilog')); + + return $parser; } /** @@ -209,6 +226,10 @@ public function execute(Arguments $args, ConsoleIo $io): int OrgIdentitiesHealth::run($this->inconn, $this->io); return BaseCommand::CODE_SUCCESS; } + if ($this->args->getOption('groups-health')) { + GroupsHealth::run($this->inconn, $this->io); + return BaseCommand::CODE_SUCCESS; + } // Load tables configuration (from JSON) and extend it with schema data $this->loadTablesConfig(); @@ -237,7 +258,13 @@ public function execute(Arguments $args, ConsoleIo $io): int // Register the current version for future upgrade purposes $this->metaTable = TableRegistry::getTableLocator()->get('Meta'); $this->metaTable->setUpgradeVersion(); - + + // Track remaining selected tables (if any) so we can exit early when done + $pendingSelected = []; + if (!empty($selected)) { + $pendingSelected = array_fill_keys($selected, true); + } + foreach(array_keys($this->tables) as $t) { $modeltableEmpty = true; $notSelected = false; @@ -283,7 +310,7 @@ public function execute(Arguments $args, ConsoleIo $io): int $modeltableEmpty = false; $io->warning("Table (" . $t . ") is not empty. We will not overwrite existing data."); } - + // Step 7: Get total count of source records for progress tracking $qualifiedTableName = $this->inconn->qualifyTableName($this->tables[$t]['source']); @@ -383,6 +410,15 @@ public function execute(Arguments $args, ConsoleIo $io): int $this->runPostTableHook($t); } + // If user selected a subset, exit as soon as all selected tables are processed + if (!empty($pendingSelected) && isset($pendingSelected[$t])) { + unset($pendingSelected[$t]); + if (empty($pendingSelected)) { + $io->info('All selected tables have been processed. Exiting.'); + return BaseCommand::CODE_SUCCESS; + } + } + // Prompt for confirmation before processing table $tables = array_keys($this->tables); $currentIndex = array_search($t, $tables); @@ -397,7 +433,7 @@ public function execute(Arguments $args, ConsoleIo $io): int return BaseCommand::CODE_SUCCESS; } - + /** * Validate incompatible/invalid "info" related options. diff --git a/app/plugins/Transmogrify/src/Lib/Traits/RowTransformationTrait.php b/app/plugins/Transmogrify/src/Lib/Traits/RowTransformationTrait.php index 104376f3b..b0065913a 100644 --- a/app/plugins/Transmogrify/src/Lib/Traits/RowTransformationTrait.php +++ b/app/plugins/Transmogrify/src/Lib/Traits/RowTransformationTrait.php @@ -30,11 +30,62 @@ namespace Transmogrify\Lib\Traits; use App\Command\Util\InvalidArgumentException; +use App\Lib\Enum\GroupTypeEnum; use Transmogrify\Lib\Util\RawSqlQueries; use Doctrine\DBAL\Exception; trait RowTransformationTrait { + + /** + * Apply validation rules for CoGroup names during transformation + * + * @param array $origRow Original row data from source database + * @param array $row Row data to be transformed, passed by reference + * @return void + * @throws \InvalidArgumentException If group name validation fails + */ + protected function applyCheckGroupNameARRule(array $origRow, array &$row): void + { + // Default selection rule for CoGroups (AR-Group-9): + // For any Standard ("S") group: + // - If name equals 'CO' (case-insensitive, trimmed), throw and skip migration (always). + // - If name contains ':', by default throw and skip migration. + // If the optional --groups-colon-replacement flag is set, replace ':' with the provided string instead. + + $gtype = $origRow['group_type'] ?? null; + $name = (string)($origRow['name'] ?? ''); + + if ($gtype === GroupTypeEnum::Standard) { + // Disallow exact "CO" (case-insensitive, trimmed) unconditionally + if ($name !== '' && strtoupper(trim($name)) === 'CO') { + throw new \InvalidArgumentException('Standard CoGroup name "CO" is invalid by default and cannot be auto-renamed'); + } + + // Handle colon rule + if ($name !== '' && str_contains($name, ':')) { + // Optional, opt-in replacement + $replacement = (string)($this->args?->getOption('groups-colon-replacement') ?? ''); + if ($replacement === '' && ($this->args?->getOption('groups-colon-replacement-dash') === true)) { + $replacement = '-'; + } + if ($replacement !== '') { + $newName = str_replace(':', $replacement, $name); + if ($newName === '') { + // Guard against accidental empty name + throw new \InvalidArgumentException('Replacing ":" produced an empty Standard CoGroup name; adjust --groups-colon-replacement'); + } + $row['name'] = $newName; + $this->io?->verbose(sprintf('Replaced ":" in Standard CoGroup name "%s" -> "%s"', $name, $newName)); + } else { + // Default: error out (no auto-replacement) + throw new \InvalidArgumentException('Standard CoGroup names cannot contain a colon by default: ' . $name); + } + } + } + } + + /** * Check if a group membership is actually asserted, and reassign ownerships. * @@ -332,7 +383,7 @@ protected function mapExternalIdentityToExternalIdentityRole(array $origRow, arr } $tableName = 'external_identity_roles'; - + // Fix up changelog and booleans prior to insert $this->populateChangelogDefaults($tableName, $roleRow, true); $this->normalizeBooleanFieldsForDb($tableName, $roleRow); @@ -354,6 +405,7 @@ private function performNoMapping(array &$row, string $oldname): void unset($row[$oldname]); } + /** * Compute value for a field via a mapping function name (without the leading &). * Reuses the original field name in-place. Throws if the mapping yields a falsy value. @@ -363,7 +415,7 @@ private function performNoMapping(array &$row, string $oldname): void * @param string $funcName Name of mapping function to call * @param string $table Table name for error reporting * @return void - * @throws \InvalidArgumentException When mapping returns falsy value + * @throws \InvalidArgumentException When mapping returns falsy value or function not found */ private function performFunctionMapping(array &$row, string $oldname, string $funcName, string $table): void { diff --git a/app/plugins/Transmogrify/src/Lib/Traits/TypeMapperTrait.php b/app/plugins/Transmogrify/src/Lib/Traits/TypeMapperTrait.php index b33945b55..ee621f9c4 100644 --- a/app/plugins/Transmogrify/src/Lib/Traits/TypeMapperTrait.php +++ b/app/plugins/Transmogrify/src/Lib/Traits/TypeMapperTrait.php @@ -59,12 +59,12 @@ trait TypeMapperTrait public const SERVER_TYPE_MAP = [ 'HT' => 'CoreServer.HttpServers', - 'KA' => 'CoreServer.KafkaServerS', - 'KC' => 'CoreServer.KdcServerS', - 'LD' => 'CoreServer.LdapServerS', - 'MT' => 'CoreServer.MatchServerS', - 'O2' => 'CoreServer.Oauth2ServerS', - 'SQ' => 'CoreServer.SqlServerS', + 'KA' => 'CoreServer.KafkaServers', + 'KC' => 'CoreServer.KdcServers', + 'LD' => 'CoreServer.LdapServers', + 'MT' => 'CoreServer.MatchServers', + 'O2' => 'CoreServer.Oauth2Servers', + 'SQ' => 'CoreServer.SqlServers', ]; /** diff --git a/app/plugins/Transmogrify/src/Lib/Util/GroupsHealth.php b/app/plugins/Transmogrify/src/Lib/Util/GroupsHealth.php new file mode 100644 index 000000000..5e7a68981 --- /dev/null +++ b/app/plugins/Transmogrify/src/Lib/Util/GroupsHealth.php @@ -0,0 +1,141 @@ +out('Running Groups health check (AR-Group-9)...'); + $sql = RawSqlQueries::STANDARD_GROUP_ARG9_SQL_QUERY; + + try { + $rows = $inconn->fetchAllAssociative($sql); + } catch (\Throwable $e) { + $io->err('Groups health check failed: ' . $e->getMessage()); + return; + } + + if (empty($rows)) { + $io->out('No results.'); + return; + } + + // Detect available columns + $first = $rows[0]; + $hasIncluded = array_key_exists('included_count', $first); + $hasExcluded = array_key_exists('excluded_count', $first); + $hasIndicator = array_key_exists('indicator', $first); + $hasCount = array_key_exists('count', $first); + + // Prepare headers based on detected columns + if ($hasIncluded || $hasExcluded) { + $headers = ['Reason', 'Included', 'Excluded']; + if ($hasIndicator) { + $headers[] = 'Indicator'; + } + } else { + // Fallback to simple reason + count (and indicator if present) + $headers = ['Reason', 'Count']; + if ($hasIndicator) { + $headers[] = 'Indicator'; + } + } + + // Compute column widths + $widths = array_fill(0, count($headers), 0); + $reasonIdx = 0; + $incIdx = array_search('Included', $headers, true); + $excIdx = array_search('Excluded', $headers, true); + $cntIdx = array_search('Count', $headers, true); + $indIdx = array_search('Indicator', $headers, true); + + // Initialize with header widths + foreach ($headers as $i => $h) { + $widths[$i] = max($widths[$i], mb_strlen($h)); + } + + // Measure data + foreach ($rows as $r) { + $reasonLen = mb_strlen((string)($r['reason'] ?? '')); + $widths[$reasonIdx] = max($widths[$reasonIdx], $reasonLen); + + if ($incIdx !== false) { + $widths[$incIdx] = max($widths[$incIdx], mb_strlen((string)($r['included_count'] ?? ''))); + } + if ($excIdx !== false) { + $widths[$excIdx] = max($widths[$excIdx], mb_strlen((string)($r['excluded_count'] ?? ''))); + } + if ($cntIdx !== false) { + $widths[$cntIdx] = max($widths[$cntIdx], mb_strlen((string)($r['count'] ?? ''))); + } + if ($indIdx !== false) { + $widths[$indIdx] = max($widths[$indIdx], mb_strlen((string)($r['indicator'] ?? ''))); + } + } + + // Helper to pad a cell + $pad = static function (string $s, int $w): string { + $len = mb_strlen($s); + if ($len >= $w) { + return $s; + } + return $s . str_repeat(' ', $w - $len); + }; + + // Print header + $lineParts = []; + foreach ($headers as $i => $h) { + $lineParts[] = $pad($h, $widths[$i]); + } + $io->out(implode(' | ', $lineParts)); + + // Print separator + $sepParts = array_map(static fn($w) => str_repeat('-', $w), $widths); + $io->out(implode('--+--', $sepParts)); + + // Print rows + foreach ($rows as $r) { + $rowParts = []; + $rowParts[] = $pad((string)($r['reason'] ?? ''), $widths[$reasonIdx]); + + if ($incIdx !== false) { + $rowParts[] = $pad((string)($r['included_count'] ?? ''), $widths[$incIdx]); + } + if ($excIdx !== false) { + $rowParts[] = $pad((string)($r['excluded_count'] ?? ''), $widths[$excIdx]); + } + if ($cntIdx !== false) { + $rowParts[] = $pad((string)($r['count'] ?? ''), $widths[$cntIdx]); + } + if ($indIdx !== false) { + $indRaw = (string)($r['indicator'] ?? ''); + $cell = $pad($indRaw, $widths[$indIdx]); + + // Colorize first visible char, leave padding spaces uncolored to preserve alignment + if ($indRaw === 'x' || $indRaw === '✓') { + $color = ($indRaw === 'x') ? "\033[31m" : "\033[32m"; // red for x, green for ✓ + $reset = "\033[0m"; + $first = mb_substr($cell, 0, 1); + $rest = mb_substr($cell, 1); + $cell = $color . $first . $reset . $rest; + } + + $rowParts[] = $cell; + } + + $io->out(implode(' | ', $rowParts)); + } + } +} diff --git a/app/plugins/Transmogrify/src/Lib/Util/RawSqlQueries.php b/app/plugins/Transmogrify/src/Lib/Util/RawSqlQueries.php index 7eccfec7e..28b7ecf2c 100644 --- a/app/plugins/Transmogrify/src/Lib/Util/RawSqlQueries.php +++ b/app/plugins/Transmogrify/src/Lib/Util/RawSqlQueries.php @@ -37,7 +37,6 @@ * Contains reusable SQL template builders and specialized queries for handling COUs and roles */ class RawSqlQueries { - // Generic SQL builders and templates moved from TransmogrifyCommand /** * Builds SQL query to get maximum ID from a table * @param string $qualifiedTableName Fully qualified table name @@ -98,8 +97,8 @@ public static function buildSelectAll(string $qualifiedTableName): string { /** * Builds SQL query to select all rows, filtering changelog records * @param string $qualifiedTableName Fully qualified table name - * @param string $changelogFK Changelog Foreign Key - * @return string SQL query string + * @param string $changelogFK Changelog Foreign Key column name + * @return string SQL query returning all non-changelog records * @since COmanage Registry v5.2.0 */ public static function buildSelectAllWithNoChangelong( @@ -111,10 +110,10 @@ public static function buildSelectAllWithNoChangelong( /** * Builds SQL query to reset sequence/auto-increment value for a table - * @param string $qualifiedTableName Fully qualified table name - * @param int $nextId Next ID value to set - * @param bool $isMySQL Whether target database is MySQL - * @return string SQL query string + * @param string $qualifiedTableName Complete table name including schema if applicable + * @param int $nextId Next sequence/auto-increment value to set + * @param bool $isMySQL True if target is MySQL, false for PostgreSQL + * @return string SQL query string to reset sequence * @since COmanage Registry v5.2.0 */ public static function buildSequenceReset(string $qualifiedTableName, int $nextId, bool $isMySQL): string { @@ -171,7 +170,6 @@ public static function setSequenceId( * @param bool $isMySQL Whether the database is MySQL * @return string SQL string to select rows from inbound database */ - public static function couSqlSelect(string $tableName, bool $isMySQL): string { if($isMySQL) { $sqlTemplate = RawSqlQueries::COU_SQL_SELECT_TEMPLATE_MYSQL; @@ -190,18 +188,16 @@ public static function couSqlSelect(string $tableName, bool $isMySQL): string { * @param bool $isMySQL Whether the database is MySQL * @return string SQL string to select rows from inbound database */ - public static function roleSqlSelect(string $tableName, bool $isMySQL): string { return RawSqlQueries::ROLE_SQL_SELECT; } - /** * Return SQL used to select Organization Identities from inbound database. * - * @param string $tableName Name of the SQL table - * @param bool $isMySQL Whether the database is MySQL - * @return string SQL string to select rows from inbound database + * @param string $tableName Name of the database table containing organization identities + * @param bool $isMySQL Whether the target database is MySQL (true) or PostgreSQL (false) + * @return string SQL string to select organization identity rows from inbound database * @since COmanage Registry v5.2.0 */ public static function orgidentitiesSqlSelect(string $tableName, bool $isMySQL): string { @@ -580,4 +576,60 @@ public static function orgidentitiesSqlSelect(string $tableName, bool $isMySQL): FROM cm_org_identities ORDER BY reason; SQL; -} \ No newline at end of file + + /** + * SQL template for checking health of groups names against AR-Group-9 rule + * - Invalid when group_type = 'S' and (name contains ":" OR name equals "CO" case-insensitively) + * - Produces human-readable reasons with included/excluded columns and an indicator + */ + final const STANDARD_GROUP_ARG9_SQL_QUERY = << Date: Tue, 14 Oct 2025 11:56:27 +0300 Subject: [PATCH 07/15] fix group members insert throwing exception --- app/plugins/Transmogrify/README.md | 10 ++++++++-- .../src/Command/TransmogrifyCommand.php | 16 ++++++++++++++++ .../src/Lib/Traits/HookRunnersTrait.php | 10 +++++----- .../src/Lib/Traits/RowTransformationTrait.php | 4 +++- 4 files changed, 32 insertions(+), 8 deletions(-) diff --git a/app/plugins/Transmogrify/README.md b/app/plugins/Transmogrify/README.md index c5a13dbbe..0826dad97 100644 --- a/app/plugins/Transmogrify/README.md +++ b/app/plugins/Transmogrify/README.md @@ -206,8 +206,14 @@ Optional remediation helper (opt‑in): colon replacement - By default, Transmogrify does not change group names and will error on invalid Standard names. - You can opt in to replace “:” in Standard group names with a safer character or string during migration. The special name “CO” remains invalid and is not auto‑renamed. -Example (replace ":" with "-"): +Example (replace ":" with "-"), shorthand when passing a lone "-" is problematic: ```bash -bin/cake transmogrify --groups-colon-replacement '-' +bin/cake transmogrify --groups-colon-replacement-dash +``` + +For every other character, use the full option: + +```bash +bin/cake transmogrify --groups-colon-replacement '@' ``` diff --git a/app/plugins/Transmogrify/src/Command/TransmogrifyCommand.php b/app/plugins/Transmogrify/src/Command/TransmogrifyCommand.php index 27519b25f..57df6ae87 100644 --- a/app/plugins/Transmogrify/src/Command/TransmogrifyCommand.php +++ b/app/plugins/Transmogrify/src/Command/TransmogrifyCommand.php @@ -31,6 +31,7 @@ use App\Lib\Traits\LabeledLogTrait; use App\Lib\Util\DBALConnection; +use App\Lib\Util\StringUtilities; use App\Model\Table\MetaTable; use App\Service\ConfigLoaderService; use App\Service\DbInfoService; @@ -364,6 +365,21 @@ public function execute(Arguments $args, ConsoleIo $io): int $qualifiedTableName = $this->outconn->qualifyTableName($t); if($modeltableEmpty && !$notSelected) { + $fkQualifiedTableName = StringUtilities::classNameToForeignKey($qualifiedTableName); + if ( + isset($this->cache['rejected']) + && !empty($this->cache['rejected'][$qualifiedTableName][$row[$fkQualifiedTableName]]) + ) { + // This row will be rejected because it references a parent record that does not exist. + // The parent record has been rejected before, so we can't insert this record. + $this->cache['rejected'][$qualifiedTableName][$row['id']] = $row; + $io->warning(sprintf( + 'Skipping record %d in table %s - parent record does not exist', + $row['id'] ?? 0, + $t + )); + continue; + } $this->outconn->insert($qualifiedTableName, $row); // Step 8: Execute any post-processing hooks after successful insertion $this->runPostRowHook($t, $origRow, $row); diff --git a/app/plugins/Transmogrify/src/Lib/Traits/HookRunnersTrait.php b/app/plugins/Transmogrify/src/Lib/Traits/HookRunnersTrait.php index 9e139c230..757e6bc5b 100644 --- a/app/plugins/Transmogrify/src/Lib/Traits/HookRunnersTrait.php +++ b/app/plugins/Transmogrify/src/Lib/Traits/HookRunnersTrait.php @@ -44,7 +44,7 @@ private function runPreTableHook(string $table): void { throw new \RuntimeException("Unknown preTable hook: $method"); } - $this->io->info('Running pre-table hook: ' . ucfirst(preg_replace('/([A-Z])/', ' $1', $method))); + $this->io->verbose('Running pre-table hook: ' . ucfirst(preg_replace('/([A-Z])/', ' $1', $method))); $this->{$method}(); } @@ -62,7 +62,7 @@ private function runPostTableHook(string $table): void { if(!method_exists($this, $method)) { throw new \RuntimeException("Unknown postTable hook: $method"); } - $this->io->info('Running post-table hook: ' . ucfirst(preg_replace('/([A-Z])/', ' $1', $method))); + $this->io->verbose('Running post-table hook: ' . ucfirst(preg_replace('/([A-Z])/', ' $1', $method))); $this->{$method}(); } @@ -82,7 +82,7 @@ private function runPreRowHook(string $table, array &$origRow, array &$row): voi if(!method_exists($this, $method)) { throw new \RuntimeException("Unknown preRow hook: $method"); } - $this->io->info('Running pre-row hook: ' . ucfirst(preg_replace('/([A-Z])/', ' $1', $method))); + $this->io->verbose('Running pre-row hook: ' . ucfirst(preg_replace('/([A-Z])/', ' $1', $method))); $this->{$method}($origRow, $row); } @@ -102,7 +102,7 @@ private function runPostRowHook(string $table, array &$origRow, array &$row): vo if(!method_exists($this, $method)) { throw new \RuntimeException("Unknown postRow hook: $method"); } - $this->io->info('Running post-row hook: ' . ucfirst(preg_replace('/([A-Z])/', ' $1', $method))); + $this->io->verbose('Running post-row hook: ' . ucfirst(preg_replace('/([A-Z])/', ' $1', $method))); $this->{$method}($origRow, $row); } @@ -123,7 +123,7 @@ private function runSqlSelectHook(string $table, string $qualifiedTableName): st if(!method_exists(RawSqlQueries::class, $method)) { throw new \RuntimeException("Unknown sqlSelect hook: $method"); } - $this->io->info('Running SQL select hook: ' . ucfirst(preg_replace('/([A-Z])/', ' $1', $method))); + $this->io->verbose('Running SQL select hook: ' . ucfirst(preg_replace('/([A-Z])/', ' $1', $method))); return RawSqlQueries::{$method}($qualifiedTableName, $this->inconn->isMySQL()); } } \ No newline at end of file diff --git a/app/plugins/Transmogrify/src/Lib/Traits/RowTransformationTrait.php b/app/plugins/Transmogrify/src/Lib/Traits/RowTransformationTrait.php index b0065913a..5516bbda1 100644 --- a/app/plugins/Transmogrify/src/Lib/Traits/RowTransformationTrait.php +++ b/app/plugins/Transmogrify/src/Lib/Traits/RowTransformationTrait.php @@ -103,6 +103,7 @@ protected function reconcileGroupMembershipOwnership(array $origRow, array &$row // in invalid membership // (3) Otherwise just return so the Membership gets created + $tableName = 'group_members'; if($origRow['owner'] && !$origRow['deleted'] && !$origRow['co_group_member_id']) { // Create a membership in the appropriate owners group, but not // on changelog entries @@ -119,7 +120,6 @@ protected function reconcileGroupMembershipOwnership(array $origRow, array &$row 'actor_identifier' => $origRow['actor_identifier'] ]; - $tableName = 'group_members'; $qualifiedTableName = $this->outconn->qualifyTableName($tableName); $this->outconn->insert($qualifiedTableName, $ownerRow); } else { @@ -128,6 +128,8 @@ protected function reconcileGroupMembershipOwnership(array $origRow, array &$row } if(!$row['member'] && !$row['owner']) { + // we are caching the rejected rows so we can reject the rows that reference them as well. + $this->cache['rejected'][$tableName][$row['id']] = $row; throw new \InvalidArgumentException('member not set on GroupMember'); } } From babf88c3bea6766dab2cb6601d3cac6cad5c38ed Mon Sep 17 00:00:00 2001 From: Ioannis Igoumenos Date: Tue, 14 Oct 2025 14:12:47 +0300 Subject: [PATCH 08/15] MVEA for person and orgidentity. --- .../Transmogrify/config/schema/tables.json | 67 +++++++------- .../src/Lib/Traits/CacheTrait.php | 6 +- .../src/Lib/Traits/TypeMapperTrait.php | 90 ++++++++++++++++--- .../src/Lib/Util/RawSqlQueries.php | 48 ++++++++++ 4 files changed, 166 insertions(+), 45 deletions(-) diff --git a/app/plugins/Transmogrify/config/schema/tables.json b/app/plugins/Transmogrify/config/schema/tables.json index d35bbfac6..803002e45 100644 --- a/app/plugins/Transmogrify/config/schema/tables.json +++ b/app/plugins/Transmogrify/config/schema/tables.json @@ -188,6 +188,7 @@ "source": "cm_names", "displayField": "id", "booleans": ["primary_name"], + "sqlSelect": "mveaSqlSelect", "fieldMap": { "co_person_id": "person_id", "org_identity_id": "external_identity_id", @@ -195,33 +196,11 @@ "type": null } }, - "ad_hoc_attributes": { - "source": "cm_ad_hoc_attributes", - "displayField": "id", - "fieldMap": { - "co_person_role_id": "person_role_id", - "org_identity_id": "external_identity_id", - "co_department_id": null, - "organization_id": null - }, - "postTable": "migrateExtendedAttributesToAdHocAttributes" - }, - "addresses": { - "source": "cm_addresses", - "displayField": "id", - "fieldMap": { - "co_person_role_id": "person_role_id", - "org_identity_id": "external_identity_id", - "type_id": "&mapAddressType", - "type": null, - "co_department_id": null, - "organization_id": null - } - }, "email_addresses": { "source": "cm_email_addresses", "displayField": "id", "booleans": ["verified"], + "sqlSelect": "mveaSqlSelect", "fieldMap": { "co_person_id": "person_id", "org_identity_id": "external_identity_id", @@ -235,6 +214,7 @@ "source": "cm_identifiers", "displayField": "id", "booleans": ["login"], + "sqlSelect": "mveaSqlSelect", "fieldMap": { "co_group_id": "group_id", "co_person_id": "person_id", @@ -247,25 +227,52 @@ }, "preRow": "mapLoginIdentifiers" }, - "telephone_numbers": { - "source": "cm_telephone_numbers", + "urls": { + "source": "cm_urls", + "displayField": "id", + "sqlSelect": "mveaSqlSelect", + "fieldMap": { + "co_person_id": "person_id", + "org_identity_id": "external_identity_id", + "type_id": "&mapUrlType", + "type": null, + "co_department_id": null, + "organization_id": null + } + }, + "ad_hoc_attributes": { + "source": "cm_ad_hoc_attributes", "displayField": "id", + "sqlSelect": "mveaSqlSelect", "fieldMap": { "co_person_role_id": "person_role_id", "org_identity_id": "external_identity_id", - "type_id": "&mapTelephoneType", + "co_department_id": null, + "organization_id": null + }, + "postTable": "migrateExtendedAttributesToAdHocAttributes" + }, + "addresses": { + "source": "cm_addresses", + "displayField": "id", + "sqlSelect": "mveaSqlSelect", + "fieldMap": { + "co_person_role_id": "person_role_id", + "org_identity_id": "external_identity_id", + "type_id": "&mapAddressType", "type": null, "co_department_id": null, "organization_id": null } }, - "urls": { - "source": "cm_urls", + "telephone_numbers": { + "source": "cm_telephone_numbers", "displayField": "id", + "sqlSelect": "mveaSqlSelect", "fieldMap": { - "co_person_id": "person_id", + "co_person_role_id": "person_role_id", "org_identity_id": "external_identity_id", - "type_id": "&mapUrlType", + "type_id": "&mapTelephoneType", "type": null, "co_department_id": null, "organization_id": null diff --git a/app/plugins/Transmogrify/src/Lib/Traits/CacheTrait.php b/app/plugins/Transmogrify/src/Lib/Traits/CacheTrait.php index 84f3beebe..309649b18 100644 --- a/app/plugins/Transmogrify/src/Lib/Traits/CacheTrait.php +++ b/app/plugins/Transmogrify/src/Lib/Traits/CacheTrait.php @@ -116,7 +116,11 @@ protected function findCoId(array $row): int return $coId; } - throw new \InvalidArgumentException('CO not found for record'); + // For the multiple value attributes we are going to get a lot of misses + // because we only move one org identity and only the latest one. No revisions. + // Which means that all the values that belong to deleted org identities are going + // to be misses. + throw new \InvalidArgumentException("CO not found for record"); } /** diff --git a/app/plugins/Transmogrify/src/Lib/Traits/TypeMapperTrait.php b/app/plugins/Transmogrify/src/Lib/Traits/TypeMapperTrait.php index ee621f9c4..2b90c7c66 100644 --- a/app/plugins/Transmogrify/src/Lib/Traits/TypeMapperTrait.php +++ b/app/plugins/Transmogrify/src/Lib/Traits/TypeMapperTrait.php @@ -30,6 +30,7 @@ namespace Transmogrify\Lib\Traits; use Doctrine\DBAL\Exception; +use Doctrine\DBAL\Exception\UniqueConstraintViolationException; use Transmogrify\Lib\Util\RawSqlQueries; use Cake\Utility\Inflector; use Transmogrify\Lib\Traits\CacheTrait; @@ -71,10 +72,10 @@ trait TypeMapperTrait * Map address type to corresponding type ID * * @param array $row Row data containing address type - * @return int Mapped type ID + * @return int|null Mapped type ID * @since COmanage Registry v5.0.0 */ - protected function mapAddressType(array $row): int + protected function mapAddressType(array $row): ?int { return $this->mapType($row, 'Addresses.type', $this->findCoId($row)); } @@ -101,10 +102,10 @@ protected function mapAffiliationType(array $row, ?int $coId = null): ?int * Map email type to corresponding type ID * * @param array $row Row data containing email type - * @return int Mapped type ID + * @return int|null Mapped type ID * @since COmanage Registry v5.0.0 */ - protected function mapEmailType(array $row): int + protected function mapEmailType(array $row): ?int { return $this->mapType($row, 'EmailAddresses.type', $this->findCoId($row)); } @@ -136,22 +137,84 @@ protected function mapExtendedType(array $row): string * Map identifier type to corresponding type ID * * @param array $row Row data containing identifier type - * @return int Mapped type ID + * @return int|null Mapped type ID * @since COmanage Registry v5.0.0 */ - protected function mapIdentifierType(array $row): int + protected function mapIdentifierType(array $row): ?int { return $this->mapType($row, 'Identifiers.type', $this->findCoId($row)); } + /** + * Map login identifiers, in accordance with the configuration. + * + * @since COmanage Registry v5.0.0 + * @param array $origRow Row of table data (original data) + * @param array $row Row of table data (post fixes) + * @throws \InvalidArgumentException + */ + protected function mapLoginIdentifiers(array $origRow, array &$row): void { + // There might be multiple reasons to copy the row, but we only want to + // copy it once. + $copyRow = false; + + if(!empty($origRow['org_identity_id'])) { + if($this->args->getOption('login-identifier-copy') + && $origRow['login']) { + $copyRow = true; + } + + // Note the argument here is the old v4 string (eg "eppn") and not the + // PE foreign key + if($this->args->getOption('login-identifier-type') + && $origRow['type'] == $this->args->getOption('login-identifier-type')) { + $copyRow = true; + } + + // Identifiers attached to External Identities do not have login flags in PE + $row['login'] = false; + } + + if($copyRow) { + // Find the Person ID associated with this External Identity ID + + if(!empty($this->cache['external_identities']['id'][ $origRow['org_identity_id'] ]['person_id'])) { + // Insert a new row attached to the Person, leave the original record + // (ie: $row) untouched + + $copiedRow = [ + 'person_id' => $this->map_org_identity_co_person_id(['id' => $origRow['org_identity_id']]), + 'identifier' => $origRow['identifier'], + 'type_id' => $this->map_identifier_type($origRow), + 'status' => $origRow['status'], + 'login' => true, + 'created' => $origRow['created'], + 'modified' => $origRow['modified'] + ]; + + // Set up changelog and fix booleans + $this->fixChangelog('identifiers', $copiedRow, true); + $this->fixBooleans('identifiers', $copiedRow); + + try { + $tableName = 'identifiers'; + $qualifiedTableName = $this->outconn->qualifyTableName($tableName); + $this->outconn->insert($qualifiedTableName, $copiedRow); + } catch (UniqueConstraintViolationException $e) { + $this->io->warning("record already exists: " . print_r($copiedRow, true)); + } + } + } + } + /** * Map name type to corresponding type ID * * @param array $row Row data containing name type - * @return int Mapped type ID + * @return int|null Mapped type ID * @since COmanage Registry v5.0.0 */ - protected function mapNameType(array $row): int + protected function mapNameType(array $row): ?int { return $this->mapType($row, 'Names.type', $this->findCoId($row)); } @@ -207,7 +270,7 @@ protected function mapOrgIdentitycoPersonId(array $row): ?int $this->cache['org_identities']['co_people'] = []; // Build cache on first use - $this->io->info('Populating org identity map...'); + $this->io->verbose('Populating org identity map...'); $tableName = 'cm_co_org_identity_links'; $changelogFK = 'co_org_identity_link_id'; @@ -249,7 +312,6 @@ protected function mapOrgIdentitycoPersonId(array $row): ?int } if (!empty($this->cache['org_identities']['co_people'][$rowId])) { - // XXX OrgIdentities with no org identity link are not supported in v5 // Return the record with the highest revision number $revisions = $this->cache['org_identities']['co_people'][$rowId]; $rev = max(array_keys($revisions)); @@ -277,10 +339,10 @@ protected function mapServerTypeToPlugin(array $row): ?string * Map telephone type to corresponding type ID * * @param array $row Row data containing telephone type - * @return int Mapped type ID + * @return int|null Mapped type ID * @since COmanage Registry v5.0.0 */ - protected function mapTelephoneType(array $row): int + protected function mapTelephoneType(array $row): ?int { return $this->mapType($row, 'TelephoneNumbers.type', $this->findCoId($row)); } @@ -323,10 +385,10 @@ protected function mapType(array $row, string $type, int $coId, string $attr = ' * Map URL type to corresponding type ID * * @param array $row Row data containing URL type - * @return int Mapped type ID + * @return int|null Mapped type ID * @since COmanage Registry v5.0.0 */ - protected function mapUrlType(array $row): int + protected function mapUrlType(array $row): ?int { return $this->mapType($row, 'Urls.type', $this->findCoId($row)); } diff --git a/app/plugins/Transmogrify/src/Lib/Util/RawSqlQueries.php b/app/plugins/Transmogrify/src/Lib/Util/RawSqlQueries.php index 28b7ecf2c..2ce1538d8 100644 --- a/app/plugins/Transmogrify/src/Lib/Util/RawSqlQueries.php +++ b/app/plugins/Transmogrify/src/Lib/Util/RawSqlQueries.php @@ -192,6 +192,19 @@ public static function roleSqlSelect(string $tableName, bool $isMySQL): string { return RawSqlQueries::ROLE_SQL_SELECT; } + + /** + * Builds SQL query to select Multiple Value Element Attributes (MVEAs) for valid org identities + * + * @param string $tableName Name of the database table containing MVEAs + * @param bool $isMySQL Whether the target database is MySQL (true) or PostgreSQL (false) + * @return string SQL query string to select MVEA rows that are linked to valid org identities + * @since COmanage Registry v5.2.0 + */ + public static function mveaSqlSelect(string $tableName, bool $isMySQL): string { + return str_replace('{table}', $tableName, RawSqlQueries::MVEA_SQL_SELECT); + } + /** * Return SQL used to select Organization Identities from inbound database. * @@ -632,4 +645,39 @@ public static function orgidentitiesSqlSelect(string $tableName, bool $isMySQL): FROM cm_co_groups ORDER BY reason; SQL; + + + /** + * SQL template for selecting Multiple Value Element Attributes (MVEAs) that are linked to valid org identities + * Filters records to only include those with: + * - Valid person or org identity references + * - Org identities that have valid non-historical links to people + * + * @since COmanage Registry v5.2.0 + */ + final const MVEA_SQL_SELECT = << Date: Tue, 14 Oct 2025 19:24:37 +0300 Subject: [PATCH 09/15] Fix mveas --- .../Transmogrify/config/schema/tables.json | 64 +++++++------ .../src/Command/TransmogrifyCommand.php | 94 +++++++++++-------- .../src/Lib/Traits/CacheTrait.php | 25 ++++- .../src/Lib/Traits/HookRunnersTrait.php | 28 ++++++ .../src/Lib/Util/RawSqlQueries.php | 87 ++++++++++++++++- 5 files changed, 225 insertions(+), 73 deletions(-) diff --git a/app/plugins/Transmogrify/config/schema/tables.json b/app/plugins/Transmogrify/config/schema/tables.json index 803002e45..ce2cc0cab 100644 --- a/app/plugins/Transmogrify/config/schema/tables.json +++ b/app/plugins/Transmogrify/config/schema/tables.json @@ -112,7 +112,7 @@ "source": "cm_co_person_roles", "sqlSelect": "roleSqlSelect", "displayField": "id", - "cache": ["status"], + "cache": ["status", "person_id", "manager_person_id", "sponsor_person_id"], "fieldMap": { "co_person_id": "person_id", "co_person_role_id": "person_role_id", @@ -173,6 +173,7 @@ "source": "cm_co_group_members", "displayField": "id", "booleans": ["member", "owner"], + "preRow": "reconcileGroupMembershipOwnership", "fieldMap": { "co_group_id": "group_id", "co_person_id": "person_id", @@ -181,8 +182,7 @@ "co_group_nesting_id": "group_nesting_id", "co_group_member_id": "group_member_id", "source_org_identity_id": null - }, - "preRow": "reconcileGroupMembershipOwnership" + } }, "names": { "source": "cm_names", @@ -190,9 +190,9 @@ "booleans": ["primary_name"], "sqlSelect": "mveaSqlSelect", "fieldMap": { + "type_id": "&mapNameType", "co_person_id": "person_id", "org_identity_id": "external_identity_id", - "type_id": "&mapNameType", "type": null } }, @@ -202,12 +202,12 @@ "booleans": ["verified"], "sqlSelect": "mveaSqlSelect", "fieldMap": { + "type_id": "&mapEmailType", "co_person_id": "person_id", "org_identity_id": "external_identity_id", - "type_id": "&mapEmailType", - "type": null, "co_department_id": null, - "organization_id": null + "organization_id": null, + "type": null } }, "identifiers": { @@ -215,68 +215,70 @@ "displayField": "id", "booleans": ["login"], "sqlSelect": "mveaSqlSelect", + "preRow": "mapLoginIdentifiers", "fieldMap": { + "type_id": "&mapIdentifierType", "co_group_id": "group_id", "co_person_id": "person_id", "org_identity_id": "external_identity_id", - "type_id": "&mapIdentifierType", - "type": null, "co_department_id": null, "co_provisioning_target_id": null, - "organization_id": null - }, - "preRow": "mapLoginIdentifiers" + "organization_id": null, + "type": null, + "language": null + } }, "urls": { "source": "cm_urls", "displayField": "id", "sqlSelect": "mveaSqlSelect", "fieldMap": { + "type_id": "&mapUrlType", "co_person_id": "person_id", "org_identity_id": "external_identity_id", - "type_id": "&mapUrlType", - "type": null, "co_department_id": null, - "organization_id": null + "organization_id": null, + "type": null, + "language": null } }, - "ad_hoc_attributes": { - "source": "cm_ad_hoc_attributes", + "addresses": { + "source": "cm_addresses", "displayField": "id", "sqlSelect": "mveaSqlSelect", "fieldMap": { + "type_id": "&mapAddressType", "co_person_role_id": "person_role_id", "org_identity_id": "external_identity_id", "co_department_id": null, - "organization_id": null - }, - "postTable": "migrateExtendedAttributesToAdHocAttributes" + "organization_id": null, + "type": null + } }, - "addresses": { - "source": "cm_addresses", + "telephone_numbers": { + "source": "cm_telephone_numbers", "displayField": "id", "sqlSelect": "mveaSqlSelect", "fieldMap": { + "type_id": "&mapTelephoneType", "co_person_role_id": "person_role_id", "org_identity_id": "external_identity_id", - "type_id": "&mapAddressType", - "type": null, "co_department_id": null, - "organization_id": null + "organization_id": null, + "type": null } }, - "telephone_numbers": { - "source": "cm_telephone_numbers", + "ad_hoc_attributes": { + "source": "cm_ad_hoc_attributes", "displayField": "id", "sqlSelect": "mveaSqlSelect", "fieldMap": { "co_person_role_id": "person_role_id", "org_identity_id": "external_identity_id", - "type_id": "&mapTelephoneType", - "type": null, "co_department_id": null, "organization_id": null - } + }, + "postTable": "migrateExtendedAttributesToAdHocAttributes" }, "history_records": { "source": "cm_history_records", @@ -320,7 +322,7 @@ "source": "cm_servers", "displayField": "description", "addChangelog": false, - "cache": ["co_id"], + "cache": ["co_id", "status"], "fieldMap": { "plugin": "&mapServerTypeToPlugin" } diff --git a/app/plugins/Transmogrify/src/Command/TransmogrifyCommand.php b/app/plugins/Transmogrify/src/Command/TransmogrifyCommand.php index 57df6ae87..eb0c16ab5 100644 --- a/app/plugins/Transmogrify/src/Command/TransmogrifyCommand.php +++ b/app/plugins/Transmogrify/src/Command/TransmogrifyCommand.php @@ -267,14 +267,21 @@ public function execute(Arguments $args, ConsoleIo $io): int } foreach(array_keys($this->tables) as $t) { + // Initializations per table migration $modeltableEmpty = true; $notSelected = false; - + $inboundQualifiedTableName = $this->inconn->qualifyTableName($this->tables[$t]['source']); + $outboundQualifiedTableName = $this->outconn->qualifyTableName($t); + $Model = TableRegistry::getTableLocator()->get($t); $io->info(message: sprintf("Transmogrifying table %s(%s)", Inflector::classify($t), $t)); - // Step 1: Check if source table exists and warn if not present + /* + * Run checks before processing the table + **/ + + // Check if source table exists and warn if not present if(!empty($this->tables[$t]['source'])) { $src = $this->tables[$t]['source']; if (!$this->tableExists($src)) { @@ -282,15 +289,33 @@ public function execute(Arguments $args, ConsoleIo $io): int continue; } } - // Step 2: Skip tables not in selected subset if specified - // We will print a warning and we will parse all the tables because we need to construct the cache. + + // Skip a table if already contains data + if($Model->find()->count() > 0) { + $modeltableEmpty = false; + $io->warning("Table (" . $t . ") is not empty. We will not overwrite existing data."); + } + + // Skip tables not in the selected subset if specified + // We will print a warning, and we will parse all the tables because we need to construct the cache. // Nevertheless, we will not allow any database processing - if (!empty($selected) && !in_array($t, $selected)) { + if ( + !empty($selected) + && !in_array($t, $selected) + ) { $notSelected = true; $io->warning("Skipping Transmogrification. Table ($t) is not in the selected subset."); } - // Step 3: Configure sequence ID for the target table + // Mark the table as skipped if it is not empty and not selected + $this->cache['skipInsert'][$outboundQualifiedTableName] = (!$modeltableEmpty && $notSelected); + + /* + * End of checks + */ + + + // Configure sequence ID for the target table if(!RawSqlQueries::setSequenceId( $this->inconn, $this->outconn, @@ -302,24 +327,18 @@ public function execute(Arguments $args, ConsoleIo $io): int return BaseCommand::CODE_ERROR; } - // Step 5: Execute any pre-processing hooks for the current table + // Execute any pre-processing hooks for the current table $this->runPreTableHook($t); - // Step 6: Skip if target table already contains data - $Model = TableRegistry::getTableLocator()->get($t); - if($Model->find()->count() > 0) { - $modeltableEmpty = false; - $io->warning("Table (" . $t . ") is not empty. We will not overwrite existing data."); - } - - // Step 7: Get total count of source records for progress tracking - $qualifiedTableName = $this->inconn->qualifyTableName($this->tables[$t]['source']); - // Step 8: Build and execute query to fetch all source records $insql = match(true) { - !empty($this->tables[$t]['sqlSelect']) => $this->runSqlSelectHook($t, $qualifiedTableName), - default => RawSqlQueries::buildSelectAllOrderedById($qualifiedTableName) + !empty($this->tables[$t]['sqlSelect']) => $this->runSqlSelectHook($t, $inboundQualifiedTableName), + default => RawSqlQueries::buildSelectAllOrderedById($inboundQualifiedTableName) }; + + // Verbose message to show the SQL query being executed + $io->verbose(sprintf('[Inbound SQL] Table=%s | %s', $t, $insql)); + // Fetch the inbound data. $stmt = $this->inconn->executeQuery($insql); $progress = new CommandLinePrinter($io, 'green', 50, true); @@ -328,7 +347,7 @@ public function execute(Arguments $args, ConsoleIo $io): int $countSql = RawSqlQueries::buildCountFromSelect($insql); $count = (int)$this->inconn->fetchOne($countSql); } else { - $count = (int)$this->inconn->fetchOne(RawSqlQueries::buildCountAll($qualifiedTableName)); + $count = (int)$this->inconn->fetchOne(RawSqlQueries::buildCountAll($inboundQualifiedTableName)); } $progress->start($count); $tally = 0; @@ -341,10 +360,12 @@ public function execute(Arguments $args, ConsoleIo $io): int } try { - // Step 1: Create a copy of the original row data to preserve it for post-processing + // Create a copy of the original row data to preserve it for post-processing $origRow = $row; - // Step 2: Execute any pre-processing hooks to transform or validate the row data + // Execute any pre-processing hooks to transform or validate the row data + // TODO: if i need to skip the insert i want something shared. I can add this in the cache + // and then skip insert from everywhere. $this->runPreRowHook($t, $origRow, $row); // Step 3: Set changelog defaults (created/modified timestamps, user IDs) @@ -355,24 +376,22 @@ public function execute(Arguments $args, ConsoleIo $io): int isset($this->tables[$t]['addChangelog']) && $this->tables[$t]['addChangelog'] ); - // Step 4: Convert boolean values to database-compatible format + // Convert boolean values to database-compatible format $this->normalizeBooleanFieldsForDb($t, $row); - // Step 5: Map old field names to new schema field names + // Map old field names to new schema field names $this->mapLegacyFieldNames($t, $row); - // Step 6: Insert the transformed row into the target database - $qualifiedTableName = $this->outconn->qualifyTableName($t); - - if($modeltableEmpty && !$notSelected) { - $fkQualifiedTableName = StringUtilities::classNameToForeignKey($qualifiedTableName); + // Insert the transformed row into the target database + if($this->cache['skipInsert'][$outboundQualifiedTableName] === false) { + $fkOutboundQualifiedTableName = StringUtilities::classNameToForeignKey($outboundQualifiedTableName); if ( isset($this->cache['rejected']) - && !empty($this->cache['rejected'][$qualifiedTableName][$row[$fkQualifiedTableName]]) + && !empty($this->cache['rejected'][$outboundQualifiedTableName][$row[$fkOutboundQualifiedTableName]]) ) { // This row will be rejected because it references a parent record that does not exist. // The parent record has been rejected before, so we can't insert this record. - $this->cache['rejected'][$qualifiedTableName][$row['id']] = $row; + $this->cache['rejected'][$outboundQualifiedTableName][$row['id']] = $row; $io->warning(sprintf( 'Skipping record %d in table %s - parent record does not exist', $row['id'] ?? 0, @@ -380,12 +399,12 @@ public function execute(Arguments $args, ConsoleIo $io): int )); continue; } - $this->outconn->insert($qualifiedTableName, $row); - // Step 8: Execute any post-processing hooks after successful insertion + $this->outconn->insert($outboundQualifiedTableName, $row); + // Execute any post-processing hooks after successful insertion $this->runPostRowHook($t, $origRow, $row); } - // Step 7: Store row data in cache for potential later use + // Store row data in cache for potential later use $this->cacheResults($t, $row); } catch(ForeignKeyConstraintViolationException $e) { // A foreign key associated with this record did not load, so we can't @@ -417,12 +436,13 @@ public function execute(Arguments $args, ConsoleIo $io): int } $progress->finish(); - // Step 10: Output final warning and error counts for the table + // Output final warning and error counts for the table $io->warning(sprintf('Warnings: %d', $warns)); $io->error(sprintf('Errors: %d', $err)); - // Step 11: Execute any post-processing hooks for the table - if ($modeltableEmpty && !$notSelected) { + // Execute any post-processing hooks for the table + if($this->cache['skipInsert'][$outboundQualifiedTableName] === false) { + $io->info('Running post-table hook for ' . $t); $this->runPostTableHook($t); } diff --git a/app/plugins/Transmogrify/src/Lib/Traits/CacheTrait.php b/app/plugins/Transmogrify/src/Lib/Traits/CacheTrait.php index 309649b18..cec7f19a5 100644 --- a/app/plugins/Transmogrify/src/Lib/Traits/CacheTrait.php +++ b/app/plugins/Transmogrify/src/Lib/Traits/CacheTrait.php @@ -104,11 +104,17 @@ protected function findCoId(array $row): int isset($row['group_id']) => $this->getCoIdFromGroupId((int)$row['group_id']), - // Legacy/preRow: org_identity_id follows the same External Identity path + isset($row['person_role_id']) => $this->getCoIdFromPersonRoleId((int)$row['person_role_id']), + + // Legacy/preRow isset($row['org_identity_id']) => $this->getCoIdFromExternalIdentityId((int)$row['org_identity_id']), isset($row['co_person_id']) => $this->getCoIdFromPersonId((int)$row['co_person_id']), + isset($row['co_group_id']) => $this->getCoIdFromGroupId((int)$row['co_group_id']), + + isset($row['co_person_role_id']) => $this->getCoIdFromPersonRoleId((int)$row['co_person_role_id']), + default => null, }; @@ -174,4 +180,21 @@ private function getCoIdFromExternalIdentityId(int $externalIdentityId): ?int $personId = $this->getPersonIdFromExternalIdentity($externalIdentityId); return $personId !== null ? $this->getCoIdFromPersonId($personId) : null; } + + + /** + * Resolve a CO ID from a Person Role ID via cache. + * + * @param int $personRoleId Person Role ID to lookup + * @return int|null CO ID if found, null otherwise + * @since COmanage Registry v5.2.0 + */ + private function getCoIdFromPersonRoleId(int $personRoleId): ?int + { + if (isset($this->cache['person_roles']['id'][$personRoleId]['person_id'])) { + $peronId = (int)$this->cache['person_roles']['id'][$personRoleId]['person_id']; + return $this->getCoIdFromPersonId($peronId); + } + return null; + } } \ No newline at end of file diff --git a/app/plugins/Transmogrify/src/Lib/Traits/HookRunnersTrait.php b/app/plugins/Transmogrify/src/Lib/Traits/HookRunnersTrait.php index 757e6bc5b..48a92b9e0 100644 --- a/app/plugins/Transmogrify/src/Lib/Traits/HookRunnersTrait.php +++ b/app/plugins/Transmogrify/src/Lib/Traits/HookRunnersTrait.php @@ -124,6 +124,34 @@ private function runSqlSelectHook(string $table, string $qualifiedTableName): st throw new \RuntimeException("Unknown sqlSelect hook: $method"); } $this->io->verbose('Running SQL select hook: ' . ucfirst(preg_replace('/([A-Z])/', ' $1', $method))); + + // Special handling for MVEA-style selects: derive FK columns from fieldMap keys dynamically + if ($method === 'mveaSqlSelect') { + // Known FK candidates we care about for MVEA presence checks + $fkCandidates = [ + 'co_person_id', + 'co_person_role_id', + 'org_identity_id', + 'co_group_id', + 'co_department_id', + 'co_provisioning_target_id', + 'organization_id' + ]; + $fieldMap = $this->tables[$table]['fieldMap'] ?? []; + $presentFks = array_values(array_intersect($fkCandidates, array_keys($fieldMap))); + + // Fallback to names-like FKs if nothing matched (defensive) + if (empty($presentFks)) { + $presentFks = ['co_person_id', 'org_identity_id']; + } + + return RawSqlQueries::mveaSqlSelect( + $qualifiedTableName, + $this->inconn->isMySQL(), + $presentFks + ); + } + return RawSqlQueries::{$method}($qualifiedTableName, $this->inconn->isMySQL()); } } \ No newline at end of file diff --git a/app/plugins/Transmogrify/src/Lib/Util/RawSqlQueries.php b/app/plugins/Transmogrify/src/Lib/Util/RawSqlQueries.php index 2ce1538d8..9d9b08db6 100644 --- a/app/plugins/Transmogrify/src/Lib/Util/RawSqlQueries.php +++ b/app/plugins/Transmogrify/src/Lib/Util/RawSqlQueries.php @@ -201,8 +201,87 @@ public static function roleSqlSelect(string $tableName, bool $isMySQL): string { * @return string SQL query string to select MVEA rows that are linked to valid org identities * @since COmanage Registry v5.2.0 */ - public static function mveaSqlSelect(string $tableName, bool $isMySQL): string { - return str_replace('{table}', $tableName, RawSqlQueries::MVEA_SQL_SELECT); + public static function mveaSqlSelect(string $tableName, bool $isMySQL, array $fkColumns = []): string { + // Defaults to co_person_id and org_identity_id + if (empty($fkColumns)) { + $fkColumns = ['co_person_id', 'org_identity_id']; + } + + // XXX Unsupported FKs for now (until their models are implemented) + // co_department_id, co_provisioning_target_id, organization_id + $unsupportedFks = [ + 'co_department_id', + 'co_provisioning_target_id', + 'organization_id' + ]; + + // In full mode, treat all as supported; otherwise split into supported/unsupported + if (false /* $fullMode */) { + $supportedInUse = array_values($fkColumns); + $unsupportedInUse = []; + } else { + // Split provided FKs into supported (for OR non-null) and unsupported (must be NULL) + $supportedInUse = array_values(array_diff($fkColumns, $unsupportedFks)); + $unsupportedInUse = array_values(array_intersect($fkColumns, $unsupportedFks)); + } + + // Require at least one SUPPORTED FK is not NULL (unsupported FKs are excluded from this OR) + $nonnullClauses = array_map( + fn(string $c) => 'n.' . $c . ' IS NOT NULL', + $supportedInUse + ); + // Keep SQL valid even if no supported FKs present + $nonnullAny = empty($nonnullClauses) ? '1=1' : '(' . implode(' OR ', $nonnullClauses) . ')'; + + // Unsupported FKs (that are present) must be NULL (AND clause) + $unsupportedNullClause = ''; + if (!empty($unsupportedInUse)) { + $unsupportedNullClause = 'AND ' . implode( + ' AND ', + array_map(fn(string $c) => "n.$c IS NULL", $unsupportedInUse) + ); + } + + // If org_identity_id is one of the FKs, apply the org identity validity EXISTS + $orgIdCheck = ''; + if (in_array('org_identity_id', $fkColumns, true)) { + // If we decide to enable the soft delete we need to take into account + // the type of database + // - (p.deleted IS NULL OR p.deleted = false) on PostgreSQL + // - (p.deleted IS NULL OR p.deleted = 0) on MySQL + $orgIdCheck = << Date: Thu, 16 Oct 2025 10:31:37 +0300 Subject: [PATCH 10/15] Job history records+servers --- app/plugins/Transmogrify/README.md | 5 +- .../Transmogrify/config/schema/tables.json | 8 +- .../src/Command/TransmogrifyCommand.php | 111 +++++++++++++++--- .../src/Lib/Traits/CacheTrait.php | 12 +- .../src/Lib/Util/CommandLinePrinter.php | 49 ++++++++ .../src/Lib/Util/RawSqlQueries.php | 95 ++++++++------- 6 files changed, 215 insertions(+), 65 deletions(-) diff --git a/app/plugins/Transmogrify/README.md b/app/plugins/Transmogrify/README.md index 0826dad97..2cceaee86 100644 --- a/app/plugins/Transmogrify/README.md +++ b/app/plugins/Transmogrify/README.md @@ -13,7 +13,10 @@ It is designed to be: - Source DB (used by Transmogrify) and target DB must both be reachable. Transmogrify initializes two Doctrine DBAL connections internally. - The default tables mapping file is at: - app/plugins/Transmogrify/config/schema/tables.json - +- Actions + - Finalize any pending Jobs. Jobs in Queued or Progress state will be skipped. + - Restore extended type defaults + - Run the health checks. ## Command Invoke from your app root: diff --git a/app/plugins/Transmogrify/config/schema/tables.json b/app/plugins/Transmogrify/config/schema/tables.json index ce2cc0cab..24e2d832d 100644 --- a/app/plugins/Transmogrify/config/schema/tables.json +++ b/app/plugins/Transmogrify/config/schema/tables.json @@ -80,7 +80,8 @@ "person_picker_email_type": null, "person_picker_identifier_type": null, "person_picker_display_types": null, - "group_create_admin_only": null + "group_create_admin_only": null, + "t_and_c_return_url_allowlist": null } }, "authentication_events": { @@ -283,6 +284,7 @@ "history_records": { "source": "cm_history_records", "displayField": "id", + "sqlSelect": "historyRecordsSqlSelect", "fieldMap": { "actor_co_person_id": "actor_person_id", "co_person_id": "person_id", @@ -312,6 +314,7 @@ "job_history_records": { "source": "cm_co_job_history_records", "displayField": "id", + "sqlSelect": "jobHistoryRecordsSqlSelect", "fieldMap": { "co_job_id": "job_id", "co_person_id": "person_id", @@ -324,7 +327,8 @@ "addChangelog": false, "cache": ["co_id", "status"], "fieldMap": { - "plugin": "&mapServerTypeToPlugin" + "plugin": "&mapServerTypeToPlugin", + "server_type": null } } } diff --git a/app/plugins/Transmogrify/src/Command/TransmogrifyCommand.php b/app/plugins/Transmogrify/src/Command/TransmogrifyCommand.php index eb0c16ab5..e903e61e9 100644 --- a/app/plugins/Transmogrify/src/Command/TransmogrifyCommand.php +++ b/app/plugins/Transmogrify/src/Command/TransmogrifyCommand.php @@ -341,6 +341,9 @@ public function execute(Arguments $args, ConsoleIo $io): int // Fetch the inbound data. $stmt = $this->inconn->executeQuery($insql); + /* + * PROGRESS STARTING + **/ $progress = new CommandLinePrinter($io, 'green', 50, true); // If a custom SELECT is used, count the exact result set; otherwise count the whole table if (!empty($this->tables[$t]['sqlSelect'])) { @@ -384,21 +387,11 @@ public function execute(Arguments $args, ConsoleIo $io): int // Insert the transformed row into the target database if($this->cache['skipInsert'][$outboundQualifiedTableName] === false) { - $fkOutboundQualifiedTableName = StringUtilities::classNameToForeignKey($outboundQualifiedTableName); - if ( - isset($this->cache['rejected']) - && !empty($this->cache['rejected'][$outboundQualifiedTableName][$row[$fkOutboundQualifiedTableName]]) - ) { - // This row will be rejected because it references a parent record that does not exist. - // The parent record has been rejected before, so we can't insert this record. - $this->cache['rejected'][$outboundQualifiedTableName][$row['id']] = $row; - $io->warning(sprintf( - 'Skipping record %d in table %s - parent record does not exist', - $row['id'] ?? 0, - $t - )); + // Check if a parent record for this row was previously rejected; if so, skip this insert + if ($this->skipIfRejectedParent(currentTable: $t, row: $row, progress: $progress)) { continue; } + $this->outconn->insert($outboundQualifiedTableName, $row); // Execute any post-processing hooks after successful insertion $this->runPostRowHook($t, $origRow, $row); @@ -412,6 +405,7 @@ public function execute(Arguments $args, ConsoleIo $io): int // did not load, perhaps because it was associated with an Org Identity // not linked to a CO Person that was not migrated. $warns++; + $this->cache['rejected'][$outboundQualifiedTableName][$row['id']] = $row; $progress->warn("Skipping $t record " . $row['id'] . " due to invalid foreign key: " . $e->getMessage()); $io->ask('Press to continue...'); } @@ -419,13 +413,14 @@ public function execute(Arguments $args, ConsoleIo $io): int // If we can't find a value for mapping we skip the record // (ie: mapLegacyFieldNames basically requires a successful mapping) $warns++; + $this->cache['rejected'][$outboundQualifiedTableName][$row['id']] = $row; $progress->warn("Skipping $t record " . $row['id'] . ": " . $e->getMessage()); $io->ask('Press to continue...'); } catch(\Exception $e) { $err++; $progress->error("$t record " . $row['id'] . ": " . $e->getMessage()); - $io->ask('Press to continue...'); + $progress->ask('Press to continue...'); } $tally++; @@ -436,6 +431,10 @@ public function execute(Arguments $args, ConsoleIo $io): int } $progress->finish(); + /** + * FINISH PROGRESS + */ + // Output final warning and error counts for the table $io->warning(sprintf('Warnings: %d', $warns)); $io->error(sprintf('Errors: %d', $err)); @@ -641,4 +640,88 @@ protected function tableExists(string $tableName): bool $tableList = $dbSchemaManager->listTableNames(); return in_array($tableName, $tableList); } + + /** + * Check whether this row references a rejected record and, if so, mark it rejected and warn. + * This preserves the original self-reference check and adds a generic parent-table check. + * + * Self-reference (original semantics): + * if (cache['rejected'][qualifiedCurrentTable][row[singular(currentTable)_id]]) then skip + * + * Cross-table parent: + * find a *_id in the row that corresponds to a known target table (eg, job_id -> jobs), + * then if (cache['rejected'][qualifiedParentTable][row[parent_fk]]) skip. + * + * @param string $currentTable Logical target table name (eg, 'job_history_records') + * @param array $row Row to insert + * @param \Transmogrify\Lib\Util\CommandLinePrinter $progress Progress printer for warnings + * @return bool True if the row should be skipped, false otherwise + */ + private function skipIfRejectedParent(string $currentTable, array $row, \Transmogrify\Lib\Util\CommandLinePrinter $progress): bool + { + if (!isset($this->cache['rejected'])) { + return false; + } + + // Compute qualified table names once + $qualifiedCurrent = $this->outconn->qualifyTableName($currentTable); + + // 1) Self-reference check (preserves the original semantics) + // With the original code, fkOutboundQualifiedTableName was derived from the table, + // effectively matching "_id". + $selfFk = StringUtilities::classNameToForeignKey($currentTable); + if ( + isset($row[$selfFk]) && + !empty($this->cache['rejected'][$qualifiedCurrent][$row[$selfFk]]) + ) { + $childId = $row['id'] ?? null; + if ($childId !== null) { + $this->cache['rejected'][$qualifiedCurrent][$childId] = $row; + } + $progress->warn(sprintf( + 'Skipping record %d in table %s - parent %s(%d) was rejected (self-reference)', + (int)($childId ?? 0), + $currentTable, + $currentTable, + (int)$row[$selfFk] + )); + return true; + } + + // 2) Cross-table parents: check ALL candidate *_id columns + foreach ($row as $col => $val) { + if ($val === null) { continue; } + if (!is_string($col) || !str_ends_with($col, '_id')) { continue; } + if ($col === 'id' || str_ends_with($col, '_type_id')) { continue; } + + $base = substr($col, 0, -3); + $candidate = Inflector::pluralize(Inflector::underscore($base)); + + // Skip self-table here (already handled) + if ($candidate === $currentTable) { continue; } + + // Only check known target tables + if (!isset($this->tables[$candidate])) { continue; } + + $qualifiedParent = $this->outconn->qualifyTableName($candidate); + $parentId = $val; + + if (!empty($this->cache['rejected'][$qualifiedParent][$parentId])) { + $childId = $row['id'] ?? null; + if ($childId !== null) { + $this->cache['rejected'][$qualifiedCurrent][$childId] = $row; + } + $progress->warn(sprintf( + 'Skipping record %d in table %s - parent %s(%d) was rejected', + (int)($childId ?? 0), + $currentTable, + $candidate, + (int)$parentId + )); + return true; + } + } + + return false; + } } \ No newline at end of file diff --git a/app/plugins/Transmogrify/src/Lib/Traits/CacheTrait.php b/app/plugins/Transmogrify/src/Lib/Traits/CacheTrait.php index cec7f19a5..44ea8934e 100644 --- a/app/plugins/Transmogrify/src/Lib/Traits/CacheTrait.php +++ b/app/plugins/Transmogrify/src/Lib/Traits/CacheTrait.php @@ -104,15 +104,13 @@ protected function findCoId(array $row): int isset($row['group_id']) => $this->getCoIdFromGroupId((int)$row['group_id']), - isset($row['person_role_id']) => $this->getCoIdFromPersonRoleId((int)$row['person_role_id']), - - // Legacy/preRow + // Legacy/preRow: org_identity_id follows the same External Identity path isset($row['org_identity_id']) => $this->getCoIdFromExternalIdentityId((int)$row['org_identity_id']), isset($row['co_person_id']) => $this->getCoIdFromPersonId((int)$row['co_person_id']), isset($row['co_group_id']) => $this->getCoIdFromGroupId((int)$row['co_group_id']), - + isset($row['co_person_role_id']) => $this->getCoIdFromPersonRoleId((int)$row['co_person_role_id']), default => null, @@ -185,15 +183,15 @@ private function getCoIdFromExternalIdentityId(int $externalIdentityId): ?int /** * Resolve a CO ID from a Person Role ID via cache. * - * @param int $personRoleId Person Role ID to lookup + * @param int $personRoleId Person Role ID to resolve * @return int|null CO ID if found, null otherwise * @since COmanage Registry v5.2.0 */ private function getCoIdFromPersonRoleId(int $personRoleId): ?int { if (isset($this->cache['person_roles']['id'][$personRoleId]['person_id'])) { - $peronId = (int)$this->cache['person_roles']['id'][$personRoleId]['person_id']; - return $this->getCoIdFromPersonId($peronId); + $personId = (int)$this->cache['person_roles']['id'][$personRoleId]['person_id']; + return $this->getCoIdFromPersonId($personId); } return null; } diff --git a/app/plugins/Transmogrify/src/Lib/Util/CommandLinePrinter.php b/app/plugins/Transmogrify/src/Lib/Util/CommandLinePrinter.php index 41352174b..0ad16e8d3 100644 --- a/app/plugins/Transmogrify/src/Lib/Util/CommandLinePrinter.php +++ b/app/plugins/Transmogrify/src/Lib/Util/CommandLinePrinter.php @@ -101,6 +101,55 @@ public function message(string $message, string $level = 'info'): void $this->messageLines += substr_count($lines, "\n"); } + /** + * Prompt the user for input while keeping the bar/message layout intact. + * Returns the provided answer (or null on EOF). + */ + public function ask(string $prompt, ?string $default = null): ?string + { + // Ensure the message area exists (one line below the bar) + if ($this->messageLines === 0) { + $this->rawWrite(PHP_EOL); + } + + $answer = null; + + if ($this->io) { + // ConsoleIo handles rendering the prompt and reading input + $answer = $this->io->ask($prompt, $default); + } else { + // Fallback to STDOUT/STDIN + $this->rawWrite($prompt . ' '); + $line = fgets(STDIN); + $answer = ($line === false) ? null : rtrim($line, "\r\n"); + if ($answer === null && $default !== null) { + $answer = $default; + } + // Ensure the cursor advances to the next line after the prompt + $this->rawWrite(PHP_EOL); + } + + // A prompt line was added to the message area + $this->messageLines += 1; + + // Redraw progress bar and return cursor to the end of the message area + $this->rawWrite("\033[u"); // restore to saved bar position + $this->rawWrite("\r" . $this->formatBar($this->current)); + $this->rawWrite("\033[s"); // save bar position again + $this->rawWrite("\033[" . $this->messageLines . "B\r"); // move down to message area + + return $answer; + } + + /** + * Convenience: prompt to continue (ENTER). + */ + public function pause(string $prompt = 'Press to continue...'): void + { + $this->ask($prompt, ''); + } + + private function colorizeLevel(string $level, string $message): string { $level = strtolower($level); diff --git a/app/plugins/Transmogrify/src/Lib/Util/RawSqlQueries.php b/app/plugins/Transmogrify/src/Lib/Util/RawSqlQueries.php index 9d9b08db6..8baeb8f2d 100644 --- a/app/plugins/Transmogrify/src/Lib/Util/RawSqlQueries.php +++ b/app/plugins/Transmogrify/src/Lib/Util/RawSqlQueries.php @@ -180,6 +180,32 @@ public static function couSqlSelect(string $tableName, bool $isMySQL): string { return str_replace('{table}', $tableName, $sqlTemplate); } + /** + * Select history records that are "current" (no changelog link) and whose + * org_identity_id is either NULL or refers to an included Org Identity + * (has a current link to a non-null co_person_id). + * + * @param string $tableName + * @param bool $isMySQL Unused here; kept for consistent signature + * @return string + */ + public static function historyRecordsSqlSelect(string $tableName, bool $isMySQL): string { + return str_replace('{table}', $tableName, RawSqlQueries::HISTORY_RECORDS_SQL_SELECT); + } + + /** + * Select job history records that are "current" (no changelog link) and whose + * org_identity_id is either NULL or refers to an included Org Identity + * (has a current link to a non-null co_person_id). + * + * @param string $tableName + * @param bool $isMySQL Unused here; kept for consistent signature + * @return string + */ + public static function jobHistoryRecordsSqlSelect(string $tableName, bool $isMySQL): string { + return str_replace('{table}', $tableName, RawSqlQueries::HISTORY_RECORDS_SQL_SELECT); + } + /** * Return SQL used to select COUs from inbound database. * @@ -320,11 +346,6 @@ public static function orgidentitiesSqlSelect(string $tableName, bool $isMySQL): // // Unfortunately PostgreSQL and MySQL do not define the same aggregate functions so we need a unique // SQL template for each below. - - /** - * MySQL template for recursive CTE query to select COU records ordered by generation - * Uses GROUP_CONCAT for string aggregation - */ final const COU_SQL_SELECT_TEMPLATE_MYSQL = << Date: Thu, 16 Oct 2025 13:02:14 +0300 Subject: [PATCH 11/15] Servers transmogrify --- .../Transmogrify/config/schema/tables.json | 94 ++++++++++++++++--- .../src/Command/TransmogrifyCommand.php | 16 ++-- .../src/Lib/Traits/CacheTrait.php | 17 ++++ .../src/Lib/Traits/TypeMapperTrait.php | 38 ++++++++ 4 files changed, 143 insertions(+), 22 deletions(-) diff --git a/app/plugins/Transmogrify/config/schema/tables.json b/app/plugins/Transmogrify/config/schema/tables.json index 24e2d832d..36ecf0939 100644 --- a/app/plugins/Transmogrify/config/schema/tables.json +++ b/app/plugins/Transmogrify/config/schema/tables.json @@ -32,6 +32,7 @@ }, "__NOTES__": "Common keys: source (DB table), displayField (for printing), addChangelog (add created/modified history), booleans (type coercion), cache (key(s) to cache rows by), sqlSelect (custom SELECT provider function), pre/postTable and pre/postRow (hook function names), fieldMap (left=new table column, right=source column name or &function). Use &mapNow to inject current timestamp, &mapExtendedType to resolve cm_co_extended_types, and mapping helpers found in Transmogrify traits. Note: There is no generic config-driven enforcement for required/defaults/unique; enforce uniqueness via database constraints." }, + "__NOTES__": "CONFIGURATION MIGRATIONS", "cos": { "source": "cm_cos", "displayField": "name", @@ -84,10 +85,6 @@ "t_and_c_return_url_allowlist": null } }, - "authentication_events": { - "source": "cm_authentication_events", - "displayField": "authenticated_identifier" - }, "api_users": { "source": "cm_api_users", "displayField": "username", @@ -101,6 +98,85 @@ "displayField": "name", "sqlSelect": "couSqlSelect" }, + "servers": { + "source": "cm_servers", + "displayField": "description", + "addChangelog": false, + "cache": ["co_id", "status"], + "fieldMap": { + "plugin": "&mapServerTypeToPlugin", + "server_type": null + } + }, + "http_servers": { + "source": "cm_http_servers", + "displayField": "serverurl", + "addChangelog": false, + "booleans": [ + "ssl_verify_host", + "ssl_verify_peer" + ], + "cache": ["server_id"], + "fieldMap": { + "serverurl": "url", + "ssl_verify_peer": "skip_ssl_verification", + "ssl_verify_host": null + } + }, + "oauth2_servers": { + "source": "cm_oauth2_servers", + "displayField": "serverurl", + "addChangelog": false, + "cache": ["server_id"], + "booleans": ["access_token_exp"], + "fieldMap": { + "serverurl": "url", + "proxy": null + } + }, + "sql_servers": { + "source": "cm_sql_servers", + "displayField": "hostname", + "cache": ["server_id"], + "addChangelog": false, + "fieldMap": { + "dbport": "port" + } + }, + "match_servers": { + "source": "cm_match_servers", + "displayField": "username", + "addChangelog": false, + "booleans": [ + "ssl_verify_peer", + "ssl_verify_host" + ], + "cache": ["server_id"], + "fieldMap": { + "serverurl": "url", + "auth_type": "?BA", + "ssl_verify_peer": "skip_ssl_verification", + "ssl_verify_host": null + } + }, + "match_server_attributes": { + "source": "cm_match_server_attributes", + "displayField": "attribute", + "addChangelog": false, + "booleans": [ + "required" + ], + "cache": ["match_server_id"], + "fieldMap": { + "type_id": "&mapMatchAttributeTypeId", + "type": null + } + }, + "__NOTES__": "DATA MIGRATIONS", + "authentication_events": { + "source": "cm_authentication_events", + "displayField": "authenticated_identifier" + }, "people": { "source": "cm_co_people", "displayField": "id", @@ -320,15 +396,5 @@ "co_person_id": "person_id", "org_identity_id": "external_identity_id" } - }, - "servers": { - "source": "cm_servers", - "displayField": "description", - "addChangelog": false, - "cache": ["co_id", "status"], - "fieldMap": { - "plugin": "&mapServerTypeToPlugin", - "server_type": null - } } } diff --git a/app/plugins/Transmogrify/src/Command/TransmogrifyCommand.php b/app/plugins/Transmogrify/src/Command/TransmogrifyCommand.php index e903e61e9..f6a049923 100644 --- a/app/plugins/Transmogrify/src/Command/TransmogrifyCommand.php +++ b/app/plugins/Transmogrify/src/Command/TransmogrifyCommand.php @@ -268,8 +268,8 @@ public function execute(Arguments $args, ConsoleIo $io): int foreach(array_keys($this->tables) as $t) { // Initializations per table migration - $modeltableEmpty = true; - $notSelected = false; + $outboundTableEmpty = true; + $skipTableTransmogrification = false; $inboundQualifiedTableName = $this->inconn->qualifyTableName($this->tables[$t]['source']); $outboundQualifiedTableName = $this->outconn->qualifyTableName($t); $Model = TableRegistry::getTableLocator()->get($t); @@ -292,7 +292,7 @@ public function execute(Arguments $args, ConsoleIo $io): int // Skip a table if already contains data if($Model->find()->count() > 0) { - $modeltableEmpty = false; + $outboundTableEmpty = false; $io->warning("Table (" . $t . ") is not empty. We will not overwrite existing data."); } @@ -303,12 +303,12 @@ public function execute(Arguments $args, ConsoleIo $io): int !empty($selected) && !in_array($t, $selected) ) { - $notSelected = true; + $skipTableTransmogrification = true; $io->warning("Skipping Transmogrification. Table ($t) is not in the selected subset."); } // Mark the table as skipped if it is not empty and not selected - $this->cache['skipInsert'][$outboundQualifiedTableName] = (!$modeltableEmpty && $notSelected); + $this->cache['skipInsert'][$outboundQualifiedTableName] = !$outboundTableEmpty || $skipTableTransmogrification; /* * End of checks @@ -371,7 +371,7 @@ public function execute(Arguments $args, ConsoleIo $io): int // and then skip insert from everywhere. $this->runPreRowHook($t, $origRow, $row); - // Step 3: Set changelog defaults (created/modified timestamps, user IDs) + // Set changelog defaults (created/modified timestamps, user IDs) // Must be done before boolean normalization as it adds new fields $this->populateChangelogDefaults( $t, @@ -407,7 +407,7 @@ public function execute(Arguments $args, ConsoleIo $io): int $warns++; $this->cache['rejected'][$outboundQualifiedTableName][$row['id']] = $row; $progress->warn("Skipping $t record " . $row['id'] . " due to invalid foreign key: " . $e->getMessage()); - $io->ask('Press to continue...'); + $progress->ask('Press to continue...'); } catch(\InvalidArgumentException $e) { // If we can't find a value for mapping we skip the record @@ -415,7 +415,7 @@ public function execute(Arguments $args, ConsoleIo $io): int $warns++; $this->cache['rejected'][$outboundQualifiedTableName][$row['id']] = $row; $progress->warn("Skipping $t record " . $row['id'] . ": " . $e->getMessage()); - $io->ask('Press to continue...'); + $progress->ask('Press to continue...'); } catch(\Exception $e) { $err++; diff --git a/app/plugins/Transmogrify/src/Lib/Traits/CacheTrait.php b/app/plugins/Transmogrify/src/Lib/Traits/CacheTrait.php index 44ea8934e..c5b55c05f 100644 --- a/app/plugins/Transmogrify/src/Lib/Traits/CacheTrait.php +++ b/app/plugins/Transmogrify/src/Lib/Traits/CacheTrait.php @@ -104,6 +104,8 @@ protected function findCoId(array $row): int isset($row['group_id']) => $this->getCoIdFromGroupId((int)$row['group_id']), + isset($row['match_server_id']) => $this->getCoIdFromMatchServer((int)$row['match_server_id']), + // Legacy/preRow: org_identity_id follows the same External Identity path isset($row['org_identity_id']) => $this->getCoIdFromExternalIdentityId((int)$row['org_identity_id']), @@ -179,6 +181,21 @@ private function getCoIdFromExternalIdentityId(int $externalIdentityId): ?int return $personId !== null ? $this->getCoIdFromPersonId($personId) : null; } + /** + * Resolve a CO ID from a Match Server ID via cache. + * + * @param int $matchServerId Match Server ID to resolve + * @return int|null CO ID if found, null otherwise + * @since COmanage Registry v5.2.0 + */ + private function getCoIdFromMatchServer(int $matchServerId): ?int + { + if (isset($this->cache['match_servers']['id'][$matchServerId]['server_id'])) { + $serverId = (int)$this->cache['match_servers']['id'][$matchServerId]['server_id']; + return $this->cache['servers']['id'][$serverId]['co_id'] ?? null; + } + return null; + } /** * Resolve a CO ID from a Person Role ID via cache. diff --git a/app/plugins/Transmogrify/src/Lib/Traits/TypeMapperTrait.php b/app/plugins/Transmogrify/src/Lib/Traits/TypeMapperTrait.php index 2b90c7c66..f96dec0aa 100644 --- a/app/plugins/Transmogrify/src/Lib/Traits/TypeMapperTrait.php +++ b/app/plugins/Transmogrify/src/Lib/Traits/TypeMapperTrait.php @@ -68,6 +68,19 @@ trait TypeMapperTrait 'SQ' => 'CoreServer.SqlServers', ]; + /** + * Map match attribute types to corresponding model names + * + * @var array + * @since COmanage Registry v5.2.0 + */ + public const MATCH_ATTRIBUTE_TYPE_MAP = [ + 'emailAddress' => 'EmailAddresses', + 'name' => 'Names', + 'identifier' => 'Identifiers', + 'dateOfBirth' => '', + ]; + /** * Map address type to corresponding type ID * @@ -145,6 +158,31 @@ protected function mapIdentifierType(array $row): ?int return $this->mapType($row, 'Identifiers.type', $this->findCoId($row)); } + + /** + * Map match attribute type to corresponding type ID based on attribute key + * + * @param array $row Row data containing match attribute + * @return int|null Mapped type ID or null if attribute not found/mapped + * @since COmanage Registry v5.2.0 + */ + protected function mapMatchAttributeTypeId(array $row): ?int + { + // Determine the model based on the match attribute map (eg, emailAddress -> EmailAddresses) + $attributeKey = $row['attribute'] ?? null; + if (!$attributeKey || !array_key_exists($attributeKey, self::MATCH_ATTRIBUTE_TYPE_MAP)) { + return null; + } + + $model = self::MATCH_ATTRIBUTE_TYPE_MAP[$attributeKey]; + + // If the mapping is empty (eg, dateOfBirth), return null + if ($model === '') { + return null; + } + return $this->mapType($row, $model . '.type', $this->findCoId($row)); + } + /** * Map login identifiers, in accordance with the configuration. * From 293d09f2447196a1929efcc7a67f30e0c46abfa4 Mon Sep 17 00:00:00 2001 From: Ioannis Igoumenos Date: Fri, 17 Oct 2025 14:28:35 +0300 Subject: [PATCH 12/15] Fix command line printer and verbosity --- .../Transmogrify/config/schema/tables.json | 12 +- .../src/Command/TransmogrifyCommand.php | 66 +- .../TransmogrifySourceToTargetCommand.php | 1178 ++++++++++------- .../src/Lib/Traits/HookRunnersTrait.php | 10 +- .../src/Lib/Traits/ManageDefaultsTrait.php | 2 +- .../src/Lib/Traits/RowTransformationTrait.php | 8 +- .../src/Lib/Traits/TypeMapperTrait.php | 6 +- .../src/Lib/Util/CommandLinePrinter.php | 412 +++++- .../src/Lib/Util/RawSqlQueries.php | 6 +- 9 files changed, 1107 insertions(+), 593 deletions(-) diff --git a/app/plugins/Transmogrify/config/schema/tables.json b/app/plugins/Transmogrify/config/schema/tables.json index 36ecf0939..74a234bc0 100644 --- a/app/plugins/Transmogrify/config/schema/tables.json +++ b/app/plugins/Transmogrify/config/schema/tables.json @@ -101,7 +101,7 @@ "servers": { "source": "cm_servers", "displayField": "description", - "addChangelog": false, + "addChangelog": true, "cache": ["co_id", "status"], "fieldMap": { "plugin": "&mapServerTypeToPlugin", @@ -111,7 +111,7 @@ "http_servers": { "source": "cm_http_servers", "displayField": "serverurl", - "addChangelog": false, + "addChangelog": true, "booleans": [ "ssl_verify_host", "ssl_verify_peer" @@ -126,7 +126,7 @@ "oauth2_servers": { "source": "cm_oauth2_servers", "displayField": "serverurl", - "addChangelog": false, + "addChangelog": true, "cache": ["server_id"], "booleans": ["access_token_exp"], "fieldMap": { @@ -138,7 +138,7 @@ "source": "cm_sql_servers", "displayField": "hostname", "cache": ["server_id"], - "addChangelog": false, + "addChangelog": true, "fieldMap": { "dbport": "port" } @@ -146,7 +146,7 @@ "match_servers": { "source": "cm_match_servers", "displayField": "username", - "addChangelog": false, + "addChangelog": true, "booleans": [ "ssl_verify_peer", "ssl_verify_host" @@ -162,7 +162,7 @@ "match_server_attributes": { "source": "cm_match_server_attributes", "displayField": "attribute", - "addChangelog": false, + "addChangelog": true, "booleans": [ "required" ], diff --git a/app/plugins/Transmogrify/src/Command/TransmogrifyCommand.php b/app/plugins/Transmogrify/src/Command/TransmogrifyCommand.php index f6a049923..24a5ecaae 100644 --- a/app/plugins/Transmogrify/src/Command/TransmogrifyCommand.php +++ b/app/plugins/Transmogrify/src/Command/TransmogrifyCommand.php @@ -80,6 +80,8 @@ class TransmogrifyCommand extends BaseCommand { protected ?Arguments $args = null; protected ?ConsoleIo $io = null; + protected ?CommandLinePrinter $cmdPrinter = null; + /** @var string Absolute path to plugin root directory */ private string $pluginRoot; @@ -189,8 +191,6 @@ protected function buildOptionParser(ConsoleOptionParser $parser): ConsoleOption 'boolean' => true ]); - - $parser->setEpilog(__d('command', 'tm.epilog')); return $parser; @@ -210,6 +210,12 @@ public function execute(Arguments $args, ConsoleIo $io): int $this->args = $args; $this->io = $io; + // Info is forced to be white + $io->setStyle('info', ['text' => '0;39']); + + // Now that BaseCommand set verbosity, construct the printer so it can detect it correctly + $this->cmdPrinter = new CommandLinePrinter($io, 'green', 50, true); + // Validate "info" option combinations and handle errors $code = $this->validateInfoOptions($io); if ($code !== null) { @@ -274,7 +280,7 @@ public function execute(Arguments $args, ConsoleIo $io): int $outboundQualifiedTableName = $this->outconn->qualifyTableName($t); $Model = TableRegistry::getTableLocator()->get($t); - $io->info(message: sprintf("Transmogrifying table %s(%s)", Inflector::classify($t), $t)); + $this->cmdPrinter->info(message: sprintf("Transmogrifying table: %s(%s)", Inflector::classify($t), $t)); /* @@ -293,7 +299,7 @@ public function execute(Arguments $args, ConsoleIo $io): int // Skip a table if already contains data if($Model->find()->count() > 0) { $outboundTableEmpty = false; - $io->warning("Table (" . $t . ") is not empty. We will not overwrite existing data."); + $this->cmdPrinter->warning("Table (" . $t . ") is not empty. We will not overwrite existing data."); } // Skip tables not in the selected subset if specified @@ -304,7 +310,7 @@ public function execute(Arguments $args, ConsoleIo $io): int && !in_array($t, $selected) ) { $skipTableTransmogrification = true; - $io->warning("Skipping Transmogrification. Table ($t) is not in the selected subset."); + $this->cmdPrinter->warning("Skipping Transmogrification. Table ($t) is not in the selected subset."); } // Mark the table as skipped if it is not empty and not selected @@ -321,9 +327,9 @@ public function execute(Arguments $args, ConsoleIo $io): int $this->outconn, $this->tables[$t]['source'], $t, - $this->io + $this->cmdPrinter )) { - $io->warning("Skipping Transmogrification. Can not properly configure the Sequence for the primary key for the Table (\"$t\""); + $this->cmdPrinter->warning("Skipping Transmogrification. Can not properly configure the Sequence for the primary key for the Table (\"$t\""); return BaseCommand::CODE_ERROR; } @@ -337,14 +343,13 @@ public function execute(Arguments $args, ConsoleIo $io): int }; // Verbose message to show the SQL query being executed - $io->verbose(sprintf('[Inbound SQL] Table=%s | %s', $t, $insql)); + $this->cmdPrinter->verbose(sprintf('[Inbound SQL] Table=%s | %s', $t, $insql)); // Fetch the inbound data. $stmt = $this->inconn->executeQuery($insql); /* * PROGRESS STARTING **/ - $progress = new CommandLinePrinter($io, 'green', 50, true); // If a custom SELECT is used, count the exact result set; otherwise count the whole table if (!empty($this->tables[$t]['sqlSelect'])) { $countSql = RawSqlQueries::buildCountFromSelect($insql); @@ -352,14 +357,14 @@ public function execute(Arguments $args, ConsoleIo $io): int } else { $count = (int)$this->inconn->fetchOne(RawSqlQueries::buildCountAll($inboundQualifiedTableName)); } - $progress->start($count); + $this->cmdPrinter->start($count); $tally = 0; $warns = 0; $err = 0; while ($row = $stmt->fetchAssociative()) { if(!empty($row[ $this->tables[$t]['displayField'] ])) { - $io->verbose("$t " . $row[ $this->tables[$t]['displayField'] ]); + $this->cmdPrinter->verbose("$t " . $row[ $this->tables[$t]['displayField'] ]); } try { @@ -388,7 +393,7 @@ public function execute(Arguments $args, ConsoleIo $io): int // Insert the transformed row into the target database if($this->cache['skipInsert'][$outboundQualifiedTableName] === false) { // Check if a parent record for this row was previously rejected; if so, skip this insert - if ($this->skipIfRejectedParent(currentTable: $t, row: $row, progress: $progress)) { + if ($this->skipIfRejectedParent(currentTable: $t, row: $row)) { continue; } @@ -406,42 +411,42 @@ public function execute(Arguments $args, ConsoleIo $io): int // not linked to a CO Person that was not migrated. $warns++; $this->cache['rejected'][$outboundQualifiedTableName][$row['id']] = $row; - $progress->warn("Skipping $t record " . $row['id'] . " due to invalid foreign key: " . $e->getMessage()); - $progress->ask('Press to continue...'); + $this->cmdPrinter->warn("Skipping $t record " . $row['id'] . " due to invalid foreign key: " . $e->getMessage()); +// $this->cmdPrinter->pause(); } catch(\InvalidArgumentException $e) { // If we can't find a value for mapping we skip the record // (ie: mapLegacyFieldNames basically requires a successful mapping) $warns++; $this->cache['rejected'][$outboundQualifiedTableName][$row['id']] = $row; - $progress->warn("Skipping $t record " . $row['id'] . ": " . $e->getMessage()); - $progress->ask('Press to continue...'); + $this->cmdPrinter->warn("Skipping $t record " . $row['id'] . ": " . $e->getMessage()); +// $this->cmdPrinter->pause(); } catch(\Exception $e) { $err++; - $progress->error("$t record " . $row['id'] . ": " . $e->getMessage()); - $progress->ask('Press to continue...'); + $this->cmdPrinter->error("$t record " . $row['id'] . ": " . $e->getMessage()); +// $this->cmdPrinter->pause(); } $tally++; if (!$this->args->getOption('quiet') && !$this->args->getOption('verbose')) { - $progress->update($tally); + $this->cmdPrinter->update($tally); } } - $progress->finish(); + $this->cmdPrinter->finish(); /** * FINISH PROGRESS */ // Output final warning and error counts for the table - $io->warning(sprintf('Warnings: %d', $warns)); - $io->error(sprintf('Errors: %d', $err)); + $this->cmdPrinter->warning(sprintf('Warnings: %d', $warns)); + $this->cmdPrinter->error(sprintf('Errors: %d', $err)); // Execute any post-processing hooks for the table if($this->cache['skipInsert'][$outboundQualifiedTableName] === false) { - $io->info('Running post-table hook for ' . $t); + $this->cmdPrinter->info('Running post-table hook for ' . $t); $this->runPostTableHook($t); } @@ -449,7 +454,7 @@ public function execute(Arguments $args, ConsoleIo $io): int if (!empty($pendingSelected) && isset($pendingSelected[$t])) { unset($pendingSelected[$t]); if (empty($pendingSelected)) { - $io->info('All selected tables have been processed. Exiting.'); + $this->cmdPrinter->info('All selected tables have been processed. Exiting.'); return BaseCommand::CODE_SUCCESS; } } @@ -458,12 +463,12 @@ public function execute(Arguments $args, ConsoleIo $io): int $tables = array_keys($this->tables); $currentIndex = array_search($t, $tables); if (isset($tables[$currentIndex + 1])) { - $io->info("Next table to process: " . $tables[$currentIndex + 1]); + $this->cmdPrinter->info("Next table to process: " . $tables[$currentIndex + 1]); } else { - $io->info("This is the last table to process."); + $this->cmdPrinter->info("This is the last table to process."); } - $io->ask('Press to continue...'); + $this->cmdPrinter->pause(); } return BaseCommand::CODE_SUCCESS; @@ -654,10 +659,9 @@ protected function tableExists(string $tableName): bool * * @param string $currentTable Logical target table name (eg, 'job_history_records') * @param array $row Row to insert - * @param \Transmogrify\Lib\Util\CommandLinePrinter $progress Progress printer for warnings * @return bool True if the row should be skipped, false otherwise */ - private function skipIfRejectedParent(string $currentTable, array $row, \Transmogrify\Lib\Util\CommandLinePrinter $progress): bool + private function skipIfRejectedParent(string $currentTable, array $row): bool { if (!isset($this->cache['rejected'])) { return false; @@ -678,7 +682,7 @@ private function skipIfRejectedParent(string $currentTable, array $row, \Transmo if ($childId !== null) { $this->cache['rejected'][$qualifiedCurrent][$childId] = $row; } - $progress->warn(sprintf( + $this->cmdPrinter->warn(sprintf( 'Skipping record %d in table %s - parent %s(%d) was rejected (self-reference)', (int)($childId ?? 0), $currentTable, @@ -711,7 +715,7 @@ private function skipIfRejectedParent(string $currentTable, array $row, \Transmo if ($childId !== null) { $this->cache['rejected'][$qualifiedCurrent][$childId] = $row; } - $progress->warn(sprintf( + $this->cmdPrinter->warn(sprintf( 'Skipping record %d in table %s - parent %s(%d) was rejected', (int)($childId ?? 0), $currentTable, diff --git a/app/plugins/Transmogrify/src/Command/TransmogrifySourceToTargetCommand.php b/app/plugins/Transmogrify/src/Command/TransmogrifySourceToTargetCommand.php index a074b9a58..c054687f3 100644 --- a/app/plugins/Transmogrify/src/Command/TransmogrifySourceToTargetCommand.php +++ b/app/plugins/Transmogrify/src/Command/TransmogrifySourceToTargetCommand.php @@ -40,53 +40,53 @@ public function __construct( * @return ConsoleOptionParser Configured parser */ protected function buildOptionParser(ConsoleOptionParser $parser): ConsoleOptionParser - { - $parser->setDescription('Print a JSON template shaped like __EXAMPLE_TABLE_TEMPLATE__ for mapping a source table to a target table. Target config is used only to read the example template; no values are copied. Paths must be absolute.'); - - // Switch from positional arguments to named options (long and short) - $parser->addOption('source-path', [ - 'short' => 's', - 'help' => 'Absolute path to the SOURCE configuration file (JSON or XML).', - ]); - $parser->addOption('target-path', [ - 'short' => 't', - 'help' => 'Absolute path to the TARGET configuration file (JSON or XML).', - ]); - $parser->addOption('source-table', [ - 'short' => 'S', - 'help' => 'Table key/name in the source configuration to map from.', - ]); - $parser->addOption('target-table', [ - 'short' => 'T', - 'help' => 'Table key/name in the target configuration to map to.', - ]); - $parser->addOption('source-prefix', [ - 'short' => 'p', - 'help' => 'Prefix to prepend to the source table in the output (eg, "cm_"). Default: empty string.', - ]); - - // Association tree printer option - $parser->addOption('assoc-tree', [ - 'short' => 'A', - 'help' => 'Print an ASCII tree of associations starting from the given --source-table based on the source XML. Use with -s and -S. Ignores other options and exits.', - 'boolean' => true, - ]); - - $epilog = []; - $epilog[] = 'Examples:'; - $epilog[] = ' bin/cake transmogrify_source_to_target \\\n --source-path /path/to/registry/app/Config/Schema/schema.xml \\\n --target-path /path/to/registry/app/config/transmogrifytables.json \\\n --source-table cm_co_people \\\n --target-table people'; - $epilog[] = ''; - $epilog[] = ' bin/cake transmogrify_source_to_target -s /abs/source.xml -t /abs/target.json -S cousins -T cous -p cm_'; - $epilog[] = ''; - $epilog[] = 'Notes:'; - $epilog[] = ' - Paths must be absolute.'; - $epilog[] = ' - Files may be JSON (.json) or XML (.xml).'; - $epilog[] = ' - The output is a single JSON object similar to __EXAMPLE_TABLE_TEMPLATE__ in tables.json.'; - $epilog[] = ' - Use --source-prefix/-p to control any prefix (like cm_) applied to the source table name in the output.'; - $parser->setEpilog(implode(PHP_EOL, $epilog)); - - return parent::buildOptionParser($parser); - } + { + $parser->setDescription('Print a JSON template shaped like __EXAMPLE_TABLE_TEMPLATE__ for mapping a source table to a target table. Target config is used only to read the example template; no values are copied. Paths must be absolute.'); + + // Switch from positional arguments to named options (long and short) + $parser->addOption('source-path', [ + 'short' => 's', + 'help' => 'Absolute path to the SOURCE configuration file (JSON or XML).', + ]); + $parser->addOption('target-path', [ + 'short' => 't', + 'help' => 'Absolute path to the TARGET configuration file (JSON or XML).', + ]); + $parser->addOption('source-table', [ + 'short' => 'S', + 'help' => 'Table key/name in the source configuration to map from.', + ]); + $parser->addOption('target-table', [ + 'short' => 'T', + 'help' => 'Table key/name in the target configuration to map to.', + ]); + $parser->addOption('source-prefix', [ + 'short' => 'p', + 'help' => 'Prefix to prepend to the source table in the output (eg, "cm_"). Default: empty string.', + ]); + + // Association tree printer option + $parser->addOption('assoc-tree', [ + 'short' => 'A', + 'help' => 'Print an ASCII tree of associations starting from the given --source-table based on the source XML. Use with -s and -S. Ignores other options and exits.', + 'boolean' => true, + ]); + + $epilog = []; + $epilog[] = 'Examples:'; + $epilog[] = ' bin/cake transmogrify_source_to_target \\\n --source-path /path/to/registry/app/Config/Schema/schema.xml \\\n --target-path /path/to/registry/app/config/transmogrifytables.json \\\n --source-table cm_co_people \\\n --target-table people'; + $epilog[] = ''; + $epilog[] = ' bin/cake transmogrify_source_to_target -s /abs/source.xml -t /abs/target.json -S cousins -T cous -p cm_'; + $epilog[] = ''; + $epilog[] = 'Notes:'; + $epilog[] = ' - Paths must be absolute.'; + $epilog[] = ' - Files may be JSON (.json) or XML (.xml).'; + $epilog[] = ' - The output is a single JSON object similar to __EXAMPLE_TABLE_TEMPLATE__ in tables.json.'; + $epilog[] = ' - Use --source-prefix/-p to control any prefix (like cm_) applied to the source table name in the output.'; + $parser->setEpilog(implode(PHP_EOL, $epilog)); + + return parent::buildOptionParser($parser); + } /** * Execute the command @@ -96,205 +96,324 @@ protected function buildOptionParser(ConsoleOptionParser $parser): ConsoleOption * @return int Exit code */ public function execute(Arguments $args, ConsoleIo $io): int - { - // capture prefix for helper methods - $this->currentPrefix = (string)($args->getOption('source-prefix') ?? ''); - $sourcePath = (string)($args->getOption('source-path') ?? ''); - $targetPath = (string)($args->getOption('target-path') ?? ''); - $sourceTable = (string)($args->getOption('source-table') ?? ''); - $targetTable = (string)($args->getOption('target-table') ?? ''); - $sourcePrefix = (string)($args->getOption('source-prefix') ?? ''); - $assocTree = (bool)($args->getOption('assoc-tree') ?? false); - - // If only association tree requested, we only need source path and source table - if ($assocTree) { - $missing = []; - if ($sourcePath === '') { $missing[] = '--source-path (-s)'; } - if ($sourceTable === '') { $missing[] = '--source-table (-S)'; } - if (!empty($missing)) { - $io->err('Missing required option(s) for --assoc-tree: ' . implode(', ', $missing)); - $io->err('Run with --help to see usage.'); - return BaseCommand::CODE_ERROR; - } - if ($sourcePath === '' || !str_starts_with($sourcePath, DIRECTORY_SEPARATOR)) { - $io->err('SOURCE path must be absolute: ' . $sourcePath); - return BaseCommand::CODE_ERROR; - } - if (!is_readable($sourcePath)) { - $io->err('SOURCE file not readable: ' . $sourcePath); - return BaseCommand::CODE_ERROR; - } + { + // capture prefix for helper methods + $this->currentPrefix = (string)($args->getOption('source-prefix') ?? ''); + $sourcePath = (string)($args->getOption('source-path') ?? ''); + $targetPath = (string)($args->getOption('target-path') ?? ''); + $sourceTable = (string)($args->getOption('source-table') ?? ''); + $targetTable = (string)($args->getOption('target-table') ?? ''); + $sourcePrefix = (string)($args->getOption('source-prefix') ?? ''); + $assocTree = (bool)($args->getOption('assoc-tree') ?? false); + + // If only association tree requested, we only need source path and source table + if ($assocTree) { + $missing = []; + if ($sourcePath === '') { $missing[] = '--source-path (-s)'; } + if ($sourceTable === '') { $missing[] = '--source-table (-S)'; } + if (!empty($missing)) { + $io->err('Missing required option(s) for --assoc-tree: ' . implode(', ', $missing)); + $io->err('Run with --help to see usage.'); + return BaseCommand::CODE_ERROR; + } + if ($sourcePath === '' || !str_starts_with($sourcePath, DIRECTORY_SEPARATOR)) { + $io->err('SOURCE path must be absolute: ' . $sourcePath); + return BaseCommand::CODE_ERROR; + } + if (!is_readable($sourcePath)) { + $io->err('SOURCE file not readable: ' . $sourcePath); + return BaseCommand::CODE_ERROR; + } + + try { + $srcCfgRaw = $this->configLoaderService->loadGeneric($sourcePath); + } catch (\Throwable $e) { + $io->err('Failed to load source configuration: ' . $e->getMessage()); + return BaseCommand::CODE_ERROR; + } + $this->printAssociationTreeFromSchema($srcCfgRaw, $sourceTable, $io); + return BaseCommand::CODE_SUCCESS; + } - try { - $srcCfgRaw = $this->configLoaderService->loadGeneric($sourcePath); - } catch (\Throwable $e) { - $io->err('Failed to load source configuration: ' . $e->getMessage()); - return BaseCommand::CODE_ERROR; - } - $this->printAssociationTreeFromSchema($srcCfgRaw, $sourceTable, $io); - return BaseCommand::CODE_SUCCESS; - } + // Validate required named options are present + $missing = []; + if ($sourcePath === '') { $missing[] = '--source-path (-s)'; } + if ($targetPath === '') { $missing[] = '--target-path (-t)'; } + if ($sourceTable === '') { $missing[] = '--source-table (-S)'; } + if ($targetTable === '') { $missing[] = '--target-table (-T)'; } + if (!empty($missing)) { + $io->err('Missing required option(s): ' . implode(', ', $missing)); + $io->err('Run with --help to see usage.'); + return BaseCommand::CODE_ERROR; + } - // Validate required named options are present - $missing = []; - if ($sourcePath === '') { $missing[] = '--source-path (-s)'; } - if ($targetPath === '') { $missing[] = '--target-path (-t)'; } - if ($sourceTable === '') { $missing[] = '--source-table (-S)'; } - if ($targetTable === '') { $missing[] = '--target-table (-T)'; } - if (!empty($missing)) { - $io->err('Missing required option(s): ' . implode(', ', $missing)); - $io->err('Run with --help to see usage.'); - return BaseCommand::CODE_ERROR; - } + // Validate absolute paths + foreach (['source' => $sourcePath, 'target' => $targetPath] as $label => $path) { + if ($path === '' || !str_starts_with($path, DIRECTORY_SEPARATOR)) { + $io->err(strtoupper($label) . ' path must be absolute: ' . $path); + return BaseCommand::CODE_ERROR; + } + if (!is_readable($path)) { + $io->err(strtoupper($label) . ' file not readable: ' . $path); + return BaseCommand::CODE_ERROR; + } + } - // Validate absolute paths - foreach (['source' => $sourcePath, 'target' => $targetPath] as $label => $path) { - if ($path === '' || !str_starts_with($path, DIRECTORY_SEPARATOR)) { - $io->err(strtoupper($label) . ' path must be absolute: ' . $path); - return BaseCommand::CODE_ERROR; - } - if (!is_readable($path)) { - $io->err(strtoupper($label) . ' file not readable: ' . $path); - return BaseCommand::CODE_ERROR; - } - } + try { + // Load target config to read __EXAMPLE_TABLE_TEMPLATE__ shape + $tgtCfg = $this->configLoaderService->loadGeneric($targetPath); + // Load source config to extract old schema (eg, from XML) + $srcCfgRaw = $this->configLoaderService->loadGeneric($sourcePath); + } catch (\Throwable $e) { + $io->err('Failed to load configuration: ' . $e->getMessage()); + return BaseCommand::CODE_ERROR; + } - try { - // Load target config to read __EXAMPLE_TABLE_TEMPLATE__ shape - $tgtCfg = $this->configLoaderService->loadGeneric($targetPath); - // Load source config to extract old schema (eg, from XML) - $srcCfgRaw = $this->configLoaderService->loadGeneric($sourcePath); - } catch (\Throwable $e) { - $io->err('Failed to load configuration: ' . $e->getMessage()); - return BaseCommand::CODE_ERROR; - } + // Load the migration config (tables.json) to derive generic co_* FK renames + $tablesJsonPath = dirname(__DIR__, 2) . DIRECTORY_SEPARATOR . 'config' . DIRECTORY_SEPARATOR . 'schema' . DIRECTORY_SEPARATOR . 'tables.json'; + $tablesCfg = []; + if (is_readable($tablesJsonPath)) { + try { + $tablesCfg = $this->configLoaderService->load($tablesJsonPath); + } catch (\Throwable $e) { + $io->warning('Could not load migration config tables.json: ' . $e->getMessage()); + } + } - // Normalize possible schema.xml structures into transmogrify-like configs for source - $srcCfg = $this->normalizeConfig(is_array($srcCfgRaw) ? $srcCfgRaw : []); - - // Extract example template (shape) from target config without filtering out __ keys first - $example = is_array($tgtCfg) && array_key_exists('__EXAMPLE_TABLE_TEMPLATE__', $tgtCfg) - ? (array)$tgtCfg['__EXAMPLE_TABLE_TEMPLATE__'] - : []; - - // Determine the output keys based on the example (ignoring any __* documentation keys) - $exampleKeys = array_filter(array_keys($example), static function($k) { - return !is_string($k) || !str_starts_with($k, '__'); - }); - - // Determine source fields from old schema configuration (if available) - $sourceFields = []; - [$srcKey, $srcTableCfg] = $this->findTableConfig($srcCfg, $sourceTable); - if (is_array($srcTableCfg)) { - if (!empty($srcTableCfg['fieldMap']) && is_array($srcTableCfg['fieldMap'])) { - $sourceFields = array_keys($srcTableCfg['fieldMap']); - } elseif (!empty($srcTableCfg['fields']) && is_array($srcTableCfg['fields'])) { - $sourceFields = array_values($srcTableCfg['fields']); - } + // Normalize possible schema.xml structures into transmogrify-like configs for source + $srcCfg = $this->normalizeConfig(is_array($srcCfgRaw) ? $srcCfgRaw : []); + + // Extract example template (shape) from target config without filtering out __ keys first + $example = is_array($tgtCfg) && array_key_exists('__EXAMPLE_TABLE_TEMPLATE__', $tgtCfg) + ? (array)$tgtCfg['__EXAMPLE_TABLE_TEMPLATE__'] + : []; + + // Determine the output keys based on the example (ignoring any __* documentation keys) + $exampleKeys = array_filter(array_keys($example), static function($k) { + return !is_string($k) || !str_starts_with($k, '__'); + }); + + // Determine source fields from old schema configuration (if available) + $sourceFields = []; + [$srcKey, $srcTableCfg] = $this->findTableConfig($srcCfg, $sourceTable); + if (is_array($srcTableCfg)) { + if (!empty($srcTableCfg['fieldMap']) && is_array($srcTableCfg['fieldMap'])) { + $sourceFields = array_keys($srcTableCfg['fieldMap']); + } elseif (!empty($srcTableCfg['fields']) && is_array($srcTableCfg['fields'])) { + $sourceFields = array_values($srcTableCfg['fields']); + } + } + $sourceFieldSet = array_flip($sourceFields); + + // Build legacy FK pattern map (eg, *_co_person_id -> *_person_id, *_co_message_template_id -> *_message_template_id) + $legacyFkPatterns = $this->buildLegacyFkPatterns($tablesCfg, (string)$sourcePrefix); + + // Inspect default database to find target table columns + try { + $conn = DBALConnection::factory($io); + $columns = []; + $candidate = $targetTable; + $alts = [ $candidate ]; + // toggle provided prefix to try alternative table naming + $prefix = (string)($sourcePrefix ?? ''); + if ($prefix !== '') { + $alts[] = str_starts_with($candidate, $prefix) ? substr($candidate, strlen($prefix)) : ($prefix . $candidate); + } + $foundTable = null; + foreach (array_unique($alts) as $tbl) { + if ($conn->isMySQL()) { + $db = $conn->fetchOne('SELECT DATABASE()'); + $cols = $conn->fetchAllAssociative('SELECT column_name, data_type, column_type FROM information_schema.columns WHERE table_schema = ? AND table_name = ? ORDER BY ordinal_position', [$db, $tbl]); + if (!empty($cols)) { $columns = $cols; $foundTable = $tbl; break; } + } else { + // PostgreSQL: search in current schema(s) + $cols = $conn->fetchAllAssociative("SELECT column_name, data_type FROM information_schema.columns WHERE table_name = ? ORDER BY ordinal_position", [$tbl]); + if (!empty($cols)) { $columns = $cols; $foundTable = $tbl; break; } } - $sourceFieldSet = array_flip($sourceFields); - - // Inspect default database to find target table columns - try { - $conn = DBALConnection::factory($io); - $columns = []; - $candidate = $targetTable; - $alts = [ $candidate ]; - // toggle provided prefix to try alternative table naming - $prefix = (string)($sourcePrefix ?? ''); - if ($prefix !== '') { - $alts[] = str_starts_with($candidate, $prefix) ? substr($candidate, strlen($prefix)) : ($prefix . $candidate); - } - $foundTable = null; - foreach (array_unique($alts) as $tbl) { - if ($conn->isMySQL()) { - $db = $conn->fetchOne('SELECT DATABASE()'); - $cols = $conn->fetchAllAssociative('SELECT column_name, data_type, column_type FROM information_schema.columns WHERE table_schema = ? AND table_name = ? ORDER BY ordinal_position', [$db, $tbl]); - if (!empty($cols)) { $columns = $cols; $foundTable = $tbl; break; } - } else { - // PostgreSQL: search in current schema(s) - $cols = $conn->fetchAllAssociative("SELECT column_name, data_type FROM information_schema.columns WHERE table_name = ? ORDER BY ordinal_position", [$tbl]); - if (!empty($cols)) { $columns = $cols; $foundTable = $tbl; break; } - } - } - if ($foundTable === null) { - $io->err('Target table not found in default database: ' . $targetTable); - return BaseCommand::CODE_ERROR; - } - // Build fieldMap using DB columns and source fields set - $fieldMap = []; - $booleanCols = []; - foreach ($columns as $col) { - $colName = $col['column_name']; - $fieldMap[$colName] = array_key_exists($colName, $sourceFieldSet) ? $colName : null; - // Detect booleans by database type - if ($conn->isMySQL()) { - $dt = strtolower((string)($col['data_type'] ?? '')); - $ct = strtolower((string)($col['column_type'] ?? '')); - $isBool = ($dt === 'tinyint' && str_starts_with($ct, 'tinyint(1)')) || ($dt === 'bit' && str_starts_with($ct, 'bit(1)')) || ($dt === 'boolean'); - if ($isBool) { $booleanCols[] = $colName; } - } else { - $dt = strtolower((string)($col['data_type'] ?? '')); - if ($dt === 'boolean' || $dt === 'bool') { $booleanCols[] = $colName; } - } + } + if ($foundTable === null) { + $io->err('Target table not found in default database: ' . $targetTable); + return BaseCommand::CODE_ERROR; + } + + // Build target column name lists + $targetCols = array_map(static fn($c) => (string)$c['column_name'], $columns); + $targetColSetAll = array_flip($targetCols); + + $singular = \Cake\Utility\Inflector::singularize($targetTable); + $metaCols = [ + $singular . '_id', + 'revision', + 'deleted', + 'actor_identifier', + 'created', + 'modified', + ]; + + $targetColSet = $targetColSetAll; + foreach ($metaCols as $mc) { + unset($targetColSet[$mc]); + } + + // Determine if target natively has changelog fields (controls addChangelog) + $targetHasChangelog = ( + array_key_exists($singular . '_id', $targetColSetAll) + && array_key_exists('revision', $targetColSetAll) + && array_key_exists('deleted', $targetColSetAll) + && array_key_exists('actor_identifier', $targetColSetAll) + ); + + // Determine if source has changelog (v4 pattern: revision, deleted, actor_identifier, and legacy self-FK) + $legacySelfFkExact = 'co_' . $singular . '_id'; + $legacySelfFkSuffix = '_co_' . $singular . '_id'; + $fieldsLower = array_map('strtolower', $sourceFields); + $sourceFieldSet = array_flip($fieldsLower); + $sourceHasChangelog = ( + isset($sourceFieldSet['revision']) + && isset($sourceFieldSet['deleted']) + && isset($sourceFieldSet['actor_identifier']) + && ( + in_array($legacySelfFkExact, $fieldsLower, true) + || (bool)array_filter($fieldsLower, fn($s) => str_ends_with($s, $legacySelfFkSuffix)) + ) + ); + + // Detect booleans by database type (for non-metadata only) + $booleanCols = []; + foreach ($columns as $col) { + $colName = (string)$col['column_name']; + if (in_array($colName, $metaCols, true)) { continue; } + if ($conn->isMySQL()) { + $dt = strtolower((string)($col['data_type'] ?? '')); + $ct = strtolower((string)($col['column_type'] ?? '')); + $isBool = ($dt === 'tinyint' && str_starts_with($ct, 'tinyint(1)')) || ($dt === 'bit' && str_starts_with($ct, 'bit(1)')) || ($dt === 'boolean'); + if ($isBool) { $booleanCols[] = $colName; } + } else { + $dt = strtolower((string)($col['data_type'] ?? '')); + if ($dt === 'boolean' || $dt === 'bool') { $booleanCols[] = $colName; } + } + } + + // Build fieldMap as source-left => target-right (include null when unknown) + $fieldMap = []; + $mappedTargets = []; + $sourceMeta = ['revision','deleted','actor_identifier','created','modified']; + foreach ($sourceFields as $scol) { + if (in_array($scol, $sourceMeta, true)) { continue; } + + // 1) exact same-name target (non-metadata set) + $tcol = array_key_exists($scol, $targetColSet) ? $scol : null; + + // 2) normalization-based mapping (generic co_* fk rename via patterns + specific one-offs) + if ($tcol === null) { + $candidate = $this->normalizeTargetNameForSourceColumn($scol, $legacyFkPatterns); + if ($candidate !== null) { + $isMetaSelfFk = ($candidate === $singular . '_id'); + // accept candidate if it's a normal column, or it's the allowed self-FK + if (array_key_exists($candidate, $targetColSet) + || ($isMetaSelfFk && array_key_exists($candidate, $targetColSetAll))) { + $tcol = $candidate; } - } catch (\Throwable $e) { - $io->err('Failed to inspect database schema: ' . $e->getMessage()); - return BaseCommand::CODE_ERROR; + } } - // Build a template strictly using the example shape. The target config is only used for the example; no values are copied. - // Default values: - // - source: from the provided --source-table option - // - displayField/addChangelog/sqlSelect/pre*/post*: null - // - booleans/cache: [] - // - fieldMap: constructed from DB columns and source schema - $defaults = [ - 'source' => $sourcePrefix . $sourceTable, - 'displayField' => $this->determineDisplayField($sourceFields), - 'addChangelog' => $this->shouldAddChangelog($srcTableCfg['source'] ?? $sourceTable, $sourceFields), - 'booleans' => $booleanCols, - 'cache' => [], - 'sqlSelect' => null, - 'preTable' => null, - 'postTable' => null, - 'preRow' => null, - 'postRow' => null, - 'fieldMap' => $fieldMap ?: new \stdClass(), - ]; - - // If example keys were found, limit output to those keys (plus ensure 'source' exists). - if (!empty($exampleKeys)) { - $template = []; - foreach ($exampleKeys as $k) { - if ($k === 'source') { - $template[$k] = $sourcePrefix . $sourceTable; - continue; - } - $template[$k] = $defaults[$k] ?? null; - } - // Ensure mandatory keys exist even if not in example - foreach (['source','fieldMap'] as $must) { - if (!array_key_exists($must, $template)) { - $template[$must] = $defaults[$must]; - } - } - } else { - // Fallback to standard template keys - $template = $defaults; + $fieldMap[$scol] = $tcol; + if ($tcol !== null) { $mappedTargets[$tcol] = true; } + } + + // Collect target-only columns (user can decide later). + $targetUnmapped = []; + foreach (array_keys($targetColSetAll) as $tcol) { + // skip global metadata from the hint list + if (in_array($tcol, $metaCols, true)) { continue; } + if (!isset($mappedTargets[$tcol])) { + $targetUnmapped[] = $tcol; } + } + } catch (\Throwable $e) { + $io->err('Failed to inspect database schema: ' . $e->getMessage()); + return BaseCommand::CODE_ERROR; + } - // Ensure JSON encoding preserves empty objects properly for fieldMap - $json = json_encode($template, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES); - if ($json === false) { - $io->err('Failed to encode output JSON'); - return BaseCommand::CODE_ERROR; + + // Build a template strictly using the example shape. The target config is only used for the example; no values are copied. + $defaults = [ + 'source' => $sourcePrefix . $sourceTable, + 'displayField' => $this->determineDisplayField($sourceFields), + // Note: we will attach/omit addChangelog below based on source/target + //'addChangelog' => '??', + 'booleans' => $booleanCols, + 'cache' => [], + 'sqlSelect' => null, + 'preTable' => null, + 'postTable' => null, + 'preRow' => null, + 'postRow' => null, + 'fieldMap' => $fieldMap ?: new \stdClass(), + ]; + + // If example keys were found, limit output to those keys (plus ensure 'source' exists). + if (!empty($exampleKeys)) { + $template = []; + foreach ($exampleKeys as $k) { + if ($k === 'source') { + $template[$k] = $sourcePrefix . $sourceTable; + continue; + } + $template[$k] = $defaults[$k] ?? null; + } + foreach (['source','fieldMap'] as $must) { + if (!array_key_exists($must, $template)) { + $template[$must] = $defaults[$must]; } + } + } else { + $template = $defaults; + } + + // Decide addChangelog per rules: + // - both have changelog: omit configuration + // - target has changelog (source doesn't): true + // - target doesn't have changelog (source does): false + $addChangelogOpt = null; + if ($targetHasChangelog && $sourceHasChangelog) { + $addChangelogOpt = null; // omit + } elseif ($targetHasChangelog && !$sourceHasChangelog) { + $addChangelogOpt = true; + } elseif (!$targetHasChangelog && $sourceHasChangelog) { + $addChangelogOpt = false; + } else { + $addChangelogOpt = null; // neither has: omit + } + + // Apply addChangelog decision + if (array_key_exists('addChangelog', $template)) { + unset($template['addChangelog']); + } + if ($addChangelogOpt !== null) { + $template['addChangelog'] = $addChangelogOpt; + } + - $io->out($json); - return BaseCommand::CODE_SUCCESS; + // Attach target-only hints for the user + if (!empty($targetUnmapped)) { + $template['targetUnmapped'] = $targetUnmapped; } + // Wrap result under the target-table key for direct insertion into tables.json + $wrapped = [ $targetTable => $template ]; + + // Ensure JSON encoding preserves empty objects properly for fieldMap + $json = json_encode($wrapped, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES); + if ($json === false) { + $io->err('Failed to encode output JSON'); + return BaseCommand::CODE_ERROR; + } + + $io->out($json); + return BaseCommand::CODE_SUCCESS; + } + /** * Filter out documentation keys (starting with __) from configuration array * @@ -302,272 +421,351 @@ public function execute(Arguments $args, ConsoleIo $io): int * @return array Filtered configuration without documentation keys */ private function filterDocKeys(array $cfg): array - { - return array_filter($cfg, static function ($value, $key) { - return !(is_string($key) && str_starts_with($key, '__')); - }, ARRAY_FILTER_USE_BOTH); - } - - /** - * Decide if addChangelog should be true based on source configuration fields. - * Requirements: revision, deleted, actor_identifier, and self-referencing FK (eg, cous -> cou_id). - * - * @param string $tableName Name/key of the source table (may be prefixed like cm_) - * @param array $sourceFields List of source column names - */ - private function shouldAddChangelog(string $tableName, array $sourceFields): bool - { - if (empty($sourceFields)) { - return false; - } - $fields = array_map('strtolower', $sourceFields); - $fieldSet = array_flip($fields); + { + return array_filter($cfg, static function ($value, $key) { + return !(is_string($key) && str_starts_with($key, '__')); + }, ARRAY_FILTER_USE_BOTH); + } - // Required fixed columns - foreach (['revision','deleted','actor_identifier'] as $req) { - if (!array_key_exists($req, $fieldSet)) { - return false; - } - } - // Determine expected self FK - $name = strtolower($tableName); - $prefix = (string)($this->currentPrefix ?? ''); - if ($prefix !== '' && str_starts_with($name, $prefix)) { - $name = substr($name, strlen($prefix)); - } - // If table name is schema-qualified (eg, public.table), take part after dot - $dot = strrpos($name, '.'); - if ($dot !== false) { - $name = substr($name, $dot + 1); - } - $base = $name; - if (str_ends_with($base, 's')) { - $base = substr($base, 0, -1); - } - $selfFk = $base . '_id'; - - return array_key_exists($selfFk, $fieldSet); - } - - /** - * Normalize a loaded configuration array. - * - If it's a schema-like array (has 'table' nodes), convert to a transmogrify-like map - * keyed by table name with at least 'source' and 'fieldMap'. - */ - private function normalizeConfig(array $cfg): array - { - // Detect schema root - $tablesNode = null; - if (array_key_exists('table', $cfg)) { - $tablesNode = $cfg['table']; - } elseif (isset($cfg['schema']) && is_array($cfg['schema']) && array_key_exists('table', $cfg['schema'])) { - $tablesNode = $cfg['schema']['table']; - } - if ($tablesNode === null) { - // Already config-like - return $cfg; - } - // Ensure list - if (!is_array($tablesNode) || (is_array($tablesNode) && array_keys($tablesNode) !== range(0, count($tablesNode)-1))) { - // Single table object -> wrap - $tables = [$tablesNode]; - } else { - $tables = $tablesNode; - } - $out = []; - foreach ($tables as $t) { - if (!is_array($t)) { continue; } - // Name is typically under '@attributes' => ['name' => ...] - $name = null; - if (isset($t['@attributes']['name'])) { - $name = (string)$t['@attributes']['name']; - } elseif (isset($t['name'])) { - // Fallback if converter placed it differently - $name = is_array($t['name']) && isset($t['name']['@attributes']) ? (string)($t['name']['@attributes']['value'] ?? '') : (string)$t['name']; - } - if ($name === null || $name === '') { continue; } - - // Build simple 1:1 field map using elements, if present - $fieldMap = new \stdClass(); - if (isset($t['field'])) { - $fieldsNode = $t['field']; - // Normalize to list - if (!is_array($fieldsNode) || (is_array($fieldsNode) && array_keys($fieldsNode) !== range(0, count($fieldsNode)-1))) { - $fields = [$fieldsNode]; - } else { - $fields = $fieldsNode; - } - $map = []; - foreach ($fields as $f) { - if (!is_array($f)) { continue; } - $fname = $f['@attributes']['name'] ?? null; - if ($fname) { - $map[$fname] = $fname; - } - } - if (!empty($map)) { - $fieldMap = $map; - } - } + /** + * Decide if addChangelog should be true based on source configuration fields. + * Requirements: revision, deleted, actor_identifier, and self-referencing FK (eg, cous -> cou_id). + * + * @param string $tableName Name/key of the source table (may be prefixed like cm_) + * @param array $sourceFields List of source column names + */ + private function shouldAddChangelog(string $tableName, array $sourceFields): bool + { + if (empty($sourceFields)) { + return false; + } + $fields = array_map('strtolower', $sourceFields); + $fieldSet = array_flip($fields); + + // Required fixed columns + foreach (['revision','deleted','actor_identifier'] as $req) { + if (!array_key_exists($req, $fieldSet)) { + return false; + } + } + // Determine expected self FK + $name = strtolower($tableName); + $prefix = (string)($this->currentPrefix ?? ''); + if ($prefix !== '' && str_starts_with($name, $prefix)) { + $name = substr($name, strlen($prefix)); + } + // If table name is schema-qualified (eg, public.table), take part after dot + $dot = strrpos($name, '.'); + if ($dot !== false) { + $name = substr($name, $dot + 1); + } + $base = $name; + if (str_ends_with($base, 's')) { + $base = substr($base, 0, -1); + } + $selfFk = $base . '_id'; - $out[$name] = [ - 'source' => $name, - 'displayField' => $this->determineDisplayField(is_array($fieldMap) ? array_keys($fieldMap) : []), - 'fieldMap' => $fieldMap - ]; - } - return $out; - } - - /** - * Locate a table configuration by key or by matching the 'source' attribute. - * Also tries matching names with/without a leading provided prefix. - * - * @param array $cfg - * @param string $nameOrSource Either the config key or the source table name - * @return array{0:string|null,1:array|null} [key, config] - */ - private function findTableConfig(array $cfg, string $nameOrSource): array - { - if (array_key_exists($nameOrSource, $cfg) && is_array($cfg[$nameOrSource])) { - return [$nameOrSource, $cfg[$nameOrSource]]; - } - $prefix = (string)($this->currentPrefix ?? ''); - $alt = $nameOrSource; - if ($prefix !== '') { - $alt = str_starts_with($nameOrSource, $prefix) ? substr($nameOrSource, strlen($prefix)) : ($prefix . $nameOrSource); - } - if (array_key_exists($alt, $cfg) && is_array($cfg[$alt])) { - return [$alt, $cfg[$alt]]; - } - foreach ($cfg as $key => $val) { - if (!is_array($val)) { continue; } - $src = $val['source'] ?? null; - if ($src === $nameOrSource || $src === $alt) { - return [$key, $val]; - } - // Also allow key matching to bare table name - if ($key === $nameOrSource || $key === $alt) { - return [$key, $val]; - } + return array_key_exists($selfFk, $fieldSet); + } + + /** + * Normalize a loaded configuration array. + * - If it's a schema-like array (has 'table' nodes), convert to a transmogrify-like map + * keyed by table name with at least 'source' and 'fieldMap'. + */ + private function normalizeConfig(array $cfg): array + { + // Detect schema root + $tablesNode = null; + if (array_key_exists('table', $cfg)) { + $tablesNode = $cfg['table']; + } elseif (isset($cfg['schema']) && is_array($cfg['schema']) && array_key_exists('table', $cfg['schema'])) { + $tablesNode = $cfg['schema']['table']; + } + if ($tablesNode === null) { + // Already config-like + return $cfg; + } + // Ensure list + if (!is_array($tablesNode) || (is_array($tablesNode) && array_keys($tablesNode) !== range(0, count($tablesNode)-1))) { + // Single table object -> wrap + $tables = [$tablesNode]; + } else { + $tables = $tablesNode; + } + $out = []; + foreach ($tables as $t) { + if (!is_array($t)) { continue; } + // Name is typically under '@attributes' => ['name' => ...] + $name = null; + if (isset($t['@attributes']['name'])) { + $name = (string)$t['@attributes']['name']; + } elseif (isset($t['name'])) { + // Fallback if converter placed it differently + $name = is_array($t['name']) && isset($t['name']['@attributes']) ? (string)($t['name']['@attributes']['value'] ?? '') : (string)$t['name']; + } + if ($name === null || $name === '') { continue; } + + // Build simple 1:1 field map using elements, if present + $fieldMap = new \stdClass(); + if (isset($t['field'])) { + $fieldsNode = $t['field']; + // Normalize to list + if (!is_array($fieldsNode) || (is_array($fieldsNode) && array_keys($fieldsNode) !== range(0, count($fieldsNode)-1))) { + $fields = [$fieldsNode]; + } else { + $fields = $fieldsNode; } - return [null, null]; - } - - /** - * Determine displayField from source table fields using provided order: - * name, display_name, username, authenticated_identifier, description, id. - * Returns null if none are present. - */ - private function determineDisplayField(array $sourceFields): ?string - { - if (empty($sourceFields)) { - return null; + $map = []; + foreach ($fields as $f) { + if (!is_array($f)) { continue; } + $fname = $f['@attributes']['name'] ?? null; + if ($fname) { + $map[$fname] = $fname; + } } - $fields = array_map('strtolower', $sourceFields); - $set = array_flip($fields); - foreach (['name','display_name','username','authenticated_identifier','description','id'] as $candidate) { - if (array_key_exists($candidate, $set)) { - return $candidate; - } + if (!empty($map)) { + $fieldMap = $map; } - return null; - } - - /** - * Print an ASCII association tree for a given starting table using the parsed XML schema array. - * Supports common schema.xml structures with elements containing @attributes["foreignTable"] - * and nested elements. - */ - private function printAssociationTreeFromSchema(array $schemaArr, string $startTable, ConsoleIo $io): void - { - // Build associative array tree from schema.xml-like array - // We expect tables under ['schema']['table'] or directly under ['table'] - $tablesNode = $schemaArr['schema']['table'] ?? ($schemaArr['table'] ?? []); - // Normalize to list of tables - if (!is_array($tablesNode) || (is_array($tablesNode) && array_keys($tablesNode) !== range(0, count($tablesNode)-1))) { - $tables = [$tablesNode]; + } + + $out[$name] = [ + 'source' => $name, + 'displayField' => $this->determineDisplayField(is_array($fieldMap) ? array_keys($fieldMap) : []), + 'fieldMap' => $fieldMap + ]; + } + return $out; + } + + /** + * Locate a table configuration by key or by matching the 'source' attribute. + * Also tries matching names with/without a leading provided prefix. + * + * @param array $cfg + * @param string $nameOrSource Either the config key or the source table name + * @return array{0:string|null,1:array|null} [key, config] + */ + private function findTableConfig(array $cfg, string $nameOrSource): array + { + if (array_key_exists($nameOrSource, $cfg) && is_array($cfg[$nameOrSource])) { + return [$nameOrSource, $cfg[$nameOrSource]]; + } + $prefix = (string)($this->currentPrefix ?? ''); + $alt = $nameOrSource; + if ($prefix !== '') { + $alt = str_starts_with($nameOrSource, $prefix) ? substr($nameOrSource, strlen($prefix)) : ($prefix . $nameOrSource); + } + if (array_key_exists($alt, $cfg) && is_array($cfg[$alt])) { + return [$alt, $cfg[$alt]]; + } + foreach ($cfg as $key => $val) { + if (!is_array($val)) { continue; } + $src = $val['source'] ?? null; + if ($src === $nameOrSource || $src === $alt) { + return [$key, $val]; + } + // Also allow key matching to bare table name + if ($key === $nameOrSource || $key === $alt) { + return [$key, $val]; + } + } + return [null, null]; + } + + /** + * Determine displayField from source table fields using provided order: + * name, display_name, username, authenticated_identifier, description, id. + * Returns null if none are present. + */ + private function determineDisplayField(array $sourceFields): ?string + { + if (empty($sourceFields)) { + return null; + } + $fields = array_map('strtolower', $sourceFields); + $set = array_flip($fields); + foreach (['name','display_name','username','authenticated_identifier','description','id'] as $candidate) { + if (array_key_exists($candidate, $set)) { + return $candidate; + } + } + return null; + } + + /** + * Print an ASCII association tree for a given starting table using the parsed XML schema array. + * Supports common schema.xml structures with elements containing @attributes["foreignTable"] + * and nested elements. + */ + private function printAssociationTreeFromSchema(array $schemaArr, string $startTable, ConsoleIo $io): void + { + // Build associative array tree from schema.xml-like array + // We expect tables under ['schema']['table'] or directly under ['table'] + $tablesNode = $schemaArr['schema']['table'] ?? ($schemaArr['table'] ?? []); + // Normalize to list of tables + if (!is_array($tablesNode) || (is_array($tablesNode) && array_keys($tablesNode) !== range(0, count($tablesNode)-1))) { + $tables = [$tablesNode]; + } else { + $tables = $tablesNode; + } + // Build map: tableName => [ children tables ... ] based on column @attributes['constraint'] value 'REFERENCE table(column)' + $children = []; + foreach ($tables as $t) { + if (!is_array($t)) { continue; } + $tbl = $t['@attributes']['name'] ?? ($t['name'] ?? null); + if ($tbl === null) { continue; } + $tbl = (string)$tbl; + if (!isset($children[$tbl])) { $children[$tbl] = []; } + // inspect columns/field definitions; schema.xml may use or + $colsNode = $t['column'] ?? ($t['field'] ?? []); + $cols = []; + if ($colsNode !== [] && $colsNode !== null) { + if (!is_array($colsNode) || (is_array($colsNode) && array_keys($colsNode) !== range(0, count($colsNode)-1))) { + $cols = [$colsNode]; } else { - $tables = $tablesNode; - } - // Build map: tableName => [ children tables ... ] based on column @attributes['constraint'] value 'REFERENCE table(column)' - $children = []; - foreach ($tables as $t) { - if (!is_array($t)) { continue; } - $tbl = $t['@attributes']['name'] ?? ($t['name'] ?? null); - if ($tbl === null) { continue; } - $tbl = (string)$tbl; - if (!isset($children[$tbl])) { $children[$tbl] = []; } - // inspect columns/field definitions; schema.xml may use or - $colsNode = $t['column'] ?? ($t['field'] ?? []); - $cols = []; - if ($colsNode !== [] && $colsNode !== null) { - if (!is_array($colsNode) || (is_array($colsNode) && array_keys($colsNode) !== range(0, count($colsNode)-1))) { - $cols = [$colsNode]; - } else { - $cols = $colsNode; - } - } - foreach ($cols as $idx => $col) { - if (!is_array($col)) { continue; } - $constraint = $col['constraint'] ?? null; - if (!is_string($constraint)) { continue; } - // Expect format: 'REFERENCE table(column)' - if (preg_match('/^REFERENCES\s+([A-Za-z0-9_\.]+)\s*\(([^)]+)\)/', $constraint, $m)) { - $refTable = $m[1]; - // Remove prefix if it was provided and matches - if ($this->currentPrefix !== '' && str_starts_with($refTable, $this->currentPrefix)) { - $refTable = substr($refTable, strlen($this->currentPrefix)); - } - - // current table depends on refTable -> edge from current to parent - // For tree of ancestors starting from $startTable, we build parent map - // but for associative array tree representation, we'll build nested children where parent has child - // i.e., parent -> [ child1, child2 ] based on references found in child - if (!isset($children[$tbl])) { $children[$tbl] = []; } - if (!in_array($refTable, $children[$tbl], true)) { - $children[$tbl][] = $refTable; - } - } - } - } - // Build recursive associative array tree starting at $startTable - $visited = []; - $buildTree = function(string $node) use (&$buildTree, &$children, &$visited) { - if (isset($visited[$node])) { - // cycle: denote specially - return ['__cycle__' => $node]; - } - $visited[$node] = true; - $kids = $children[$node] ?? []; - $tree = []; - foreach ($kids as $k) { - $tree[$k] = $buildTree($k); - } - return $tree; - }; - $tree = [ $startTable => $buildTree($startTable) ]; - // Output the array via Console IO as JSON for readability - $io->out(json_encode($tree, JSON_PRETTY_PRINT)); - } - - private function printAncestorTree(string $node, array $parents, array &$visited, ConsoleIo $io, string $prefix): void - { - if (isset($visited[$node])) { - $io->out($prefix . '↺ ' . $node . ' (cycle)'); - return; + $cols = $colsNode; } - $visited[$node] = true; - $edges = $parents[$node] ?? []; - $count = count($edges); - foreach ($edges as $idx => $edge) { - $isLast = ($idx === $count - 1); - $branch = $isLast ? '└── ' : '├── '; - $nextPrefix = $prefix . ($isLast ? ' ' : '│ '); - $label = $edge['to'] ?? '?'; // parent table - $via = $edge['via'] ?? ''; - $io->out($prefix . $branch . $label . ($via !== '' ? (' [' . $via . ']') : '')); - $this->printAncestorTree((string)$label, $parents, $visited, $io, $nextPrefix); + } + foreach ($cols as $idx => $col) { + if (!is_array($col)) { continue; } + $constraint = $col['constraint'] ?? null; + if (!is_string($constraint)) { continue; } + // Expect format: 'REFERENCE table(column)' + if (preg_match('/^REFERENCES\s+([A-Za-z0-9_\.]+)\s*\(([^)]+)\)/', $constraint, $m)) { + $refTable = $m[1]; + // Remove prefix if it was provided and matches + if ($this->currentPrefix !== '' && str_starts_with($refTable, $this->currentPrefix)) { + $refTable = substr($refTable, strlen($this->currentPrefix)); + } + + // current table depends on refTable -> edge from current to parent + // For tree of ancestors starting from $startTable, we build parent map + // but for associative array tree representation, we'll build nested children where parent has child + // i.e., parent -> [ child1, child2 ] based on references found in child + if (!isset($children[$tbl])) { $children[$tbl] = []; } + if (!in_array($refTable, $children[$tbl], true)) { + $children[$tbl][] = $refTable; + } } + } + } + // Build recursive associative array tree starting at $startTable + $visited = []; + $buildTree = function(string $node) use (&$buildTree, &$children, &$visited) { + if (isset($visited[$node])) { + // cycle: denote specially + return ['__cycle__' => $node]; + } + $visited[$node] = true; + $kids = $children[$node] ?? []; + $tree = []; + foreach ($kids as $k) { + $tree[$k] = $buildTree($k); + } + return $tree; + }; + $tree = [ $startTable => $buildTree($startTable) ]; + // Output the array via Console IO as JSON for readability + $io->out(json_encode($tree, JSON_PRETTY_PRINT)); + } + + /** + * Build legacy co_* foreign key rename patterns based on migration config. + * Returns an array of [suffixFrom => suffixTo], eg: + * '_co_person_id' => '_person_id' + * '_co_group_id' => '_group_id' + * '_co_message_template_id' => '_message_template_id' + * + * The patterns are applied as “ends-with” replacements to source column names. + */ + private function buildLegacyFkPatterns(array $tablesCfg, string $sourcePrefix): array + { + if (!is_array($tablesCfg) || empty($tablesCfg)) { + return [ + '_co_person_id' => '_person_id', + '_co_group_id' => '_group_id', + // Also support exact-form defaults for standalone columns + 'co_person_id' => 'person_id', + 'co_group_id' => 'group_id', + ]; + } + + // Filter out documentation keys and reduce to table entries that have a 'source' + $entries = []; + foreach ($tablesCfg as $k => $v) { + if (!is_array($v)) { continue; } + if (is_string($k) && str_starts_with($k, '__')) { continue; } + if (!isset($v['source']) || !is_string($v['source'])) { continue; } + $entries[$k] = $v['source']; } + + $patterns = [ + // suffix form (covers prefix + co_* cases) + '_co_person_id' => '_person_id', + '_co_group_id' => '_group_id', + // exact form (covers exact co_* columns with no left prefix) + 'co_person_id' => 'person_id', + 'co_group_id' => 'group_id', + ]; + + foreach ($entries as $targetKey => $src) { + $bareSource = $src; + if ($sourcePrefix !== '' && str_starts_with($bareSource, $sourcePrefix)) { + $bareSource = substr($bareSource, strlen($sourcePrefix)); + } + + // If bare source starts with "co_", derive both forms for this target + // targetKey 'message_templates' -> singular 'message_template' + if (str_starts_with($bareSource, 'co_')) { + $singular = \Cake\Utility\Inflector::singularize($targetKey); + + // suffix form: ..._co__id -> ...__id + $fromSuffix = '_co_' . $singular . '_id'; + $toSuffix = '_' . $singular . '_id'; + $patterns[$fromSuffix] = $toSuffix; + + // exact form: co__id -> _id + $fromExact = 'co_' . $singular . '_id'; + $toExact = $singular . '_id'; + $patterns[$fromExact] = $toExact; + } + } + + return $patterns; + } + + + /** + * Normalize a v4 source column name to a likely v5 target column name based on: + * - generic co_* foreign key patterns derived from migration config + * - specific one-offs + */ + private function normalizeTargetNameForSourceColumn(string $sourceCol, array $legacyFkPatterns): ?string + { + // Try exact-form first (eg, 'co_message_template_id' -> 'message_template_id') + if (isset($legacyFkPatterns[$sourceCol])) { + return $legacyFkPatterns[$sourceCol]; + } + + // Then apply suffix-based rewrites (eg, 'approver_co_group_id' -> 'approver_group_id') + foreach ($legacyFkPatterns as $from => $to) { + if ($from === $sourceCol) { continue; } // already handled exact + // only treat entries starting with '_' as suffix rules + if (str_starts_with($from, '_') && str_ends_with($sourceCol, $from)) { + $prefix = substr($sourceCol, 0, -strlen($from)); + return $prefix . $to; + } + } + + // Specific one-offs + if ($sourceCol === 'source_url') { + return 'source'; + } + if ($sourceCol === 'email_body') { + return 'email_body_text'; + } + + return null; + } } diff --git a/app/plugins/Transmogrify/src/Lib/Traits/HookRunnersTrait.php b/app/plugins/Transmogrify/src/Lib/Traits/HookRunnersTrait.php index 48a92b9e0..8ddf6c05c 100644 --- a/app/plugins/Transmogrify/src/Lib/Traits/HookRunnersTrait.php +++ b/app/plugins/Transmogrify/src/Lib/Traits/HookRunnersTrait.php @@ -44,7 +44,7 @@ private function runPreTableHook(string $table): void { throw new \RuntimeException("Unknown preTable hook: $method"); } - $this->io->verbose('Running pre-table hook: ' . ucfirst(preg_replace('/([A-Z])/', ' $1', $method))); + $this->cmdPrinter->verbose('Running pre-table hook: ' . ucfirst(preg_replace('/([A-Z])/', ' $1', $method))); $this->{$method}(); } @@ -62,7 +62,7 @@ private function runPostTableHook(string $table): void { if(!method_exists($this, $method)) { throw new \RuntimeException("Unknown postTable hook: $method"); } - $this->io->verbose('Running post-table hook: ' . ucfirst(preg_replace('/([A-Z])/', ' $1', $method))); + $this->cmdPrinter->verbose('Running post-table hook: ' . ucfirst(preg_replace('/([A-Z])/', ' $1', $method))); $this->{$method}(); } @@ -82,7 +82,7 @@ private function runPreRowHook(string $table, array &$origRow, array &$row): voi if(!method_exists($this, $method)) { throw new \RuntimeException("Unknown preRow hook: $method"); } - $this->io->verbose('Running pre-row hook: ' . ucfirst(preg_replace('/([A-Z])/', ' $1', $method))); + $this->cmdPrinter->verbose('Running pre-row hook: ' . ucfirst(preg_replace('/([A-Z])/', ' $1', $method))); $this->{$method}($origRow, $row); } @@ -102,7 +102,7 @@ private function runPostRowHook(string $table, array &$origRow, array &$row): vo if(!method_exists($this, $method)) { throw new \RuntimeException("Unknown postRow hook: $method"); } - $this->io->verbose('Running post-row hook: ' . ucfirst(preg_replace('/([A-Z])/', ' $1', $method))); + $this->cmdPrinter->verbose('Running post-row hook: ' . ucfirst(preg_replace('/([A-Z])/', ' $1', $method))); $this->{$method}($origRow, $row); } @@ -123,7 +123,7 @@ private function runSqlSelectHook(string $table, string $qualifiedTableName): st if(!method_exists(RawSqlQueries::class, $method)) { throw new \RuntimeException("Unknown sqlSelect hook: $method"); } - $this->io->verbose('Running SQL select hook: ' . ucfirst(preg_replace('/([A-Z])/', ' $1', $method))); + $this->cmdPrinter->verbose('Running SQL select hook: ' . ucfirst(preg_replace('/([A-Z])/', ' $1', $method))); // Special handling for MVEA-style selects: derive FK columns from fieldMap keys dynamically if ($method === 'mveaSqlSelect') { diff --git a/app/plugins/Transmogrify/src/Lib/Traits/ManageDefaultsTrait.php b/app/plugins/Transmogrify/src/Lib/Traits/ManageDefaultsTrait.php index 2afe257bf..8fcc56395 100644 --- a/app/plugins/Transmogrify/src/Lib/Traits/ManageDefaultsTrait.php +++ b/app/plugins/Transmogrify/src/Lib/Traits/ManageDefaultsTrait.php @@ -67,7 +67,7 @@ protected function createOwnersGroups(): void } } catch(\Exception $e) { - $this->io->error("Failed to create owners group for " + $this->cmdPrinter->error("Failed to create owners group for " . $group->name . " (" . $group->id . "): " . $e->getMessage()); } diff --git a/app/plugins/Transmogrify/src/Lib/Traits/RowTransformationTrait.php b/app/plugins/Transmogrify/src/Lib/Traits/RowTransformationTrait.php index 5516bbda1..6d5b0943a 100644 --- a/app/plugins/Transmogrify/src/Lib/Traits/RowTransformationTrait.php +++ b/app/plugins/Transmogrify/src/Lib/Traits/RowTransformationTrait.php @@ -76,7 +76,7 @@ protected function applyCheckGroupNameARRule(array $origRow, array &$row): void throw new \InvalidArgumentException('Replacing ":" produced an empty Standard CoGroup name; adjust --groups-colon-replacement'); } $row['name'] = $newName; - $this->io?->verbose(sprintf('Replaced ":" in Standard CoGroup name "%s" -> "%s"', $name, $newName)); + $this->cmdPrinter?->verbose(sprintf('Replaced ":" in Standard CoGroup name "%s" -> "%s"', $name, $newName)); } else { // Default: error out (no auto-replacement) throw new \InvalidArgumentException('Standard CoGroup names cannot contain a colon by default: ' . $name); @@ -123,7 +123,7 @@ protected function reconcileGroupMembershipOwnership(array $origRow, array &$row $qualifiedTableName = $this->outconn->qualifyTableName($tableName); $this->outconn->insert($qualifiedTableName, $ownerRow); } else { - $this->io->error("Could not find owners group for CoGroupMember " . $origRow['id']); + $this->cmdPrinter->error("Could not find owners group for CoGroupMember " . $origRow['id']); } } @@ -304,7 +304,7 @@ protected function migrateExtendedAttributesToAdHocAttributes(): void $qualifiedTableName = $this->outconn->qualifyTableName($tableName); $this->outconn->insert($qualifiedTableName, $adhocRow); } catch (\Doctrine\DBAL\Exception\UniqueConstraintViolationException $e) { - $this->io->warning("record already exists: " . print_r($adhocRow, true)); + $this->cmdPrinter->warning("record already exists: " . print_r($adhocRow, true)); } } } @@ -369,7 +369,7 @@ protected function mapExternalIdentityToExternalIdentityRole(array $origRow, arr ); } } catch (\Exception $e) { - $this->io->warning("Failed to map affiliation type: " . $e->getMessage()); + $this->cmdPrinter->warning("Failed to map affiliation type: " . $e->getMessage()); if ( isset($origRow['co_id']) && (!isset($this->cache['cos'][$origRow['co_id']]) diff --git a/app/plugins/Transmogrify/src/Lib/Traits/TypeMapperTrait.php b/app/plugins/Transmogrify/src/Lib/Traits/TypeMapperTrait.php index f96dec0aa..c6cfdbb33 100644 --- a/app/plugins/Transmogrify/src/Lib/Traits/TypeMapperTrait.php +++ b/app/plugins/Transmogrify/src/Lib/Traits/TypeMapperTrait.php @@ -239,7 +239,7 @@ protected function mapLoginIdentifiers(array $origRow, array &$row): void { $qualifiedTableName = $this->outconn->qualifyTableName($tableName); $this->outconn->insert($qualifiedTableName, $copiedRow); } catch (UniqueConstraintViolationException $e) { - $this->io->warning("record already exists: " . print_r($copiedRow, true)); + $this->cmdPrinter->warning("record already exists: " . print_r($copiedRow, true)); } } } @@ -308,7 +308,7 @@ protected function mapOrgIdentitycoPersonId(array $row): ?int $this->cache['org_identities']['co_people'] = []; // Build cache on first use - $this->io->verbose('Populating org identity map...'); + $this->cmdPrinter->verbose('Populating org identity map...'); $tableName = 'cm_co_org_identity_links'; $changelogFK = 'co_org_identity_link_id'; @@ -332,7 +332,7 @@ protected function mapOrgIdentitycoPersonId(array $row): ?int // improper unpooling from a legacy deployment. We'll accept only the // first record and throw warnings on the others. - $this->io->verbose("Found existing CO Person for Org Identity " . $oid . ", skipping"); + $this->cmdPrinter->verbose("Found existing CO Person for Org Identity " . $oid . ", skipping"); } else { // Store as-is; we'll resolve the latest revision on lookup $this->cache['org_identities']['co_people'][ $oid ][ $rowRev ] = (int)$r['co_person_id']; diff --git a/app/plugins/Transmogrify/src/Lib/Util/CommandLinePrinter.php b/app/plugins/Transmogrify/src/Lib/Util/CommandLinePrinter.php index 0ad16e8d3..06ae4ff25 100644 --- a/app/plugins/Transmogrify/src/Lib/Util/CommandLinePrinter.php +++ b/app/plugins/Transmogrify/src/Lib/Util/CommandLinePrinter.php @@ -5,105 +5,314 @@ use Cake\Console\ConsoleIo; /** - * DualAreaProgress + * Handles console output formatting including progress bars and colored messages * - * Keeps a progress bar on the top line and logs messages beneath it. - * Works with plain STDOUT or Cake's ConsoleIo. Uses ANSI escape codes. + * @since COmanage Registry v5.2.0 */ class CommandLinePrinter { + + /** + * Console IO instance for handling input/output + * + * @var ConsoleIo|null + * @since COmanage Registry v5.2.0 + */ private ?ConsoleIo $io; + + /** + * Total number of steps in the progress + * + * @var int + * @since COmanage Registry v5.2.0 + */ private int $total = 0; + + /** + * Current step in the progress + * + * @var int + * @since COmanage Registry v5.2.0 + */ private int $current = 0; - private int $messageLines = 0; // how many newline-terminated lines printed under the bar - private string $barColor; // 'blue' or 'green' + + /** + * Number of message lines printed + * + * @var int + * @since COmanage Registry v5.2.0 + */ + private int $messageLines = 0; + + /** + * Color of the progress bar + * + * @var string + * @since COmanage Registry v5.2.0 + */ + private string $barColor; + + /** + * Width of the progress bar in characters + * + * @var int + * @since COmanage Registry v5.2.0 + */ private int $barWidth; + + /** + * Whether to use ANSI colors in output + * + * @var bool + * @since COmanage Registry v5.2.0 + */ private bool $useColors; + /** + * Whether verbose output is enabled + * + * @var bool + * @since COmanage Registry v5.2.0 + */ + private bool $verbose; + + /** + * Whether the progress bar is currently active + * + * @var bool + * @since COmanage Registry v5.2.0 + */ + private bool $barActive = false; + + /** + * Constructor + * + * @param ConsoleIo|null $io Console IO instance for handling input/output + * @param string $barColor Color of the progress bar ('blue' or 'green') + * @param int $barWidth Width of the progress bar in characters + * @param bool $useColors Whether to use ANSI colors in output + * @since COmanage Registry v5.2.0 + */ public function __construct(?ConsoleIo $io = null, string $barColor = 'blue', int $barWidth = 50, bool $useColors = true) { $this->io = $io; $this->barColor = in_array($barColor, ['blue', 'green'], true) ? $barColor : 'blue'; $this->barWidth = max(10, $barWidth); $this->useColors = $useColors; + // Try to detect verbose mode from ConsoleIo if available + $this->verbose = $this->detectVerboseFromIo(); + } - /** Initialize the two-area layout and draw the 0% bar */ + /** + * Start displaying a new progress bar + * + * @param int $total Total number of steps + * @return void + * @since COmanage Registry v5.2.0 + */ public function start(int $total): void { + // When verbose is enabled, do not draw the progress bar at all + if ($this->verbose) { + return; + } + $this->total = max(0, $total); $this->current = 0; $this->messageLines = 0; + $this->barActive = true; + + // Hard reset the current line, then move to a brand new line + // \r -> move to column 0 + // \033[K -> clear to end of line + // \n -> go to next line + $this->rawWrite("\r\033[2K"); - // Draw initial progress bar line (without trailing newline) - $this->rawWrite("\r" . $this->formatBar(0)); + // Draw initial progress bar line (no leading \r, we are already at col 0) + $this->rawWrite($this->formatBar(0)); // Save cursor position at the end of the progress bar line $this->rawWrite("\033[s"); - // Move cursor back down ONLY if messages exist - if ($this->messageLines > 0) { - $this->rawWrite("\033[" . $this->messageLines . "B\r"); - } + // Move to message area (one line below the bar) + $this->rawWrite("\033[1B\r"); } - /** Update the progress bar (0..total). Call as work advances. */ + /** + * Update the progress bar to show current progress + * + * @param int $current Current step number + * @return void + * @since COmanage Registry v5.2.0 + */ public function update(int $current): void { + // When verbose is enabled, do not draw the progress bar at all + if ($this->verbose || !$this->barActive) { + return; + } + $this->current = min(max(0, $current), $this->total); - // Restore to saved position (progress bar line), redraw, save again - $this->rawWrite("\033[u"); // restore saved cursor (bar line) + // Restore to bar line, redraw bar, save, then go back to message area + $this->rawWrite("\033[u"); // restore saved cursor (bar line end) $this->rawWrite("\r" . $this->formatBar($this->current)); - $this->rawWrite("\033[s"); // re-save position at end of bar - - // Move cursor back down to where messages continue - if ($this->messageLines > 0) { - $this->rawWrite("\033[" . $this->messageLines . "B\r"); - } else { - // exactly one line below the bar is the first message line - $this->rawWrite("\033[1B\r"); - } + $this->rawWrite("\033[s"); // save again at end of bar + $down = 1 + $this->messageLines; // one below bar + existing messages + $this->rawWrite("\033[" . $down . "B\r"); } - /** Finish: force bar to 100% and add a newline separating it from any further output */ + /** + * Complete and cleanup the progress bar display + * + * @return void + * @since COmanage Registry v5.2.0 + */ public function finish(): void { + // When verbose is enabled, do not draw the progress bar at all + if ($this->verbose || !$this->barActive) { + return; + } + $this->update($this->total); - // Move cursor to end of messages (already there), and ensure a newline after the last message block - $this->rawWrite(PHP_EOL); + + // Move the cursor to the line AFTER the whole message area, + // so subsequent output doesn't overwrite the bar. + $this->rawWrite("\033[u"); // restore saved position at end of bar line + $down = 1 + $this->messageLines; // one below bar + all message lines + $this->rawWrite("\033[" . $down . "B\r"); // move down + $this->rawWrite(PHP_EOL); // clean newline below + + // Re-anchor saved cursor to this clean line so future restores are safe + $this->rawWrite("\033[s"); + + // Reset internal counters so next run starts fresh + $this->messageLines = 0; + $this->barActive = false; + $this->total = 0; + $this->current = 0; } - // Convenience wrappers for messages under the bar - public function info(string $message): void { $this->message($message, 'info'); } - public function warn(string $message): void { $this->message($message, 'warn'); } - public function error(string $message): void { $this->message($message, 'error'); } - public function debug(string $message): void { $this->message($message, 'debug'); } + /** + * Display an info level message + * + * @param string $message Message to display + * @return void + * @since COmanage Registry v5.2.0 + */ + public function info(string $message): void + { + $this->message($message, 'info'); + } + + /** + * Display a warning level message + * + * @param string $message Message to display + * @return void + * @since COmanage Registry v5.2.0 + */ + public function warn(string $message): void + { + $this->message($message, 'warn'); + } - /** Print a message under the bar. Handles multi-line strings. */ + /** + * Display a warning level message (alias of warn()) + * + * @param string $message Message to display + * @return void + * @since COmanage Registry v5.2.0 + */ + public function warning(string $message): void + { + $this->message($message, 'warn'); + } + + /** + * Display an error level message + * + * @param string $message Message to display + * @return void + * @since COmanage Registry v5.2.0 + */ + public function error(string $message): void + { + $this->message($message, 'error'); + } + + /** + * Display a debug level message + * + * @param string $message Message to display + * @return void + * @since COmanage Registry v5.2.0 + */ + public function debug(string $message): void + { + $this->message($message, 'debug'); + } + + /** + * Display a verbose level message + * + * @param string $message Message to display + * @return void + * @since COmanage Registry v5.2.0 + */ + public function verbose(string $message): void { $this->message($message, 'verbose'); } + + /** + * Display a message with the specified level + * + * @param string $message Message to display + * @param string $level Message level (info, warn, error, debug, verbose) + * @return void + * @since COmanage Registry v5.2.0 + */ public function message(string $message, string $level = 'info'): void { + // Suppress verbose messages unless verbose mode is enabled + if (strtolower($level) === 'verbose' && $this->verbose !== true) { + return; + } + $lines = $this->colorizeLevel($level, $message); // Ensure the message ends with a newline so we can count line advances if ($lines === '' || substr($lines, -1) !== "\n") { $lines .= "\n"; } - // If this is the first message, create the message area by moving to the next line - if ($this->messageLines === 0) { - $this->rawWrite(PHP_EOL); - } - $this->rawWrite($lines); + if ($this->barActive) { + // Always print below the bar + $this->rawWrite("\033[u"); // restore to end of bar line + $down = 1 + $this->messageLines; + $this->rawWrite("\033[" . $down . "B"); // move to message area line + // Clear the entire message line and reset cursor to column 0 + $this->rawWrite("\r\033[2K"); + $this->rawWrite($lines); + $this->messageLines += substr_count($lines, "\n"); - // Count how many terminal lines were added based on newlines seen. - // Note: This doesn’t account for soft-wrapped lines; keep messages reasonably short. - $this->messageLines += substr_count($lines, "\n"); + // Return to bar line and re-save for next update + $this->rawWrite("\033[u"); + $this->rawWrite("\033[s"); + } else { + // No bar: clear current line completely and print from column 0 + $this->rawWrite("\r\033[2K"); + $this->rawWrite($lines); + } } /** - * Prompt the user for input while keeping the bar/message layout intact. - * Returns the provided answer (or null on EOF). + * Prompt for user input + * + * @param string $prompt Prompt to display + * @param string|null $default Default value if no input provided + * @return string|null User input or default value + * @since COmanage Registry v5.2.0 */ public function ask(string $prompt, ?string $default = null): ?string { @@ -142,38 +351,55 @@ public function ask(string $prompt, ?string $default = null): ?string } /** - * Convenience: prompt to continue (ENTER). + * Pause execution until user presses enter + * + * @param string $prompt Prompt to display + * @return void + * @since COmanage Registry v5.2.0 */ public function pause(string $prompt = 'Press to continue...'): void { $this->ask($prompt, ''); } - + /** + * Add color formatting to a message based on its level + * + * @param string $level Message level + * @param string $message Message to colorize + * @return string Formatted message + * @since COmanage Registry v5.2.0 + */ private function colorizeLevel(string $level, string $message): string { $level = strtolower($level); $prefix = ''; - $color = null; switch ($level) { case 'warn': case 'warning': $prefix = '[WARN] '; - $color = 'yellow'; break; case 'error': $prefix = '[ERROR] '; - $color = 'red'; break; case 'debug': $prefix = '[DEBUG] '; - $color = 'cyan'; break; + case 'verbose': + $prefix = '[VERBOSE] '; + break; + default: $prefix = '[INFO] '; - $color = null; // default terminal color } + // For INFO, render the label (up to the first colon) in white, value unchanged (or green if white info default) + if ($level === 'info') { + $formatted = $this->useColors ? $this->formatInfoLabelWhite($message) : $message; + return $prefix . $formatted; + } + + $color = $this->defaultColorForLevel($level); $text = $prefix . $message; if ($this->useColors && $color) { return $this->wrapColor($text, $color); @@ -181,6 +407,31 @@ private function colorizeLevel(string $level, string $message): string return $text; } + /** + * Get the default color for a message level + * + * @param string $level Message level + * @return string|null Color name or null if no color + * @since COmanage Registry v5.2.0 + */ + private function defaultColorForLevel(string $level): ?string + { + $level = strtolower($level); + return match ($level) { + 'warn', 'warning' => 'yellow', + 'error' => 'red', + 'debug' => 'cyan', + default => 'white', + }; + } + + /** + * Format the progress bar string + * + * @param int $current Current progress value + * @return string Formatted progress bar + * @since COmanage Registry v5.2.0 + */ private function formatBar(int $current): string { $total = max(1, $this->total); @@ -205,6 +456,14 @@ private function formatBar(int $current): string return $bar; } + /** + * Wrap text in ANSI color codes + * + * @param string $text Text to colorize + * @param string $color Color name + * @return string Color-wrapped text + * @since COmanage Registry v5.2.0 + */ private function wrapColor(string $text, string $color): string { $map = [ @@ -214,12 +473,20 @@ private function wrapColor(string $text, string $color): string 'blue' => '0;34', 'magenta'=> '0;35', 'cyan' => '0;36', + 'white' => '0;39', ]; $code = $map[$color] ?? null; if (!$code) { return $text; } return "\033[{$code}m{$text}\033[0m"; } + /** + * Write raw string to output + * + * @param string $str String to write + * @return void + * @since COmanage Registry v5.2.0 + */ private function rawWrite(string $str): void { if ($this->io) { @@ -230,4 +497,49 @@ private function rawWrite(string $str): void echo $str; } } + + /** + * Format info message with white label + * + * @param string $message Message to format + * @return string Formatted message + * @since COmanage Registry v5.2.0 + */ + private function formatInfoLabelWhite(string $message): string + { + $lines = explode("\n", $message); + foreach ($lines as $i => $line) { + if ($line === '') { continue; } + if (preg_match('/^([^:\r\n]+:)(.*)$/', $line, $m)) { + $second = $m[2]; + if ($this->useColors && $this->defaultColorForLevel('info') === 'white' && $second !== '') { + $second = $this->wrapColor($second, 'green'); + } + $lines[$i] = $this->wrapColor($m[1], 'white') . $second; + } + } + return implode("\n", $lines); + } + + /** + * Detect verbose mode from ConsoleIO instance + * + * @return bool Whether verbose mode is enabled + * @since COmanage Registry v5.2.0 + */ + private function detectVerboseFromIo(): bool + { + if ($this->io === null) { + return false; + } + if (method_exists($this->io, 'level')) { + try { + $level = (int)$this->io->level(); + return $level >= 2; // VERBOSE + } catch (\Throwable $e) { + return false; + } + } + return false; + } } diff --git a/app/plugins/Transmogrify/src/Lib/Util/RawSqlQueries.php b/app/plugins/Transmogrify/src/Lib/Util/RawSqlQueries.php index 8baeb8f2d..d716c5736 100644 --- a/app/plugins/Transmogrify/src/Lib/Util/RawSqlQueries.php +++ b/app/plugins/Transmogrify/src/Lib/Util/RawSqlQueries.php @@ -131,7 +131,7 @@ public static function buildSequenceReset(string $qualifiedTableName, int $nextI * @param DBALConnection $outconn Connection to target database * @param string $sourceTable Name of source table * @param string $targetTable Name of target table - * @param ConsoleIo $cio Console IO object for output + * @param CommandLinePrinter CommandLinePrinter IO object for output * @return bool True if sequence was reset successfully, false otherwise * @since COmanage Registry v5.2.0 */ @@ -140,14 +140,14 @@ public static function setSequenceId( DBALConnection $outconn, string $sourceTable, string $targetTable, - ConsoleIo $cio, + CommandLinePrinter $cmdPrinter, ): bool { $qualifiedTableName = $inconn->qualifyTableName($sourceTable); $maxId = $inconn->fetchOne(self::buildSelectMaxId($qualifiedTableName)); $maxId = ((int)($maxId ?? 0)) + 1; $qualifiedTableName = $outconn->qualifyTableName($targetTable); - $cio->info("Resetting primary key sequence for $qualifiedTableName to $maxId"); + $cmdPrinter->info("Resetting primary key sequence for $qualifiedTableName to $maxId"); // Strictly speaking we should use prepared statements, but we control the // data here, and also we're executing a maintenance operation (so query From bf1da9f0ee585bb9592d0d3e555f7d091c8e0584 Mon Sep 17 00:00:00 2001 From: Ioannis Igoumenos Date: Fri, 17 Oct 2025 20:00:48 +0300 Subject: [PATCH 13/15] Fix verbosity print --- .../src/Command/TransmogrifyCommand.php | 22 ++-- .../src/Lib/Util/CommandLinePrinter.php | 104 ++++++++++++------ 2 files changed, 81 insertions(+), 45 deletions(-) diff --git a/app/plugins/Transmogrify/src/Command/TransmogrifyCommand.php b/app/plugins/Transmogrify/src/Command/TransmogrifyCommand.php index 24a5ecaae..9bd4daaa8 100644 --- a/app/plugins/Transmogrify/src/Command/TransmogrifyCommand.php +++ b/app/plugins/Transmogrify/src/Command/TransmogrifyCommand.php @@ -280,7 +280,7 @@ public function execute(Arguments $args, ConsoleIo $io): int $outboundQualifiedTableName = $this->outconn->qualifyTableName($t); $Model = TableRegistry::getTableLocator()->get($t); - $this->cmdPrinter->info(message: sprintf("Transmogrifying table: %s(%s)", Inflector::classify($t), $t)); + $this->cmdPrinter->out(message: sprintf("Transmogrifying table: %s(%s)", Inflector::classify($t), $t)); /* @@ -372,8 +372,6 @@ public function execute(Arguments $args, ConsoleIo $io): int $origRow = $row; // Execute any pre-processing hooks to transform or validate the row data - // TODO: if i need to skip the insert i want something shared. I can add this in the cache - // and then skip insert from everywhere. $this->runPreRowHook($t, $origRow, $row); // Set changelog defaults (created/modified timestamps, user IDs) @@ -412,7 +410,7 @@ public function execute(Arguments $args, ConsoleIo $io): int $warns++; $this->cache['rejected'][$outboundQualifiedTableName][$row['id']] = $row; $this->cmdPrinter->warn("Skipping $t record " . $row['id'] . " due to invalid foreign key: " . $e->getMessage()); -// $this->cmdPrinter->pause(); + $this->cmdPrinter->pause(); } catch(\InvalidArgumentException $e) { // If we can't find a value for mapping we skip the record @@ -420,19 +418,17 @@ public function execute(Arguments $args, ConsoleIo $io): int $warns++; $this->cache['rejected'][$outboundQualifiedTableName][$row['id']] = $row; $this->cmdPrinter->warn("Skipping $t record " . $row['id'] . ": " . $e->getMessage()); -// $this->cmdPrinter->pause(); + $this->cmdPrinter->pause(); } catch(\Exception $e) { $err++; $this->cmdPrinter->error("$t record " . $row['id'] . ": " . $e->getMessage()); -// $this->cmdPrinter->pause(); + $this->cmdPrinter->pause(); } $tally++; - - if (!$this->args->getOption('quiet') && !$this->args->getOption('verbose')) { - $this->cmdPrinter->update($tally); - } + // Always delegate progress updates to the printer; it will decide what to draw + $this->cmdPrinter->update($tally); } $this->cmdPrinter->finish(); @@ -446,7 +442,7 @@ public function execute(Arguments $args, ConsoleIo $io): int // Execute any post-processing hooks for the table if($this->cache['skipInsert'][$outboundQualifiedTableName] === false) { - $this->cmdPrinter->info('Running post-table hook for ' . $t); + $this->cmdPrinter->out('Running post-table hook for ' . $t); $this->runPostTableHook($t); } @@ -454,7 +450,7 @@ public function execute(Arguments $args, ConsoleIo $io): int if (!empty($pendingSelected) && isset($pendingSelected[$t])) { unset($pendingSelected[$t]); if (empty($pendingSelected)) { - $this->cmdPrinter->info('All selected tables have been processed. Exiting.'); + $this->cmdPrinter->out('All selected tables have been processed. Exiting.'); return BaseCommand::CODE_SUCCESS; } } @@ -465,7 +461,7 @@ public function execute(Arguments $args, ConsoleIo $io): int if (isset($tables[$currentIndex + 1])) { $this->cmdPrinter->info("Next table to process: " . $tables[$currentIndex + 1]); } else { - $this->cmdPrinter->info("This is the last table to process."); + $this->cmdPrinter->out(PHP_EOL. "Table import complete. Exiting."); } $this->cmdPrinter->pause(); diff --git a/app/plugins/Transmogrify/src/Lib/Util/CommandLinePrinter.php b/app/plugins/Transmogrify/src/Lib/Util/CommandLinePrinter.php index 06ae4ff25..052d5646f 100644 --- a/app/plugins/Transmogrify/src/Lib/Util/CommandLinePrinter.php +++ b/app/plugins/Transmogrify/src/Lib/Util/CommandLinePrinter.php @@ -68,14 +68,6 @@ class CommandLinePrinter */ private bool $useColors; - /** - * Whether verbose output is enabled - * - * @var bool - * @since COmanage Registry v5.2.0 - */ - private bool $verbose; - /** * Whether the progress bar is currently active * @@ -99,9 +91,6 @@ public function __construct(?ConsoleIo $io = null, string $barColor = 'blue', in $this->barColor = in_array($barColor, ['blue', 'green'], true) ? $barColor : 'blue'; $this->barWidth = max(10, $barWidth); $this->useColors = $useColors; - // Try to detect verbose mode from ConsoleIo if available - $this->verbose = $this->detectVerboseFromIo(); - } /** @@ -114,7 +103,7 @@ public function __construct(?ConsoleIo $io = null, string $barColor = 'blue', in public function start(int $total): void { // When verbose is enabled, do not draw the progress bar at all - if ($this->verbose) { + if ($this->getVerboseLevel() > 1) { return; } @@ -149,7 +138,7 @@ public function start(int $total): void public function update(int $current): void { // When verbose is enabled, do not draw the progress bar at all - if ($this->verbose || !$this->barActive) { + if ($this->getVerboseLevel() > 1 || !$this->barActive) { return; } @@ -172,7 +161,7 @@ public function update(int $current): void public function finish(): void { // When verbose is enabled, do not draw the progress bar at all - if ($this->verbose || !$this->barActive) { + if ($this->getVerboseLevel() > 1 || !$this->barActive) { return; } @@ -195,6 +184,18 @@ public function finish(): void $this->current = 0; } + /** + * Display an out level message + * + * @param string $message Message to display + * @return void + * @since COmanage Registry v5.2.0 + */ + public function out(string $message): void + { + $this->message($message, 'out'); + } + /** * Display an info level message * @@ -262,7 +263,10 @@ public function debug(string $message): void * @return void * @since COmanage Registry v5.2.0 */ - public function verbose(string $message): void { $this->message($message, 'verbose'); } + public function verbose(string $message): void + { + $this->message($message, 'verbose'); + } /** * Display a message with the specified level @@ -275,7 +279,7 @@ public function verbose(string $message): void { $this->message($message, 'verb public function message(string $message, string $level = 'info'): void { // Suppress verbose messages unless verbose mode is enabled - if (strtolower($level) === 'verbose' && $this->verbose !== true) { + if (!$this->shouldPrintLevel($level)) { return; } @@ -373,7 +377,6 @@ public function pause(string $prompt = 'Press to continue...'): void private function colorizeLevel(string $level, string $message): string { $level = strtolower($level); - $prefix = ''; switch ($level) { case 'warn': case 'warning': @@ -388,14 +391,16 @@ private function colorizeLevel(string $level, string $message): string case 'verbose': $prefix = '[VERBOSE] '; break; - - default: + case 'info': $prefix = '[INFO] '; + break; + default: + $prefix = ''; } // For INFO, render the label (up to the first colon) in white, value unchanged (or green if white info default) - if ($level === 'info') { - $formatted = $this->useColors ? $this->formatInfoLabelWhite($message) : $message; + if ($level === 'info' || $level === 'out') { + $formatted = $this->useColors ? $this->formatLabelWhite($message) : $message; return $prefix . $formatted; } @@ -491,7 +496,7 @@ private function rawWrite(string $str): void { if ($this->io) { // ConsoleIo::out() defaults to a trailing newline; we want raw text - $this->io->out($str, 0); + $this->io->out($str, 0, 0); } else { // Fallback to STDOUT echo $str; @@ -505,14 +510,18 @@ private function rawWrite(string $str): void * @return string Formatted message * @since COmanage Registry v5.2.0 */ - private function formatInfoLabelWhite(string $message): string + private function formatLabelWhite(string $message): string { $lines = explode("\n", $message); foreach ($lines as $i => $line) { if ($line === '') { continue; } if (preg_match('/^([^:\r\n]+:)(.*)$/', $line, $m)) { $second = $m[2]; - if ($this->useColors && $this->defaultColorForLevel('info') === 'white' && $second !== '') { + if ( + $this->useColors + && ($this->defaultColorForLevel('info') === 'white' || $this->defaultColorForLevel('out') === 'white') + && $second !== '' + ) { $second = $this->wrapColor($second, 'green'); } $lines[$i] = $this->wrapColor($m[1], 'white') . $second; @@ -522,24 +531,55 @@ private function formatInfoLabelWhite(string $message): string } /** - * Detect verbose mode from ConsoleIO instance + * Detect verbose level from ConsoleIO instance * - * @return bool Whether verbose mode is enabled + * @return int Verbose level * @since COmanage Registry v5.2.0 */ - private function detectVerboseFromIo(): bool + private function detectVerboseFromIo(): int { + $default = 0; if ($this->io === null) { - return false; + return $default; } if (method_exists($this->io, 'level')) { try { - $level = (int)$this->io->level(); - return $level >= 2; // VERBOSE + return $this->io->level(); } catch (\Throwable $e) { - return false; + return $default; } } - return false; + return $default; + } + + /** + * Determine if a message of the given level should be printed based on verbosity setting + * + * @param string $level Message level to check + * @return bool Whether the message should be printed + * @since COmanage Registry v5.2.0 + */ + private function shouldPrintLevel(string $level): bool + { + $level = strtolower($level); + $verboseLevel = $this->detectVerboseFromIo(); + + return match ($verboseLevel) { + 0 => in_array($level, ['out', 'error'], true), // only errors and out + 1 => in_array($level, ['out', 'info', 'warn', 'warning', 'error'], true), // no verbose/debug + 2 => in_array($level, ['out', 'info', 'warn', 'warning', 'error', 'debug', 'verbose'], true), // all messages + default => true, + }; + } + + /** + * Get the current verbosity level + * + * @return int Verbosity level (0=quiet, 1=normal, 2=verbose) + * @since COmanage Registry v5.2.0 + */ + private function getVerboseLevel(): int + { + return $this->detectVerboseFromIo(); } } From 76378fc45aebd6ab42a323bd80dd8468a4e928b9 Mon Sep 17 00:00:00 2001 From: Ioannis Igoumenos Date: Sat, 18 Oct 2025 16:51:16 +0300 Subject: [PATCH 14/15] cleaning --- .../src/Command/TransmogrifyCommand.php | 11 ++++------- .../src/Lib/Util/CommandLinePrinter.php | 16 ++++------------ 2 files changed, 8 insertions(+), 19 deletions(-) diff --git a/app/plugins/Transmogrify/src/Command/TransmogrifyCommand.php b/app/plugins/Transmogrify/src/Command/TransmogrifyCommand.php index 9bd4daaa8..17377e906 100644 --- a/app/plugins/Transmogrify/src/Command/TransmogrifyCommand.php +++ b/app/plugins/Transmogrify/src/Command/TransmogrifyCommand.php @@ -210,9 +210,6 @@ public function execute(Arguments $args, ConsoleIo $io): int $this->args = $args; $this->io = $io; - // Info is forced to be white - $io->setStyle('info', ['text' => '0;39']); - // Now that BaseCommand set verbosity, construct the printer so it can detect it correctly $this->cmdPrinter = new CommandLinePrinter($io, 'green', 50, true); @@ -409,7 +406,7 @@ public function execute(Arguments $args, ConsoleIo $io): int // not linked to a CO Person that was not migrated. $warns++; $this->cache['rejected'][$outboundQualifiedTableName][$row['id']] = $row; - $this->cmdPrinter->warn("Skipping $t record " . $row['id'] . " due to invalid foreign key: " . $e->getMessage()); + $this->cmdPrinter->warning("Skipping $t record " . $row['id'] . " due to invalid foreign key: " . $e->getMessage()); $this->cmdPrinter->pause(); } catch(\InvalidArgumentException $e) { @@ -417,7 +414,7 @@ public function execute(Arguments $args, ConsoleIo $io): int // (ie: mapLegacyFieldNames basically requires a successful mapping) $warns++; $this->cache['rejected'][$outboundQualifiedTableName][$row['id']] = $row; - $this->cmdPrinter->warn("Skipping $t record " . $row['id'] . ": " . $e->getMessage()); + $this->cmdPrinter->warning("Skipping $t record " . $row['id'] . ": " . $e->getMessage()); $this->cmdPrinter->pause(); } catch(\Exception $e) { @@ -678,7 +675,7 @@ private function skipIfRejectedParent(string $currentTable, array $row): bool if ($childId !== null) { $this->cache['rejected'][$qualifiedCurrent][$childId] = $row; } - $this->cmdPrinter->warn(sprintf( + $this->cmdPrinter->warning(sprintf( 'Skipping record %d in table %s - parent %s(%d) was rejected (self-reference)', (int)($childId ?? 0), $currentTable, @@ -711,7 +708,7 @@ private function skipIfRejectedParent(string $currentTable, array $row): bool if ($childId !== null) { $this->cache['rejected'][$qualifiedCurrent][$childId] = $row; } - $this->cmdPrinter->warn(sprintf( + $this->cmdPrinter->warning(sprintf( 'Skipping record %d in table %s - parent %s(%d) was rejected', (int)($childId ?? 0), $currentTable, diff --git a/app/plugins/Transmogrify/src/Lib/Util/CommandLinePrinter.php b/app/plugins/Transmogrify/src/Lib/Util/CommandLinePrinter.php index 052d5646f..80f2f5a04 100644 --- a/app/plugins/Transmogrify/src/Lib/Util/CommandLinePrinter.php +++ b/app/plugins/Transmogrify/src/Lib/Util/CommandLinePrinter.php @@ -91,6 +91,10 @@ public function __construct(?ConsoleIo $io = null, string $barColor = 'blue', in $this->barColor = in_array($barColor, ['blue', 'green'], true) ? $barColor : 'blue'; $this->barWidth = max(10, $barWidth); $this->useColors = $useColors; + + // Info is forced to be white + $this->io->setStyle('info', ['text' => '0;39']); + $this->io->setStyle('question', ['text' => '0;39']); } /** @@ -208,18 +212,6 @@ public function info(string $message): void $this->message($message, 'info'); } - /** - * Display a warning level message - * - * @param string $message Message to display - * @return void - * @since COmanage Registry v5.2.0 - */ - public function warn(string $message): void - { - $this->message($message, 'warn'); - } - /** * Display a warning level message (alias of warn()) * From 0d234ee187439b6d510f150539639d27bf121c4b Mon Sep 17 00:00:00 2001 From: Ioannis Igoumenos Date: Tue, 21 Oct 2025 11:12:18 +0300 Subject: [PATCH 15/15] Put warn counting in cache.Load the plugin in the Application.php. --- .../src/Command/TransmogrifyCommand.php | 14 +++++++------- .../src/Lib/Traits/ManageDefaultsTrait.php | 1 + .../src/Lib/Traits/RowTransformationTrait.php | 1 + app/src/Application.php | 6 ++++++ 4 files changed, 15 insertions(+), 7 deletions(-) diff --git a/app/plugins/Transmogrify/src/Command/TransmogrifyCommand.php b/app/plugins/Transmogrify/src/Command/TransmogrifyCommand.php index 17377e906..73a325a1e 100644 --- a/app/plugins/Transmogrify/src/Command/TransmogrifyCommand.php +++ b/app/plugins/Transmogrify/src/Command/TransmogrifyCommand.php @@ -356,8 +356,8 @@ public function execute(Arguments $args, ConsoleIo $io): int } $this->cmdPrinter->start($count); $tally = 0; - $warns = 0; - $err = 0; + $this->cache['error'] = 0; + $this->cache['warns'] = 0; while ($row = $stmt->fetchAssociative()) { if(!empty($row[ $this->tables[$t]['displayField'] ])) { @@ -404,7 +404,7 @@ public function execute(Arguments $args, ConsoleIo $io): int // load this record. This can happen, eg, because the source_field_id // did not load, perhaps because it was associated with an Org Identity // not linked to a CO Person that was not migrated. - $warns++; + $this->cache['warns'] += 1; $this->cache['rejected'][$outboundQualifiedTableName][$row['id']] = $row; $this->cmdPrinter->warning("Skipping $t record " . $row['id'] . " due to invalid foreign key: " . $e->getMessage()); $this->cmdPrinter->pause(); @@ -412,13 +412,13 @@ public function execute(Arguments $args, ConsoleIo $io): int catch(\InvalidArgumentException $e) { // If we can't find a value for mapping we skip the record // (ie: mapLegacyFieldNames basically requires a successful mapping) - $warns++; + $this->cache['warns'] += 1; $this->cache['rejected'][$outboundQualifiedTableName][$row['id']] = $row; $this->cmdPrinter->warning("Skipping $t record " . $row['id'] . ": " . $e->getMessage()); $this->cmdPrinter->pause(); } catch(\Exception $e) { - $err++; + $this->cache['error'] += 1; $this->cmdPrinter->error("$t record " . $row['id'] . ": " . $e->getMessage()); $this->cmdPrinter->pause(); } @@ -434,8 +434,8 @@ public function execute(Arguments $args, ConsoleIo $io): int */ // Output final warning and error counts for the table - $this->cmdPrinter->warning(sprintf('Warnings: %d', $warns)); - $this->cmdPrinter->error(sprintf('Errors: %d', $err)); + $this->cmdPrinter->warning(sprintf('Warnings: %d', $this->cache['warns'])); + $this->cmdPrinter->error(sprintf('Errors: %d', $this->cache['error'])); // Execute any post-processing hooks for the table if($this->cache['skipInsert'][$outboundQualifiedTableName] === false) { diff --git a/app/plugins/Transmogrify/src/Lib/Traits/ManageDefaultsTrait.php b/app/plugins/Transmogrify/src/Lib/Traits/ManageDefaultsTrait.php index 8fcc56395..d1f4a8ec9 100644 --- a/app/plugins/Transmogrify/src/Lib/Traits/ManageDefaultsTrait.php +++ b/app/plugins/Transmogrify/src/Lib/Traits/ManageDefaultsTrait.php @@ -67,6 +67,7 @@ protected function createOwnersGroups(): void } } catch(\Exception $e) { + $this->cache['error'] += 1; $this->cmdPrinter->error("Failed to create owners group for " . $group->name . " (" . $group->id . "): " . $e->getMessage()); diff --git a/app/plugins/Transmogrify/src/Lib/Traits/RowTransformationTrait.php b/app/plugins/Transmogrify/src/Lib/Traits/RowTransformationTrait.php index 6d5b0943a..dd7d1194c 100644 --- a/app/plugins/Transmogrify/src/Lib/Traits/RowTransformationTrait.php +++ b/app/plugins/Transmogrify/src/Lib/Traits/RowTransformationTrait.php @@ -124,6 +124,7 @@ protected function reconcileGroupMembershipOwnership(array $origRow, array &$row $this->outconn->insert($qualifiedTableName, $ownerRow); } else { $this->cmdPrinter->error("Could not find owners group for CoGroupMember " . $origRow['id']); + $this->cache['error'] += 1; } } diff --git a/app/src/Application.php b/app/src/Application.php index 1e57c5454..ae0b8a6dd 100644 --- a/app/src/Application.php +++ b/app/src/Application.php @@ -69,6 +69,12 @@ public function bootstrap(): void foreach($activePlugins as $p) { $this->addPlugin($p->plugin); } + + // The Transmogrify plugin is required for all applications + // It is a special use case. + if (!\Cake\Core\Plugin::isLoaded('Transmogrify')) { + $this->addPlugin('Transmogrify'); + } } catch(\Cake\Database\Exception\DatabaseException $e) { // Most likely we are performing the initial database setup and