diff --git a/app/plugins/Transmogrify/config/schema/tables.json b/app/plugins/Transmogrify/config/schema/tables.json index 803002e45..ce2cc0cab 100644 --- a/app/plugins/Transmogrify/config/schema/tables.json +++ b/app/plugins/Transmogrify/config/schema/tables.json @@ -112,7 +112,7 @@ "source": "cm_co_person_roles", "sqlSelect": "roleSqlSelect", "displayField": "id", - "cache": ["status"], + "cache": ["status", "person_id", "manager_person_id", "sponsor_person_id"], "fieldMap": { "co_person_id": "person_id", "co_person_role_id": "person_role_id", @@ -173,6 +173,7 @@ "source": "cm_co_group_members", "displayField": "id", "booleans": ["member", "owner"], + "preRow": "reconcileGroupMembershipOwnership", "fieldMap": { "co_group_id": "group_id", "co_person_id": "person_id", @@ -181,8 +182,7 @@ "co_group_nesting_id": "group_nesting_id", "co_group_member_id": "group_member_id", "source_org_identity_id": null - }, - "preRow": "reconcileGroupMembershipOwnership" + } }, "names": { "source": "cm_names", @@ -190,9 +190,9 @@ "booleans": ["primary_name"], "sqlSelect": "mveaSqlSelect", "fieldMap": { + "type_id": "&mapNameType", "co_person_id": "person_id", "org_identity_id": "external_identity_id", - "type_id": "&mapNameType", "type": null } }, @@ -202,12 +202,12 @@ "booleans": ["verified"], "sqlSelect": "mveaSqlSelect", "fieldMap": { + "type_id": "&mapEmailType", "co_person_id": "person_id", "org_identity_id": "external_identity_id", - "type_id": "&mapEmailType", - "type": null, "co_department_id": null, - "organization_id": null + "organization_id": null, + "type": null } }, "identifiers": { @@ -215,68 +215,70 @@ "displayField": "id", "booleans": ["login"], "sqlSelect": "mveaSqlSelect", + "preRow": "mapLoginIdentifiers", "fieldMap": { + "type_id": "&mapIdentifierType", "co_group_id": "group_id", "co_person_id": "person_id", "org_identity_id": "external_identity_id", - "type_id": "&mapIdentifierType", - "type": null, "co_department_id": null, "co_provisioning_target_id": null, - "organization_id": null - }, - "preRow": "mapLoginIdentifiers" + "organization_id": null, + "type": null, + "language": null + } }, "urls": { "source": "cm_urls", "displayField": "id", "sqlSelect": "mveaSqlSelect", "fieldMap": { + "type_id": "&mapUrlType", "co_person_id": "person_id", "org_identity_id": "external_identity_id", - "type_id": "&mapUrlType", - "type": null, "co_department_id": null, - "organization_id": null + "organization_id": null, + "type": null, + "language": null } }, - "ad_hoc_attributes": { - "source": "cm_ad_hoc_attributes", + "addresses": { + "source": "cm_addresses", "displayField": "id", "sqlSelect": "mveaSqlSelect", "fieldMap": { + "type_id": "&mapAddressType", "co_person_role_id": "person_role_id", "org_identity_id": "external_identity_id", "co_department_id": null, - "organization_id": null - }, - "postTable": "migrateExtendedAttributesToAdHocAttributes" + "organization_id": null, + "type": null + } }, - "addresses": { - "source": "cm_addresses", + "telephone_numbers": { + "source": "cm_telephone_numbers", "displayField": "id", "sqlSelect": "mveaSqlSelect", "fieldMap": { + "type_id": "&mapTelephoneType", "co_person_role_id": "person_role_id", "org_identity_id": "external_identity_id", - "type_id": "&mapAddressType", - "type": null, "co_department_id": null, - "organization_id": null + "organization_id": null, + "type": null } }, - "telephone_numbers": { - "source": "cm_telephone_numbers", + "ad_hoc_attributes": { + "source": "cm_ad_hoc_attributes", "displayField": "id", "sqlSelect": "mveaSqlSelect", "fieldMap": { "co_person_role_id": "person_role_id", "org_identity_id": "external_identity_id", - "type_id": "&mapTelephoneType", - "type": null, "co_department_id": null, "organization_id": null - } + }, + "postTable": "migrateExtendedAttributesToAdHocAttributes" }, "history_records": { "source": "cm_history_records", @@ -320,7 +322,7 @@ "source": "cm_servers", "displayField": "description", "addChangelog": false, - "cache": ["co_id"], + "cache": ["co_id", "status"], "fieldMap": { "plugin": "&mapServerTypeToPlugin" } diff --git a/app/plugins/Transmogrify/src/Command/TransmogrifyCommand.php b/app/plugins/Transmogrify/src/Command/TransmogrifyCommand.php index 57df6ae87..eb0c16ab5 100644 --- a/app/plugins/Transmogrify/src/Command/TransmogrifyCommand.php +++ b/app/plugins/Transmogrify/src/Command/TransmogrifyCommand.php @@ -267,14 +267,21 @@ public function execute(Arguments $args, ConsoleIo $io): int } foreach(array_keys($this->tables) as $t) { + // Initializations per table migration $modeltableEmpty = true; $notSelected = false; - + $inboundQualifiedTableName = $this->inconn->qualifyTableName($this->tables[$t]['source']); + $outboundQualifiedTableName = $this->outconn->qualifyTableName($t); + $Model = TableRegistry::getTableLocator()->get($t); $io->info(message: sprintf("Transmogrifying table %s(%s)", Inflector::classify($t), $t)); - // Step 1: Check if source table exists and warn if not present + /* + * Run checks before processing the table + **/ + + // Check if source table exists and warn if not present if(!empty($this->tables[$t]['source'])) { $src = $this->tables[$t]['source']; if (!$this->tableExists($src)) { @@ -282,15 +289,33 @@ public function execute(Arguments $args, ConsoleIo $io): int continue; } } - // Step 2: Skip tables not in selected subset if specified - // We will print a warning and we will parse all the tables because we need to construct the cache. + + // Skip a table if already contains data + if($Model->find()->count() > 0) { + $modeltableEmpty = false; + $io->warning("Table (" . $t . ") is not empty. We will not overwrite existing data."); + } + + // Skip tables not in the selected subset if specified + // We will print a warning, and we will parse all the tables because we need to construct the cache. // Nevertheless, we will not allow any database processing - if (!empty($selected) && !in_array($t, $selected)) { + if ( + !empty($selected) + && !in_array($t, $selected) + ) { $notSelected = true; $io->warning("Skipping Transmogrification. Table ($t) is not in the selected subset."); } - // Step 3: Configure sequence ID for the target table + // Mark the table as skipped if it is not empty and not selected + $this->cache['skipInsert'][$outboundQualifiedTableName] = (!$modeltableEmpty && $notSelected); + + /* + * End of checks + */ + + + // Configure sequence ID for the target table if(!RawSqlQueries::setSequenceId( $this->inconn, $this->outconn, @@ -302,24 +327,18 @@ public function execute(Arguments $args, ConsoleIo $io): int return BaseCommand::CODE_ERROR; } - // Step 5: Execute any pre-processing hooks for the current table + // Execute any pre-processing hooks for the current table $this->runPreTableHook($t); - // Step 6: Skip if target table already contains data - $Model = TableRegistry::getTableLocator()->get($t); - if($Model->find()->count() > 0) { - $modeltableEmpty = false; - $io->warning("Table (" . $t . ") is not empty. We will not overwrite existing data."); - } - - // Step 7: Get total count of source records for progress tracking - $qualifiedTableName = $this->inconn->qualifyTableName($this->tables[$t]['source']); - // Step 8: Build and execute query to fetch all source records $insql = match(true) { - !empty($this->tables[$t]['sqlSelect']) => $this->runSqlSelectHook($t, $qualifiedTableName), - default => RawSqlQueries::buildSelectAllOrderedById($qualifiedTableName) + !empty($this->tables[$t]['sqlSelect']) => $this->runSqlSelectHook($t, $inboundQualifiedTableName), + default => RawSqlQueries::buildSelectAllOrderedById($inboundQualifiedTableName) }; + + // Verbose message to show the SQL query being executed + $io->verbose(sprintf('[Inbound SQL] Table=%s | %s', $t, $insql)); + // Fetch the inbound data. $stmt = $this->inconn->executeQuery($insql); $progress = new CommandLinePrinter($io, 'green', 50, true); @@ -328,7 +347,7 @@ public function execute(Arguments $args, ConsoleIo $io): int $countSql = RawSqlQueries::buildCountFromSelect($insql); $count = (int)$this->inconn->fetchOne($countSql); } else { - $count = (int)$this->inconn->fetchOne(RawSqlQueries::buildCountAll($qualifiedTableName)); + $count = (int)$this->inconn->fetchOne(RawSqlQueries::buildCountAll($inboundQualifiedTableName)); } $progress->start($count); $tally = 0; @@ -341,10 +360,12 @@ public function execute(Arguments $args, ConsoleIo $io): int } try { - // Step 1: Create a copy of the original row data to preserve it for post-processing + // Create a copy of the original row data to preserve it for post-processing $origRow = $row; - // Step 2: Execute any pre-processing hooks to transform or validate the row data + // Execute any pre-processing hooks to transform or validate the row data + // TODO: if i need to skip the insert i want something shared. I can add this in the cache + // and then skip insert from everywhere. $this->runPreRowHook($t, $origRow, $row); // Step 3: Set changelog defaults (created/modified timestamps, user IDs) @@ -355,24 +376,22 @@ public function execute(Arguments $args, ConsoleIo $io): int isset($this->tables[$t]['addChangelog']) && $this->tables[$t]['addChangelog'] ); - // Step 4: Convert boolean values to database-compatible format + // Convert boolean values to database-compatible format $this->normalizeBooleanFieldsForDb($t, $row); - // Step 5: Map old field names to new schema field names + // Map old field names to new schema field names $this->mapLegacyFieldNames($t, $row); - // Step 6: Insert the transformed row into the target database - $qualifiedTableName = $this->outconn->qualifyTableName($t); - - if($modeltableEmpty && !$notSelected) { - $fkQualifiedTableName = StringUtilities::classNameToForeignKey($qualifiedTableName); + // Insert the transformed row into the target database + if($this->cache['skipInsert'][$outboundQualifiedTableName] === false) { + $fkOutboundQualifiedTableName = StringUtilities::classNameToForeignKey($outboundQualifiedTableName); if ( isset($this->cache['rejected']) - && !empty($this->cache['rejected'][$qualifiedTableName][$row[$fkQualifiedTableName]]) + && !empty($this->cache['rejected'][$outboundQualifiedTableName][$row[$fkOutboundQualifiedTableName]]) ) { // This row will be rejected because it references a parent record that does not exist. // The parent record has been rejected before, so we can't insert this record. - $this->cache['rejected'][$qualifiedTableName][$row['id']] = $row; + $this->cache['rejected'][$outboundQualifiedTableName][$row['id']] = $row; $io->warning(sprintf( 'Skipping record %d in table %s - parent record does not exist', $row['id'] ?? 0, @@ -380,12 +399,12 @@ public function execute(Arguments $args, ConsoleIo $io): int )); continue; } - $this->outconn->insert($qualifiedTableName, $row); - // Step 8: Execute any post-processing hooks after successful insertion + $this->outconn->insert($outboundQualifiedTableName, $row); + // Execute any post-processing hooks after successful insertion $this->runPostRowHook($t, $origRow, $row); } - // Step 7: Store row data in cache for potential later use + // Store row data in cache for potential later use $this->cacheResults($t, $row); } catch(ForeignKeyConstraintViolationException $e) { // A foreign key associated with this record did not load, so we can't @@ -417,12 +436,13 @@ public function execute(Arguments $args, ConsoleIo $io): int } $progress->finish(); - // Step 10: Output final warning and error counts for the table + // Output final warning and error counts for the table $io->warning(sprintf('Warnings: %d', $warns)); $io->error(sprintf('Errors: %d', $err)); - // Step 11: Execute any post-processing hooks for the table - if ($modeltableEmpty && !$notSelected) { + // Execute any post-processing hooks for the table + if($this->cache['skipInsert'][$outboundQualifiedTableName] === false) { + $io->info('Running post-table hook for ' . $t); $this->runPostTableHook($t); } diff --git a/app/plugins/Transmogrify/src/Lib/Traits/CacheTrait.php b/app/plugins/Transmogrify/src/Lib/Traits/CacheTrait.php index 309649b18..cec7f19a5 100644 --- a/app/plugins/Transmogrify/src/Lib/Traits/CacheTrait.php +++ b/app/plugins/Transmogrify/src/Lib/Traits/CacheTrait.php @@ -104,11 +104,17 @@ protected function findCoId(array $row): int isset($row['group_id']) => $this->getCoIdFromGroupId((int)$row['group_id']), - // Legacy/preRow: org_identity_id follows the same External Identity path + isset($row['person_role_id']) => $this->getCoIdFromPersonRoleId((int)$row['person_role_id']), + + // Legacy/preRow isset($row['org_identity_id']) => $this->getCoIdFromExternalIdentityId((int)$row['org_identity_id']), isset($row['co_person_id']) => $this->getCoIdFromPersonId((int)$row['co_person_id']), + isset($row['co_group_id']) => $this->getCoIdFromGroupId((int)$row['co_group_id']), + + isset($row['co_person_role_id']) => $this->getCoIdFromPersonRoleId((int)$row['co_person_role_id']), + default => null, }; @@ -174,4 +180,21 @@ private function getCoIdFromExternalIdentityId(int $externalIdentityId): ?int $personId = $this->getPersonIdFromExternalIdentity($externalIdentityId); return $personId !== null ? $this->getCoIdFromPersonId($personId) : null; } + + + /** + * Resolve a CO ID from a Person Role ID via cache. + * + * @param int $personRoleId Person Role ID to lookup + * @return int|null CO ID if found, null otherwise + * @since COmanage Registry v5.2.0 + */ + private function getCoIdFromPersonRoleId(int $personRoleId): ?int + { + if (isset($this->cache['person_roles']['id'][$personRoleId]['person_id'])) { + $peronId = (int)$this->cache['person_roles']['id'][$personRoleId]['person_id']; + return $this->getCoIdFromPersonId($peronId); + } + return null; + } } \ No newline at end of file diff --git a/app/plugins/Transmogrify/src/Lib/Traits/HookRunnersTrait.php b/app/plugins/Transmogrify/src/Lib/Traits/HookRunnersTrait.php index 757e6bc5b..48a92b9e0 100644 --- a/app/plugins/Transmogrify/src/Lib/Traits/HookRunnersTrait.php +++ b/app/plugins/Transmogrify/src/Lib/Traits/HookRunnersTrait.php @@ -124,6 +124,34 @@ private function runSqlSelectHook(string $table, string $qualifiedTableName): st throw new \RuntimeException("Unknown sqlSelect hook: $method"); } $this->io->verbose('Running SQL select hook: ' . ucfirst(preg_replace('/([A-Z])/', ' $1', $method))); + + // Special handling for MVEA-style selects: derive FK columns from fieldMap keys dynamically + if ($method === 'mveaSqlSelect') { + // Known FK candidates we care about for MVEA presence checks + $fkCandidates = [ + 'co_person_id', + 'co_person_role_id', + 'org_identity_id', + 'co_group_id', + 'co_department_id', + 'co_provisioning_target_id', + 'organization_id' + ]; + $fieldMap = $this->tables[$table]['fieldMap'] ?? []; + $presentFks = array_values(array_intersect($fkCandidates, array_keys($fieldMap))); + + // Fallback to names-like FKs if nothing matched (defensive) + if (empty($presentFks)) { + $presentFks = ['co_person_id', 'org_identity_id']; + } + + return RawSqlQueries::mveaSqlSelect( + $qualifiedTableName, + $this->inconn->isMySQL(), + $presentFks + ); + } + return RawSqlQueries::{$method}($qualifiedTableName, $this->inconn->isMySQL()); } } \ No newline at end of file diff --git a/app/plugins/Transmogrify/src/Lib/Util/RawSqlQueries.php b/app/plugins/Transmogrify/src/Lib/Util/RawSqlQueries.php index 2ce1538d8..9d9b08db6 100644 --- a/app/plugins/Transmogrify/src/Lib/Util/RawSqlQueries.php +++ b/app/plugins/Transmogrify/src/Lib/Util/RawSqlQueries.php @@ -201,8 +201,87 @@ public static function roleSqlSelect(string $tableName, bool $isMySQL): string { * @return string SQL query string to select MVEA rows that are linked to valid org identities * @since COmanage Registry v5.2.0 */ - public static function mveaSqlSelect(string $tableName, bool $isMySQL): string { - return str_replace('{table}', $tableName, RawSqlQueries::MVEA_SQL_SELECT); + public static function mveaSqlSelect(string $tableName, bool $isMySQL, array $fkColumns = []): string { + // Defaults to co_person_id and org_identity_id + if (empty($fkColumns)) { + $fkColumns = ['co_person_id', 'org_identity_id']; + } + + // XXX Unsupported FKs for now (until their models are implemented) + // co_department_id, co_provisioning_target_id, organization_id + $unsupportedFks = [ + 'co_department_id', + 'co_provisioning_target_id', + 'organization_id' + ]; + + // In full mode, treat all as supported; otherwise split into supported/unsupported + if (false /* $fullMode */) { + $supportedInUse = array_values($fkColumns); + $unsupportedInUse = []; + } else { + // Split provided FKs into supported (for OR non-null) and unsupported (must be NULL) + $supportedInUse = array_values(array_diff($fkColumns, $unsupportedFks)); + $unsupportedInUse = array_values(array_intersect($fkColumns, $unsupportedFks)); + } + + // Require at least one SUPPORTED FK is not NULL (unsupported FKs are excluded from this OR) + $nonnullClauses = array_map( + fn(string $c) => 'n.' . $c . ' IS NOT NULL', + $supportedInUse + ); + // Keep SQL valid even if no supported FKs present + $nonnullAny = empty($nonnullClauses) ? '1=1' : '(' . implode(' OR ', $nonnullClauses) . ')'; + + // Unsupported FKs (that are present) must be NULL (AND clause) + $unsupportedNullClause = ''; + if (!empty($unsupportedInUse)) { + $unsupportedNullClause = 'AND ' . implode( + ' AND ', + array_map(fn(string $c) => "n.$c IS NULL", $unsupportedInUse) + ); + } + + // If org_identity_id is one of the FKs, apply the org identity validity EXISTS + $orgIdCheck = ''; + if (in_array('org_identity_id', $fkColumns, true)) { + // If we decide to enable the soft delete we need to take into account + // the type of database + // - (p.deleted IS NULL OR p.deleted = false) on PostgreSQL + // - (p.deleted IS NULL OR p.deleted = 0) on MySQL + $orgIdCheck = <<