Skip to content

Commit

Permalink
Fix mveas
Browse files Browse the repository at this point in the history
  • Loading branch information
@Ioannis
Ioannis committed Oct 18, 2025
1 parent babf88c commit 1667f7d
Show file tree
Hide file tree
Showing 5 changed files with 225 additions and 73 deletions.
64 changes: 33 additions & 31 deletions app/plugins/Transmogrify/config/schema/tables.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -112,7 +112,7 @@
"source": "cm_co_person_roles",
"sqlSelect": "roleSqlSelect",
"displayField": "id",
"cache": ["status"],
"cache": ["status", "person_id", "manager_person_id", "sponsor_person_id"],
"fieldMap": {
"co_person_id": "person_id",
"co_person_role_id": "person_role_id",
Expand Down Expand Up @@ -173,6 +173,7 @@
"source": "cm_co_group_members",
"displayField": "id",
"booleans": ["member", "owner"],
"preRow": "reconcileGroupMembershipOwnership",
"fieldMap": {
"co_group_id": "group_id",
"co_person_id": "person_id",
Expand All @@ -181,18 +182,17 @@
"co_group_nesting_id": "group_nesting_id",
"co_group_member_id": "group_member_id",
"source_org_identity_id": null
},
"preRow": "reconcileGroupMembershipOwnership"
}
},
"names": {
"source": "cm_names",
"displayField": "id",
"booleans": ["primary_name"],
"sqlSelect": "mveaSqlSelect",
"fieldMap": {
"type_id": "&mapNameType",
"co_person_id": "person_id",
"org_identity_id": "external_identity_id",
"type_id": "&mapNameType",
"type": null
}
},
Expand All @@ -202,81 +202,83 @@
"booleans": ["verified"],
"sqlSelect": "mveaSqlSelect",
"fieldMap": {
"type_id": "&mapEmailType",
"co_person_id": "person_id",
"org_identity_id": "external_identity_id",
"type_id": "&mapEmailType",
"type": null,
"co_department_id": null,
"organization_id": null
"organization_id": null,
"type": null
}
},
"identifiers": {
"source": "cm_identifiers",
"displayField": "id",
"booleans": ["login"],
"sqlSelect": "mveaSqlSelect",
"preRow": "mapLoginIdentifiers",
"fieldMap": {
"type_id": "&mapIdentifierType",
"co_group_id": "group_id",
"co_person_id": "person_id",
"org_identity_id": "external_identity_id",
"type_id": "&mapIdentifierType",
"type": null,
"co_department_id": null,
"co_provisioning_target_id": null,
"organization_id": null
},
"preRow": "mapLoginIdentifiers"
"organization_id": null,
"type": null,
"language": null
}
},
"urls": {
"source": "cm_urls",
"displayField": "id",
"sqlSelect": "mveaSqlSelect",
"fieldMap": {
"type_id": "&mapUrlType",
"co_person_id": "person_id",
"org_identity_id": "external_identity_id",
"type_id": "&mapUrlType",
"type": null,
"co_department_id": null,
"organization_id": null
"organization_id": null,
"type": null,
"language": null
}
},
"ad_hoc_attributes": {
"source": "cm_ad_hoc_attributes",
"addresses": {
"source": "cm_addresses",
"displayField": "id",
"sqlSelect": "mveaSqlSelect",
"fieldMap": {
"type_id": "&mapAddressType",
"co_person_role_id": "person_role_id",
"org_identity_id": "external_identity_id",
"co_department_id": null,
"organization_id": null
},
"postTable": "migrateExtendedAttributesToAdHocAttributes"
"organization_id": null,
"type": null
}
},
"addresses": {
"source": "cm_addresses",
"telephone_numbers": {
"source": "cm_telephone_numbers",
"displayField": "id",
"sqlSelect": "mveaSqlSelect",
"fieldMap": {
"type_id": "&mapTelephoneType",
"co_person_role_id": "person_role_id",
"org_identity_id": "external_identity_id",
"type_id": "&mapAddressType",
"type": null,
"co_department_id": null,
"organization_id": null
"organization_id": null,
"type": null
}
},
"telephone_numbers": {
"source": "cm_telephone_numbers",
"ad_hoc_attributes": {
"source": "cm_ad_hoc_attributes",
"displayField": "id",
"sqlSelect": "mveaSqlSelect",
"fieldMap": {
"co_person_role_id": "person_role_id",
"org_identity_id": "external_identity_id",
"type_id": "&mapTelephoneType",
"type": null,
"co_department_id": null,
"organization_id": null
}
},
"postTable": "migrateExtendedAttributesToAdHocAttributes"
},
"history_records": {
"source": "cm_history_records",
Expand Down Expand Up @@ -320,7 +322,7 @@
"source": "cm_servers",
"displayField": "description",
"addChangelog": false,
"cache": ["co_id"],
"cache": ["co_id", "status"],
"fieldMap": {
"plugin": "&mapServerTypeToPlugin"
}
Expand Down
94 changes: 57 additions & 37 deletions app/plugins/Transmogrify/src/Command/TransmogrifyCommand.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -267,30 +267,55 @@ public function execute(Arguments $args, ConsoleIo $io): int
}

foreach(array_keys($this->tables) as $t) {
// Initializations per table migration
$modeltableEmpty = true;
$notSelected = false;

$inboundQualifiedTableName = $this->inconn->qualifyTableName($this->tables[$t]['source']);
$outboundQualifiedTableName = $this->outconn->qualifyTableName($t);
$Model = TableRegistry::getTableLocator()->get($t);

$io->info(message: sprintf("Transmogrifying table %s(%s)", Inflector::classify($t), $t));


// Step 1: Check if source table exists and warn if not present
/*
* Run checks before processing the table
**/

// Check if source table exists and warn if not present
if(!empty($this->tables[$t]['source'])) {
$src = $this->tables[$t]['source'];
if (!$this->tableExists($src)) {
$this->io->warning("Source table '$src' does not exist in source database, skipping table '$t'");
continue;
}
}
// Step 2: Skip tables not in selected subset if specified
// We will print a warning and we will parse all the tables because we need to construct the cache.

// Skip a table if already contains data
if($Model->find()->count() > 0) {
$modeltableEmpty = false;
$io->warning("Table (" . $t . ") is not empty. We will not overwrite existing data.");
}

// Skip tables not in the selected subset if specified
// We will print a warning, and we will parse all the tables because we need to construct the cache.
// Nevertheless, we will not allow any database processing
if (!empty($selected) && !in_array($t, $selected)) {
if (
!empty($selected)
&& !in_array($t, $selected)
) {
$notSelected = true;
$io->warning("Skipping Transmogrification. Table ($t) is not in the selected subset.");
}

// Step 3: Configure sequence ID for the target table
// Mark the table as skipped if it is not empty and not selected
$this->cache['skipInsert'][$outboundQualifiedTableName] = (!$modeltableEmpty && $notSelected);

/*
* End of checks
*/


// Configure sequence ID for the target table
if(!RawSqlQueries::setSequenceId(
$this->inconn,
$this->outconn,
Expand All @@ -302,24 +327,18 @@ public function execute(Arguments $args, ConsoleIo $io): int
return BaseCommand::CODE_ERROR;
}

// Step 5: Execute any pre-processing hooks for the current table
// Execute any pre-processing hooks for the current table
$this->runPreTableHook($t);

// Step 6: Skip if target table already contains data
$Model = TableRegistry::getTableLocator()->get($t);
if($Model->find()->count() > 0) {
$modeltableEmpty = false;
$io->warning("Table (" . $t . ") is not empty. We will not overwrite existing data.");
}

// Step 7: Get total count of source records for progress tracking
$qualifiedTableName = $this->inconn->qualifyTableName($this->tables[$t]['source']);

// Step 8: Build and execute query to fetch all source records
$insql = match(true) {
!empty($this->tables[$t]['sqlSelect']) => $this->runSqlSelectHook($t, $qualifiedTableName),
default => RawSqlQueries::buildSelectAllOrderedById($qualifiedTableName)
!empty($this->tables[$t]['sqlSelect']) => $this->runSqlSelectHook($t, $inboundQualifiedTableName),
default => RawSqlQueries::buildSelectAllOrderedById($inboundQualifiedTableName)
};

// Verbose message to show the SQL query being executed
$io->verbose(sprintf('[Inbound SQL] Table=%s | %s', $t, $insql));
// Fetch the inbound data.
$stmt = $this->inconn->executeQuery($insql);

$progress = new CommandLinePrinter($io, 'green', 50, true);
Expand All @@ -328,7 +347,7 @@ public function execute(Arguments $args, ConsoleIo $io): int
$countSql = RawSqlQueries::buildCountFromSelect($insql);
$count = (int)$this->inconn->fetchOne($countSql);
} else {
$count = (int)$this->inconn->fetchOne(RawSqlQueries::buildCountAll($qualifiedTableName));
$count = (int)$this->inconn->fetchOne(RawSqlQueries::buildCountAll($inboundQualifiedTableName));
}
$progress->start($count);
$tally = 0;
Expand All @@ -341,10 +360,12 @@ public function execute(Arguments $args, ConsoleIo $io): int
}

try {
// Step 1: Create a copy of the original row data to preserve it for post-processing
// Create a copy of the original row data to preserve it for post-processing
$origRow = $row;

// Step 2: Execute any pre-processing hooks to transform or validate the row data
// Execute any pre-processing hooks to transform or validate the row data
// TODO: if i need to skip the insert i want something shared. I can add this in the cache
// and then skip insert from everywhere.
$this->runPreRowHook($t, $origRow, $row);

// Step 3: Set changelog defaults (created/modified timestamps, user IDs)
Expand All @@ -355,37 +376,35 @@ public function execute(Arguments $args, ConsoleIo $io): int
isset($this->tables[$t]['addChangelog']) && $this->tables[$t]['addChangelog']
);

// Step 4: Convert boolean values to database-compatible format
// Convert boolean values to database-compatible format
$this->normalizeBooleanFieldsForDb($t, $row);

// Step 5: Map old field names to new schema field names
// Map old field names to new schema field names
$this->mapLegacyFieldNames($t, $row);

// Step 6: Insert the transformed row into the target database
$qualifiedTableName = $this->outconn->qualifyTableName($t);

if($modeltableEmpty && !$notSelected) {
$fkQualifiedTableName = StringUtilities::classNameToForeignKey($qualifiedTableName);
// Insert the transformed row into the target database
if($this->cache['skipInsert'][$outboundQualifiedTableName] === false) {
$fkOutboundQualifiedTableName = StringUtilities::classNameToForeignKey($outboundQualifiedTableName);
if (
isset($this->cache['rejected'])
&& !empty($this->cache['rejected'][$qualifiedTableName][$row[$fkQualifiedTableName]])
&& !empty($this->cache['rejected'][$outboundQualifiedTableName][$row[$fkOutboundQualifiedTableName]])
) {
// This row will be rejected because it references a parent record that does not exist.
// The parent record has been rejected before, so we can't insert this record.
$this->cache['rejected'][$qualifiedTableName][$row['id']] = $row;
$this->cache['rejected'][$outboundQualifiedTableName][$row['id']] = $row;
$io->warning(sprintf(
'Skipping record %d in table %s - parent record does not exist',
$row['id'] ?? 0,
$t
));
continue;
}
$this->outconn->insert($qualifiedTableName, $row);
// Step 8: Execute any post-processing hooks after successful insertion
$this->outconn->insert($outboundQualifiedTableName, $row);
// Execute any post-processing hooks after successful insertion
$this->runPostRowHook($t, $origRow, $row);
}

// Step 7: Store row data in cache for potential later use
// Store row data in cache for potential later use
$this->cacheResults($t, $row);
} catch(ForeignKeyConstraintViolationException $e) {
// A foreign key associated with this record did not load, so we can't
Expand Down Expand Up @@ -417,12 +436,13 @@ public function execute(Arguments $args, ConsoleIo $io): int
}

$progress->finish();
// Step 10: Output final warning and error counts for the table
// Output final warning and error counts for the table
$io->warning(sprintf('Warnings: %d', $warns));
$io->error(sprintf('Errors: %d', $err));

// Step 11: Execute any post-processing hooks for the table
if ($modeltableEmpty && !$notSelected) {
// Execute any post-processing hooks for the table
if($this->cache['skipInsert'][$outboundQualifiedTableName] === false) {
$io->info('Running post-table hook for ' . $t);
$this->runPostTableHook($t);
}

Expand Down
25 changes: 24 additions & 1 deletion app/plugins/Transmogrify/src/Lib/Traits/CacheTrait.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -104,11 +104,17 @@ protected function findCoId(array $row): int

isset($row['group_id']) => $this->getCoIdFromGroupId((int)$row['group_id']),

// Legacy/preRow: org_identity_id follows the same External Identity path
isset($row['person_role_id']) => $this->getCoIdFromPersonRoleId((int)$row['person_role_id']),

// Legacy/preRow
isset($row['org_identity_id']) => $this->getCoIdFromExternalIdentityId((int)$row['org_identity_id']),

isset($row['co_person_id']) => $this->getCoIdFromPersonId((int)$row['co_person_id']),

isset($row['co_group_id']) => $this->getCoIdFromGroupId((int)$row['co_group_id']),

isset($row['co_person_role_id']) => $this->getCoIdFromPersonRoleId((int)$row['co_person_role_id']),

default => null,
};

Expand Down Expand Up @@ -174,4 +180,21 @@ private function getCoIdFromExternalIdentityId(int $externalIdentityId): ?int
$personId = $this->getPersonIdFromExternalIdentity($externalIdentityId);
return $personId !== null ? $this->getCoIdFromPersonId($personId) : null;
}


/**
* Resolve a CO ID from a Person Role ID via cache.
*
* @param int $personRoleId Person Role ID to lookup
* @return int|null CO ID if found, null otherwise
* @since COmanage Registry v5.2.0
*/
private function getCoIdFromPersonRoleId(int $personRoleId): ?int
{
if (isset($this->cache['person_roles']['id'][$personRoleId]['person_id'])) {
$peronId = (int)$this->cache['person_roles']['id'][$personRoleId]['person_id'];
return $this->getCoIdFromPersonId($peronId);
}
return null;
}
}
Loading

0 comments on commit 1667f7d

Please sign in to comment.