Skip to content

Commit

Permalink
Improvements.Fixed org identities transmogrify.
Browse files Browse the repository at this point in the history
  • Loading branch information
@Ioannis
Ioannis committed Nov 7, 2025
1 parent edc7b39 commit 67ab3e5
Show file tree
Hide file tree
Showing 7 changed files with 93 additions and 56 deletions.
16 changes: 6 additions & 10 deletions app/plugins/Transmogrify/config/schema/tables.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -128,6 +128,8 @@
"external_identities": {
"source": "cm_org_identities",
"displayField": "id",
"cache": ["person_id"],
"postRow": "mapExternalIdentityToExternalIdentityRole",
"fieldMap": {
"co_id": null,
"person_id": "&mapOrgIdentitycoPersonId",
Expand All @@ -140,9 +142,7 @@
"sponsor_identifier": null,
"valid_from": null,
"valid_through": null
},
"postRow": "mapExternalIdentityToExternalIdentityRole",
"cache": ["person_id"]
}
},
"groups": {
"source": "cm_co_groups",
Expand All @@ -152,7 +152,8 @@
"fieldMap": {
"auto": null,
"co_group_id": "group_id",
"group_type": "?S"
"group_type": "?S",
"introduction": null
},
"postTable": "createOwnersGroups"
},
Expand Down Expand Up @@ -310,12 +311,7 @@
"source": "cm_servers",
"displayField": "description",
"addChangelog": false,
"cache": ["status"],
"sqlSelect": null,
"preTable": null,
"postTable": null,
"preRow": null,
"postRow": null,
"cache": ["co_id"],
"fieldMap": {
"plugin": "&mapServerTypeToPlugin"
}
Expand Down
7 changes: 5 additions & 2 deletions app/plugins/Transmogrify/src/Command/TransmogrifyCommand.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -332,16 +332,19 @@ public function execute(Arguments $args, ConsoleIo $io): int
// not linked to a CO Person that was not migrated.
$warns++;
$progress->warn("Skipping $t record " . $row['id'] . " due to invalid foreign key: " . $e->getMessage());
$io->ask('Press <enter> to continue...');
}
catch(\InvalidArgumentException $e) {
// If we can't find a value for mapping we skip the record
// (ie: mapLegacyFieldNames basically requires a successful mapping)
$warns++;
$progress->warn("Skipping $t record " . $row['id'] . ": " . $e->getMessage());
$io->ask('Press <enter> to continue...');
}
catch(\Exception $e) {
$err++;
$progress->error("$t record " . $row['id'] . ": " . $e->getMessage());
$io->ask('Press <enter> to continue...');
}

$tally++;
Expand All @@ -357,7 +360,7 @@ public function execute(Arguments $args, ConsoleIo $io): int
$io->error(sprintf('Errors: %d', $err));

// Step 11: Execute any post-processing hooks for the table
if ($modeltableEmpty && $notSelected) {
if ($modeltableEmpty && !$notSelected) {
$this->runPostTableHook($t);
}

Expand All @@ -370,7 +373,7 @@ public function execute(Arguments $args, ConsoleIo $io): int
$io->info("This is the last table to process.");
}

$io->ask('Press <return> to continue...');
$io->ask('Press <enter> to continue...');
}

return BaseCommand::CODE_SUCCESS;
Expand Down
1 change: 1 addition & 0 deletions app/plugins/Transmogrify/src/Lib/Traits/CacheTrait.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ trait CacheTrait
* @var array
*/
protected array $cache = [];

/**
* Cache results as configured for the specified table.
*
Expand Down
6 changes: 6 additions & 0 deletions app/plugins/Transmogrify/src/Lib/Traits/HookRunnersTrait.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,8 @@ private function runPreTableHook(string $table): void {
if(!method_exists($this, $method)) {
throw new \RuntimeException("Unknown preTable hook: $method");
}

$this->io->info('Running pre-table hook: ' . ucfirst(preg_replace('/([A-Z])/', ' $1', $method)));
$this->{$method}();
}

Expand All @@ -60,6 +62,7 @@ private function runPostTableHook(string $table): void {
if(!method_exists($this, $method)) {
throw new \RuntimeException("Unknown postTable hook: $method");
}
$this->io->info('Running post-table hook: ' . ucfirst(preg_replace('/([A-Z])/', ' $1', $method)));
$this->{$method}();
}

Expand All @@ -79,6 +82,7 @@ private function runPreRowHook(string $table, array &$origRow, array &$row): voi
if(!method_exists($this, $method)) {
throw new \RuntimeException("Unknown preRow hook: $method");
}
$this->io->info('Running pre-row hook: ' . ucfirst(preg_replace('/([A-Z])/', ' $1', $method)));
$this->{$method}($origRow, $row);
}

Expand All @@ -98,6 +102,7 @@ private function runPostRowHook(string $table, array &$origRow, array &$row): vo
if(!method_exists($this, $method)) {
throw new \RuntimeException("Unknown postRow hook: $method");
}
$this->io->info('Running post-row hook: ' . ucfirst(preg_replace('/([A-Z])/', ' $1', $method)));
$this->{$method}($origRow, $row);
}

Expand All @@ -118,6 +123,7 @@ private function runSqlSelectHook(string $table, string $qualifiedTableName): st
if(!method_exists(RawSqlQueries::class, $method)) {
throw new \RuntimeException("Unknown sqlSelect hook: $method");
}
$this->io->info('Running SQL select hook: ' . ucfirst(preg_replace('/([A-Z])/', ' $1', $method)));
return RawSqlQueries::{$method}($qualifiedTableName, $this->inconn->isMySQL());
}
}
45 changes: 38 additions & 7 deletions app/plugins/Transmogrify/src/Lib/Traits/RowTransformationTrait.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -298,12 +298,37 @@ protected function mapExternalIdentityToExternalIdentityRole(array $origRow, arr
$roleRow[$newKey] = $origRow[$oldKey];
}

if(!empty($origRow['affiliation'])) {
$row['affiliation'] = $origRow['affiliation'];
$roleRow['affiliation_type_id'] = $this->mapAffiliationType(
row: $row,
coId: $origRow['co_id'] ?? null
);
// Rationale: mapOrgIdentitycoPersonId accepts only the first mapping it finds
// (later mappings are treated as legacy/unpooled anomalies and ignored).
// Therefore, each ExternalIdentity produces at most one ExternalIdentityRole.
// To avoid inserting a bogus self-referential changelog link, do not carry any
// legacy key into external_identity_role_id. Start a fresh changelog chain.
$roleRow['external_identity_role_id'] = null;
$roleRow['revision'] = 0;


try {
if (!empty($origRow['affiliation'])) {
$row['affiliation'] = $origRow['affiliation'];
$roleRow['affiliation_type_id'] = $this->mapAffiliationType(
row: $row,
coId: $origRow['co_id'] ?? null
);
}
} catch (\Exception $e) {
$this->io->warning("Failed to map affiliation type: " . $e->getMessage());
if (
isset($origRow['co_id'])
&& (!isset($this->cache['cos'][$origRow['co_id']])
|| ($this->cache['cos'][$origRow['co_id']]['status'])
&& $this->cache['cos'][$origRow['co_id']]['status'] == 'TR')
) {
// This CO has been deleted, so we can't map the type. We will return null
$roleRow['affiliation_type_id'] = null;
} else {
// Rethrow the exception
throw $e;
}
}

$tableName = 'external_identity_roles';
Expand All @@ -313,7 +338,13 @@ protected function mapExternalIdentityToExternalIdentityRole(array $origRow, arr
$this->normalizeBooleanFieldsForDb($tableName, $roleRow);

$qualifiedTableName = $this->outconn->qualifyTableName($tableName);
$this->outconn->insert($qualifiedTableName, $roleRow);


try {
$this->outconn->insert($qualifiedTableName, $roleRow);
} catch (\Exception $e) {
$this->io->warning("record already exists: " . print_r($roleRow, true));
}
}

/**
Expand Down
49 changes: 24 additions & 25 deletions app/plugins/Transmogrify/src/Lib/Traits/TypeMapperTrait.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@

namespace Transmogrify\Lib\Traits;

use App\Lib\Enum\StatusEnum;
use Doctrine\DBAL\Exception;
use Transmogrify\Lib\Util\RawSqlQueries;
use Cake\Utility\Inflector;
use Transmogrify\Lib\Traits\CacheTrait;
Expand Down Expand Up @@ -176,9 +176,10 @@ protected function mapNow(array $row) {
/**
* Map an Org Identity ID to a CO Person ID
*
* @since COmanage Registry v5.0.0
* @param array $row Row of Org Identity table data
* @param array $row Row of Org Identity table data
* @return int|null CO Person ID
* @throws Exception
* @since COmanage Registry v5.0.0
*/
protected function mapOrgIdentitycoPersonId(array $row): ?int
{
Expand All @@ -200,7 +201,7 @@ protected function mapOrgIdentitycoPersonId(array $row): ?int
// do that.) Historical information remains available in history_records,
// and if the deployer keeps an archive of the old database.

$oid = (int)$row['id'];
$rowId = (int)$row['id'];

if(empty($this->cache['org_identities']['co_people'])) {
$this->cache['org_identities']['co_people'] = [];
Expand All @@ -213,42 +214,40 @@ protected function mapOrgIdentitycoPersonId(array $row): ?int
$qualifiedTableName = $this->inconn->qualifyTableName($tableName);

// Only fetch current rows (historical/deleted rows are filtered out)
$mapsql = RawSqlQueries::buildSelectAll(
$mapsql = RawSqlQueries::buildSelectAllWithNoChangelong(
qualifiedTableName: $qualifiedTableName,
onlyCurrent: true,
changelogFK: $changelogFK
);

$stmt = $this->inconn->executeQuery($mapsql);

while ($r = $stmt->fetchAssociative()) {
if (!empty($r['org_identity_id'])) {
$rOid = (int)$r['org_identity_id'];
$rOrevision = (int)$r['revision'];
$cop = isset($r['co_person_id']) ? (int)$r['co_person_id'] : null;

if ($cop === null) {
$this->io->warning('Org Identity ' . $rOid . ' has no mapped CO Person ID, skipping');
continue;
}
while($r = $stmt->fetchAssociative()) {
$oid = $r['org_identity_id'] ?? null;

if (isset($this->cache['org_identities']['co_people'][$rOid][$rOrevision])) {
if(!empty($oid)) {
$rowRev = $r['revision'];
if(isset($this->cache['org_identities']['co_people'][ $oid ][ $rowRev ])) {
// If for some reason we already have a record, it's probably due to
// improper unpooling from a legacy deployment. We'll accept only the
// first record and throw warnings on the others.
$this->io->verbose('Found duplicate current CO Person mapping for Org Identity ' . $rOid . ', skipping');
continue;
}

$this->cache['org_identities']['co_people'][$rOid][$rOrevision] = $cop;
$this->io->verbose("Found existing CO Person for Org Identity " . $oid . ", skipping");
} else {
$this->cache['org_identities']['co_people'][ $oid ][ $rowRev ] = $r['co_person_id'];
}
}
}

// Sort the array
sort($this->cache['org_identities']['co_people']);
}

if (!empty($this->cache['org_identities']['co_people'][$oid])) {
if(!empty($this->cache['org_identities']['co_people'][ $rowId ])) {
// XXX OrgIdentities with no org identity link are not supported in v5
// Return the record with the highest revision number
$rev = max(array_keys($this->cache['org_identities']['co_people'][ $oid ]));
return $this->cache['org_identities']['co_people'][$oid][$rev];
$rev = max(array_keys($this->cache['org_identities']['co_people'][ $rowId ]));

return $this->cache['org_identities']['co_people'][ $rowId ][$rev];
}

return null;
Expand Down Expand Up @@ -302,7 +301,7 @@ protected function mapType(array $row, string $type, int $coId, string $attr = '
if(empty($this->cache['types']['co_id+attribute+value+'][$key])) {
if (
!isset($this->cache['cos'][$coId])
|| ($this->cache['cos'][$coId]['status'] && $this->cache['cos'][$coId]['status'] == 'TR')
|| ($this->cache['cos'][$coId]['status'] && in_array($this->cache['cos'][$coId]['status'], ['TR']))
) {
// This CO has been deleted, so we can't map the type. We will return null
return null;
Expand Down
25 changes: 13 additions & 12 deletions app/plugins/Transmogrify/src/Lib/Util/RawSqlQueries.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,22 +69,23 @@ public static function buildSelectAllOrderedById(string $qualifiedTableName): st
/**
* Builds SQL query to select all rows, optionally filtering changelog records
* @param string $qualifiedTableName Fully qualified table name
* @param bool $onlyCurrent Whether to include changelog records (default true)
* @param string|null $changelogFK Name of the changelog foreign key column
* @return string SQL query string
*/
public static function buildSelectAll(
public static function buildSelectAll(string $qualifiedTableName): string {
return "SELECT * FROM $qualifiedTableName";
}

/**
* Builds SQL query to select all rows, filtering changelog records
* @param string $qualifiedTableName Fully qualified table name
* @param string $changelogFK Changelog Foreign Key
* @return string SQL query string
*/
public static function buildSelectAllWithNoChangelong(
string $qualifiedTableName,
bool $onlyCurrent = false,
string $changelogFK = null
string $changelogFK
): string {
if ($onlyCurrent) {
if (empty($changelogFK)) {
throw new \InvalidArgumentException('changelogFK is required when onlyCurrent is true');
}
return "SELECT * FROM $qualifiedTableName WHERE $changelogFK IS NULL";
}
return "SELECT * FROM $qualifiedTableName";
return "SELECT * FROM $qualifiedTableName WHERE $changelogFK IS NULL";
}

/**
Expand Down

0 comments on commit 67ab3e5

Please sign in to comment.