diff --git a/app/plugins/Transmogrify/README.md b/app/plugins/Transmogrify/README.md index 05c513bb5..5a3163fbf 100644 --- a/app/plugins/Transmogrify/README.md +++ b/app/plugins/Transmogrify/README.md @@ -60,6 +60,9 @@ These options come directly from TransmogrifyCommand::buildOptionParser. - --login-identifier-type TYPE - Identifier type value to use for login identifiers when --login-identifier-copy is set. +- --orgidentities-health + - Run Org Identities health check (eligibility/exclusion breakdown based on non-historical links and person existence) and print a transmogrification readiness report, then exit. + ## Typical usage diff --git a/app/plugins/Transmogrify/config/schema/tables.json b/app/plugins/Transmogrify/config/schema/tables.json index 4e84bef37..8e732ec5c 100644 --- a/app/plugins/Transmogrify/config/schema/tables.json +++ b/app/plugins/Transmogrify/config/schema/tables.json @@ -129,6 +129,7 @@ "source": "cm_org_identities", "displayField": "id", "cache": ["person_id"], + "sqlSelect": "orgidentitiesSqlSelect", "postRow": "mapExternalIdentityToExternalIdentityRole", "fieldMap": { "co_id": null, diff --git a/app/plugins/Transmogrify/src/Command/TransmogrifyCommand.php b/app/plugins/Transmogrify/src/Command/TransmogrifyCommand.php index 0793576ea..ff3ae1535 100644 --- a/app/plugins/Transmogrify/src/Command/TransmogrifyCommand.php +++ b/app/plugins/Transmogrify/src/Command/TransmogrifyCommand.php @@ -51,6 +51,7 @@ use Transmogrify\Lib\Util\CommandLinePrinter; use Transmogrify\Lib\Util\DbInfoPrinter; use Transmogrify\Lib\Util\RawSqlQueries; +use Transmogrify\Lib\Util\OrgIdentitiesHealth; class TransmogrifyCommand extends BaseCommand { use CacheTrait; @@ -166,6 +167,12 @@ protected function buildOptionParser(ConsoleOptionParser $parser): ConsoleOption 'help' => __d('command', 'tm.login-identifier-type') ]); + // Health report option (Org Identities readiness) + $parser->addOption('orgidentities-health', [ + 'help' => 'Run Org Identities health check (eligibility/exclusion breakdown) and exit', + 'boolean' => true + ]); + $parser->setEpilog(__d('command', 'tm.epilog')); return $parser; @@ -197,6 +204,12 @@ public function execute(Arguments $args, ConsoleIo $io): int return $code; } + // Health report: run and exit + if ($this->args->getOption('orgidentities-health')) { + OrgIdentitiesHealth::run($this->inconn, $this->io); + return BaseCommand::CODE_SUCCESS; + } + // Load tables configuration (from JSON) and extend it with schema data $this->loadTablesConfig(); @@ -273,7 +286,6 @@ public function execute(Arguments $args, ConsoleIo $io): int // Step 7: Get total count of source records for progress tracking $qualifiedTableName = $this->inconn->qualifyTableName($this->tables[$t]['source']); - $count = $this->inconn->fetchOne(RawSqlQueries::buildCountAll($qualifiedTableName)); // Step 8: Build and execute query to fetch all source records $insql = match(true) { @@ -283,6 +295,13 @@ public function execute(Arguments $args, ConsoleIo $io): int $stmt = $this->inconn->executeQuery($insql); $progress = new CommandLinePrinter($io, 'green', 50, true); + // If a custom SELECT is used, count the exact result set; otherwise count the whole table + if (!empty($this->tables[$t]['sqlSelect'])) { + $countSql = RawSqlQueries::buildCountFromSelect($insql); + $count = (int)$this->inconn->fetchOne($countSql); + } else { + $count = (int)$this->inconn->fetchOne(RawSqlQueries::buildCountAll($qualifiedTableName)); + } $progress->start($count); $tally = 0; $warns = 0; diff --git a/app/plugins/Transmogrify/src/Lib/Traits/RowTransformationTrait.php b/app/plugins/Transmogrify/src/Lib/Traits/RowTransformationTrait.php index 35cb34b9a..104376f3b 100644 --- a/app/plugins/Transmogrify/src/Lib/Traits/RowTransformationTrait.php +++ b/app/plugins/Transmogrify/src/Lib/Traits/RowTransformationTrait.php @@ -339,12 +339,7 @@ protected function mapExternalIdentityToExternalIdentityRole(array $origRow, arr $qualifiedTableName = $this->outconn->qualifyTableName($tableName); - - try { - $this->outconn->insert($qualifiedTableName, $roleRow); - } catch (\Exception $e) { - $this->io->warning("record already exists: " . print_r($roleRow, true)); - } + $this->outconn->insert($qualifiedTableName, $roleRow); } /** diff --git a/app/plugins/Transmogrify/src/Lib/Traits/TypeMapperTrait.php b/app/plugins/Transmogrify/src/Lib/Traits/TypeMapperTrait.php index c3a706411..b33945b55 100644 --- a/app/plugins/Transmogrify/src/Lib/Traits/TypeMapperTrait.php +++ b/app/plugins/Transmogrify/src/Lib/Traits/TypeMapperTrait.php @@ -203,7 +203,7 @@ protected function mapOrgIdentitycoPersonId(array $row): ?int $rowId = (int)$row['id']; - if(empty($this->cache['org_identities']['co_people'])) { + if (empty($this->cache['org_identities']['co_people'])) { $this->cache['org_identities']['co_people'] = []; // Build cache on first use @@ -213,7 +213,7 @@ protected function mapOrgIdentitycoPersonId(array $row): ?int $changelogFK = 'co_org_identity_link_id'; $qualifiedTableName = $this->inconn->qualifyTableName($tableName); - // Only fetch current rows (historical/deleted rows are filtered out) + // Only fetch current rows (historical/changelog rows are filtered out) $mapsql = RawSqlQueries::buildSelectAllWithNoChangelong( qualifiedTableName: $qualifiedTableName, changelogFK: $changelogFK @@ -221,7 +221,7 @@ protected function mapOrgIdentitycoPersonId(array $row): ?int $stmt = $this->inconn->executeQuery($mapsql); - while($r = $stmt->fetchAssociative()) { + while ($r = $stmt->fetchAssociative()) { $oid = $r['org_identity_id'] ?? null; if(!empty($oid)) { @@ -233,23 +233,31 @@ protected function mapOrgIdentitycoPersonId(array $row): ?int $this->io->verbose("Found existing CO Person for Org Identity " . $oid . ", skipping"); } else { - $this->cache['org_identities']['co_people'][ $oid ][ $rowRev ] = $r['co_person_id']; + // Store as-is; we'll resolve the latest revision on lookup + $this->cache['org_identities']['co_people'][ $oid ][ $rowRev ] = (int)$r['co_person_id']; } } } - // Sort the array - sort($this->cache['org_identities']['co_people']); + // Preserve keys while providing the deterministic order of revisions + foreach ($this->cache['org_identities']['co_people'] as &$revisions) { + if (is_array($revisions)) { + ksort($revisions, SORT_NUMERIC); + } + } + unset($revisions); } - if(!empty($this->cache['org_identities']['co_people'][ $rowId ])) { + if (!empty($this->cache['org_identities']['co_people'][$rowId])) { // XXX OrgIdentities with no org identity link are not supported in v5 // Return the record with the highest revision number - $rev = max(array_keys($this->cache['org_identities']['co_people'][ $rowId ])); + $revisions = $this->cache['org_identities']['co_people'][$rowId]; + $rev = max(array_keys($revisions)); - return $this->cache['org_identities']['co_people'][ $rowId ][$rev]; + return (int)$revisions[$rev]; } + // No current mapping found for this Org Identity return null; } diff --git a/app/plugins/Transmogrify/src/Lib/Util/OrgIdentitiesHealth.php b/app/plugins/Transmogrify/src/Lib/Util/OrgIdentitiesHealth.php new file mode 100644 index 000000000..b585c071f --- /dev/null +++ b/app/plugins/Transmogrify/src/Lib/Util/OrgIdentitiesHealth.php @@ -0,0 +1,141 @@ +out('Running Org Identities health check...'); + $sql = RawSqlQueries::ORGIDENTITIES_HEALTH_SQL_QUERY; + + try { + $rows = $inconn->fetchAllAssociative($sql); + } catch (\Throwable $e) { + $io->err('Org Identities health check failed: ' . $e->getMessage()); + return; + } + + if (empty($rows)) { + $io->out('No results.'); + return; + } + + // Detect available columns + $first = $rows[0]; + $hasIncluded = array_key_exists('included_count', $first); + $hasExcluded = array_key_exists('excluded_count', $first); + $hasIndicator = array_key_exists('indicator', $first); + $hasCount = array_key_exists('count', $first); + + // Prepare headers based on detected columns + if ($hasIncluded || $hasExcluded) { + $headers = ['Reason', 'Included', 'Excluded']; + if ($hasIndicator) { + $headers[] = 'Indicator'; + } + } else { + // Fallback to simple reason + count (and indicator if present) + $headers = ['Reason', 'Count']; + if ($hasIndicator) { + $headers[] = 'Indicator'; + } + } + + // Compute column widths + $widths = array_fill(0, count($headers), 0); + $reasonIdx = 0; + $incIdx = array_search('Included', $headers, true); + $excIdx = array_search('Excluded', $headers, true); + $cntIdx = array_search('Count', $headers, true); + $indIdx = array_search('Indicator', $headers, true); + + // Initialize with header widths + foreach ($headers as $i => $h) { + $widths[$i] = max($widths[$i], mb_strlen($h)); + } + + // Measure data + foreach ($rows as $r) { + $reasonLen = mb_strlen((string)($r['reason'] ?? '')); + $widths[$reasonIdx] = max($widths[$reasonIdx], $reasonLen); + + if ($incIdx !== false) { + $widths[$incIdx] = max($widths[$incIdx], mb_strlen((string)($r['included_count'] ?? ''))); + } + if ($excIdx !== false) { + $widths[$excIdx] = max($widths[$excIdx], mb_strlen((string)($r['excluded_count'] ?? ''))); + } + if ($cntIdx !== false) { + $widths[$cntIdx] = max($widths[$cntIdx], mb_strlen((string)($r['count'] ?? ''))); + } + if ($indIdx !== false) { + $widths[$indIdx] = max($widths[$indIdx], mb_strlen((string)($r['indicator'] ?? ''))); + } + } + + // Helper to pad a cell + $pad = static function (string $s, int $w): string { + $len = mb_strlen($s); + if ($len >= $w) { + return $s; + } + return $s . str_repeat(' ', $w - $len); + }; + + // Print header + $lineParts = []; + foreach ($headers as $i => $h) { + $lineParts[] = $pad($h, $widths[$i]); + } + $io->out(implode(' | ', $lineParts)); + + // Print separator + $sepParts = array_map(static fn($w) => str_repeat('-', $w), $widths); + $io->out(implode('--+--', $sepParts)); + + // Print rows + foreach ($rows as $r) { + $rowParts = []; + $rowParts[] = $pad((string)($r['reason'] ?? ''), $widths[$reasonIdx]); + + if ($incIdx !== false) { + $rowParts[] = $pad((string)($r['included_count'] ?? ''), $widths[$incIdx]); + } + if ($excIdx !== false) { + $rowParts[] = $pad((string)($r['excluded_count'] ?? ''), $widths[$excIdx]); + } + if ($cntIdx !== false) { + $rowParts[] = $pad((string)($r['count'] ?? ''), $widths[$cntIdx]); + } + if ($indIdx !== false) { + $indRaw = (string)($r['indicator'] ?? ''); + $cell = $pad($indRaw, $widths[$indIdx]); + + // Colorize first visible char, leave padding spaces uncolored to preserve alignment + if ($indRaw === 'x' || $indRaw === '✓') { + $color = ($indRaw === 'x') ? "\033[31m" : "\033[32m"; // red for x, green for ✓ + $reset = "\033[0m"; + $first = mb_substr($cell, 0, 1); + $rest = mb_substr($cell, 1); + $cell = $color . $first . $reset . $rest; + } + + $rowParts[] = $cell; + } + + $io->out(implode(' | ', $rowParts)); + } + } +} diff --git a/app/plugins/Transmogrify/src/Lib/Util/RawSqlQueries.php b/app/plugins/Transmogrify/src/Lib/Util/RawSqlQueries.php index 2c622ce5d..7eccfec7e 100644 --- a/app/plugins/Transmogrify/src/Lib/Util/RawSqlQueries.php +++ b/app/plugins/Transmogrify/src/Lib/Util/RawSqlQueries.php @@ -42,6 +42,7 @@ class RawSqlQueries { * Builds SQL query to get maximum ID from a table * @param string $qualifiedTableName Fully qualified table name * @return string SQL query string + * @since COmanage Registry v5.2.0 */ public static function buildSelectMaxId(string $qualifiedTableName): string { return 'SELECT MAX(id) FROM ' . $qualifiedTableName; @@ -51,15 +52,33 @@ public static function buildSelectMaxId(string $qualifiedTableName): string { * Builds SQL query to count all rows in a table * @param string $qualifiedTableName Fully qualified table name * @return string SQL query string + * @since COmanage Registry v5.2.0 */ public static function buildCountAll(string $qualifiedTableName): string { return 'SELECT COUNT(*) FROM ' . $qualifiedTableName; } + /** + * Build a portable COUNT(*) wrapper around an arbitrary SELECT statement. + * Strips a trailing ORDER BY to satisfy engines that disallow ORDER BY in subqueries. + * + * @param string $selectSql Arbitrary SELECT SQL + * @return string SQL that returns a single COUNT(*) + * @since COmanage Registry v5.2.0 + */ + public static function buildCountFromSelect(string $selectSql): string { + // Remove trailing ORDER BY ... (simple heuristic, works for our generated queries) + $sql = preg_replace('/\s+ORDER\s+BY\s+[\s\S]*$/i', '', $selectSql); + + return "SELECT COUNT(*) FROM ($sql) subq"; + } + + /** * Builds SQL query to select all rows ordered by ID ascending * @param string $qualifiedTableName Fully qualified table name * @return string SQL query string + * @since COmanage Registry v5.2.0 */ public static function buildSelectAllOrderedById(string $qualifiedTableName): string { return 'SELECT * FROM ' . $qualifiedTableName . ' ORDER BY id ASC'; @@ -70,6 +89,7 @@ public static function buildSelectAllOrderedById(string $qualifiedTableName): st * Builds SQL query to select all rows, optionally filtering changelog records * @param string $qualifiedTableName Fully qualified table name * @return string SQL query string + * @since COmanage Registry v5.2.0 */ public static function buildSelectAll(string $qualifiedTableName): string { return "SELECT * FROM $qualifiedTableName"; @@ -80,6 +100,7 @@ public static function buildSelectAll(string $qualifiedTableName): string { * @param string $qualifiedTableName Fully qualified table name * @param string $changelogFK Changelog Foreign Key * @return string SQL query string + * @since COmanage Registry v5.2.0 */ public static function buildSelectAllWithNoChangelong( string $qualifiedTableName, @@ -94,6 +115,7 @@ public static function buildSelectAllWithNoChangelong( * @param int $nextId Next ID value to set * @param bool $isMySQL Whether target database is MySQL * @return string SQL query string + * @since COmanage Registry v5.2.0 */ public static function buildSequenceReset(string $qualifiedTableName, int $nextId, bool $isMySQL): string { if($isMySQL) { @@ -173,6 +195,19 @@ public static function roleSqlSelect(string $tableName, bool $isMySQL): string { return RawSqlQueries::ROLE_SQL_SELECT; } + + /** + * Return SQL used to select Organization Identities from inbound database. + * + * @param string $tableName Name of the SQL table + * @param bool $isMySQL Whether the database is MySQL + * @return string SQL string to select rows from inbound database + * @since COmanage Registry v5.2.0 + */ + public static function orgidentitiesSqlSelect(string $tableName, bool $isMySQL): string { + return RawSqlQueries::ORG_IDENTITIES_SQL_SELECT; + } + // Any COU at any time can be made the child of another COU // and so during transmogrification we cannot simply select // the rows of the COU table by ascending id because it leads @@ -328,10 +363,32 @@ public static function roleSqlSelect(string $tableName, bool $isMySQL): string { ORDER BY MAX(generation_number) ASC; SQL; + /** + * SQL template for selecting organization identities that have at least one org identity link + */ + final const ORG_IDENTITIES_SQL_SELECT = <<