From c47ae95b37995cd5ce4654f179bdf658866f657e Mon Sep 17 00:00:00 2001 From: Ioannis Igoumenos Date: Thu, 16 Oct 2025 10:31:37 +0300 Subject: [PATCH] Job history records+servers --- app/plugins/Transmogrify/README.md | 5 +- .../Transmogrify/config/schema/tables.json | 8 +- .../src/Command/TransmogrifyCommand.php | 111 +++++++++++++++--- .../src/Lib/Traits/CacheTrait.php | 12 +- .../src/Lib/Util/CommandLinePrinter.php | 49 ++++++++ .../src/Lib/Util/RawSqlQueries.php | 95 ++++++++------- 6 files changed, 215 insertions(+), 65 deletions(-) diff --git a/app/plugins/Transmogrify/README.md b/app/plugins/Transmogrify/README.md index 0826dad97..2cceaee86 100644 --- a/app/plugins/Transmogrify/README.md +++ b/app/plugins/Transmogrify/README.md @@ -13,7 +13,10 @@ It is designed to be: - Source DB (used by Transmogrify) and target DB must both be reachable. Transmogrify initializes two Doctrine DBAL connections internally. - The default tables mapping file is at: - app/plugins/Transmogrify/config/schema/tables.json - +- Actions + - Finalize any pending Jobs. Jobs in Queued or Progress state will be skipped. + - Restore extended type defaults + - Run the health checks. ## Command Invoke from your app root: diff --git a/app/plugins/Transmogrify/config/schema/tables.json b/app/plugins/Transmogrify/config/schema/tables.json index ce2cc0cab..24e2d832d 100644 --- a/app/plugins/Transmogrify/config/schema/tables.json +++ b/app/plugins/Transmogrify/config/schema/tables.json @@ -80,7 +80,8 @@ "person_picker_email_type": null, "person_picker_identifier_type": null, "person_picker_display_types": null, - "group_create_admin_only": null + "group_create_admin_only": null, + "t_and_c_return_url_allowlist": null } }, "authentication_events": { @@ -283,6 +284,7 @@ "history_records": { "source": "cm_history_records", "displayField": "id", + "sqlSelect": "historyRecordsSqlSelect", "fieldMap": { "actor_co_person_id": "actor_person_id", "co_person_id": "person_id", @@ -312,6 +314,7 @@ "job_history_records": { "source": "cm_co_job_history_records", "displayField": "id", + "sqlSelect": "jobHistoryRecordsSqlSelect", "fieldMap": { "co_job_id": "job_id", "co_person_id": "person_id", @@ -324,7 +327,8 @@ "addChangelog": false, "cache": ["co_id", "status"], "fieldMap": { - "plugin": "&mapServerTypeToPlugin" + "plugin": "&mapServerTypeToPlugin", + "server_type": null } } } diff --git a/app/plugins/Transmogrify/src/Command/TransmogrifyCommand.php b/app/plugins/Transmogrify/src/Command/TransmogrifyCommand.php index eb0c16ab5..e903e61e9 100644 --- a/app/plugins/Transmogrify/src/Command/TransmogrifyCommand.php +++ b/app/plugins/Transmogrify/src/Command/TransmogrifyCommand.php @@ -341,6 +341,9 @@ public function execute(Arguments $args, ConsoleIo $io): int // Fetch the inbound data. $stmt = $this->inconn->executeQuery($insql); + /* + * PROGRESS STARTING + **/ $progress = new CommandLinePrinter($io, 'green', 50, true); // If a custom SELECT is used, count the exact result set; otherwise count the whole table if (!empty($this->tables[$t]['sqlSelect'])) { @@ -384,21 +387,11 @@ public function execute(Arguments $args, ConsoleIo $io): int // Insert the transformed row into the target database if($this->cache['skipInsert'][$outboundQualifiedTableName] === false) { - $fkOutboundQualifiedTableName = StringUtilities::classNameToForeignKey($outboundQualifiedTableName); - if ( - isset($this->cache['rejected']) - && !empty($this->cache['rejected'][$outboundQualifiedTableName][$row[$fkOutboundQualifiedTableName]]) - ) { - // This row will be rejected because it references a parent record that does not exist. - // The parent record has been rejected before, so we can't insert this record. - $this->cache['rejected'][$outboundQualifiedTableName][$row['id']] = $row; - $io->warning(sprintf( - 'Skipping record %d in table %s - parent record does not exist', - $row['id'] ?? 0, - $t - )); + // Check if a parent record for this row was previously rejected; if so, skip this insert + if ($this->skipIfRejectedParent(currentTable: $t, row: $row, progress: $progress)) { continue; } + $this->outconn->insert($outboundQualifiedTableName, $row); // Execute any post-processing hooks after successful insertion $this->runPostRowHook($t, $origRow, $row); @@ -412,6 +405,7 @@ public function execute(Arguments $args, ConsoleIo $io): int // did not load, perhaps because it was associated with an Org Identity // not linked to a CO Person that was not migrated. $warns++; + $this->cache['rejected'][$outboundQualifiedTableName][$row['id']] = $row; $progress->warn("Skipping $t record " . $row['id'] . " due to invalid foreign key: " . $e->getMessage()); $io->ask('Press to continue...'); } @@ -419,13 +413,14 @@ public function execute(Arguments $args, ConsoleIo $io): int // If we can't find a value for mapping we skip the record // (ie: mapLegacyFieldNames basically requires a successful mapping) $warns++; + $this->cache['rejected'][$outboundQualifiedTableName][$row['id']] = $row; $progress->warn("Skipping $t record " . $row['id'] . ": " . $e->getMessage()); $io->ask('Press to continue...'); } catch(\Exception $e) { $err++; $progress->error("$t record " . $row['id'] . ": " . $e->getMessage()); - $io->ask('Press to continue...'); + $progress->ask('Press to continue...'); } $tally++; @@ -436,6 +431,10 @@ public function execute(Arguments $args, ConsoleIo $io): int } $progress->finish(); + /** + * FINISH PROGRESS + */ + // Output final warning and error counts for the table $io->warning(sprintf('Warnings: %d', $warns)); $io->error(sprintf('Errors: %d', $err)); @@ -641,4 +640,88 @@ protected function tableExists(string $tableName): bool $tableList = $dbSchemaManager->listTableNames(); return in_array($tableName, $tableList); } + + /** + * Check whether this row references a rejected record and, if so, mark it rejected and warn. + * This preserves the original self-reference check and adds a generic parent-table check. + * + * Self-reference (original semantics): + * if (cache['rejected'][qualifiedCurrentTable][row[singular(currentTable)_id]]) then skip + * + * Cross-table parent: + * find a *_id in the row that corresponds to a known target table (eg, job_id -> jobs), + * then if (cache['rejected'][qualifiedParentTable][row[parent_fk]]) skip. + * + * @param string $currentTable Logical target table name (eg, 'job_history_records') + * @param array $row Row to insert + * @param \Transmogrify\Lib\Util\CommandLinePrinter $progress Progress printer for warnings + * @return bool True if the row should be skipped, false otherwise + */ + private function skipIfRejectedParent(string $currentTable, array $row, \Transmogrify\Lib\Util\CommandLinePrinter $progress): bool + { + if (!isset($this->cache['rejected'])) { + return false; + } + + // Compute qualified table names once + $qualifiedCurrent = $this->outconn->qualifyTableName($currentTable); + + // 1) Self-reference check (preserves the original semantics) + // With the original code, fkOutboundQualifiedTableName was derived from the table, + // effectively matching "_id". + $selfFk = StringUtilities::classNameToForeignKey($currentTable); + if ( + isset($row[$selfFk]) && + !empty($this->cache['rejected'][$qualifiedCurrent][$row[$selfFk]]) + ) { + $childId = $row['id'] ?? null; + if ($childId !== null) { + $this->cache['rejected'][$qualifiedCurrent][$childId] = $row; + } + $progress->warn(sprintf( + 'Skipping record %d in table %s - parent %s(%d) was rejected (self-reference)', + (int)($childId ?? 0), + $currentTable, + $currentTable, + (int)$row[$selfFk] + )); + return true; + } + + // 2) Cross-table parents: check ALL candidate *_id columns + foreach ($row as $col => $val) { + if ($val === null) { continue; } + if (!is_string($col) || !str_ends_with($col, '_id')) { continue; } + if ($col === 'id' || str_ends_with($col, '_type_id')) { continue; } + + $base = substr($col, 0, -3); + $candidate = Inflector::pluralize(Inflector::underscore($base)); + + // Skip self-table here (already handled) + if ($candidate === $currentTable) { continue; } + + // Only check known target tables + if (!isset($this->tables[$candidate])) { continue; } + + $qualifiedParent = $this->outconn->qualifyTableName($candidate); + $parentId = $val; + + if (!empty($this->cache['rejected'][$qualifiedParent][$parentId])) { + $childId = $row['id'] ?? null; + if ($childId !== null) { + $this->cache['rejected'][$qualifiedCurrent][$childId] = $row; + } + $progress->warn(sprintf( + 'Skipping record %d in table %s - parent %s(%d) was rejected', + (int)($childId ?? 0), + $currentTable, + $candidate, + (int)$parentId + )); + return true; + } + } + + return false; + } } \ No newline at end of file diff --git a/app/plugins/Transmogrify/src/Lib/Traits/CacheTrait.php b/app/plugins/Transmogrify/src/Lib/Traits/CacheTrait.php index cec7f19a5..44ea8934e 100644 --- a/app/plugins/Transmogrify/src/Lib/Traits/CacheTrait.php +++ b/app/plugins/Transmogrify/src/Lib/Traits/CacheTrait.php @@ -104,15 +104,13 @@ protected function findCoId(array $row): int isset($row['group_id']) => $this->getCoIdFromGroupId((int)$row['group_id']), - isset($row['person_role_id']) => $this->getCoIdFromPersonRoleId((int)$row['person_role_id']), - - // Legacy/preRow + // Legacy/preRow: org_identity_id follows the same External Identity path isset($row['org_identity_id']) => $this->getCoIdFromExternalIdentityId((int)$row['org_identity_id']), isset($row['co_person_id']) => $this->getCoIdFromPersonId((int)$row['co_person_id']), isset($row['co_group_id']) => $this->getCoIdFromGroupId((int)$row['co_group_id']), - + isset($row['co_person_role_id']) => $this->getCoIdFromPersonRoleId((int)$row['co_person_role_id']), default => null, @@ -185,15 +183,15 @@ private function getCoIdFromExternalIdentityId(int $externalIdentityId): ?int /** * Resolve a CO ID from a Person Role ID via cache. * - * @param int $personRoleId Person Role ID to lookup + * @param int $personRoleId Person Role ID to resolve * @return int|null CO ID if found, null otherwise * @since COmanage Registry v5.2.0 */ private function getCoIdFromPersonRoleId(int $personRoleId): ?int { if (isset($this->cache['person_roles']['id'][$personRoleId]['person_id'])) { - $peronId = (int)$this->cache['person_roles']['id'][$personRoleId]['person_id']; - return $this->getCoIdFromPersonId($peronId); + $personId = (int)$this->cache['person_roles']['id'][$personRoleId]['person_id']; + return $this->getCoIdFromPersonId($personId); } return null; } diff --git a/app/plugins/Transmogrify/src/Lib/Util/CommandLinePrinter.php b/app/plugins/Transmogrify/src/Lib/Util/CommandLinePrinter.php index 41352174b..0ad16e8d3 100644 --- a/app/plugins/Transmogrify/src/Lib/Util/CommandLinePrinter.php +++ b/app/plugins/Transmogrify/src/Lib/Util/CommandLinePrinter.php @@ -101,6 +101,55 @@ public function message(string $message, string $level = 'info'): void $this->messageLines += substr_count($lines, "\n"); } + /** + * Prompt the user for input while keeping the bar/message layout intact. + * Returns the provided answer (or null on EOF). + */ + public function ask(string $prompt, ?string $default = null): ?string + { + // Ensure the message area exists (one line below the bar) + if ($this->messageLines === 0) { + $this->rawWrite(PHP_EOL); + } + + $answer = null; + + if ($this->io) { + // ConsoleIo handles rendering the prompt and reading input + $answer = $this->io->ask($prompt, $default); + } else { + // Fallback to STDOUT/STDIN + $this->rawWrite($prompt . ' '); + $line = fgets(STDIN); + $answer = ($line === false) ? null : rtrim($line, "\r\n"); + if ($answer === null && $default !== null) { + $answer = $default; + } + // Ensure the cursor advances to the next line after the prompt + $this->rawWrite(PHP_EOL); + } + + // A prompt line was added to the message area + $this->messageLines += 1; + + // Redraw progress bar and return cursor to the end of the message area + $this->rawWrite("\033[u"); // restore to saved bar position + $this->rawWrite("\r" . $this->formatBar($this->current)); + $this->rawWrite("\033[s"); // save bar position again + $this->rawWrite("\033[" . $this->messageLines . "B\r"); // move down to message area + + return $answer; + } + + /** + * Convenience: prompt to continue (ENTER). + */ + public function pause(string $prompt = 'Press to continue...'): void + { + $this->ask($prompt, ''); + } + + private function colorizeLevel(string $level, string $message): string { $level = strtolower($level); diff --git a/app/plugins/Transmogrify/src/Lib/Util/RawSqlQueries.php b/app/plugins/Transmogrify/src/Lib/Util/RawSqlQueries.php index 9d9b08db6..8baeb8f2d 100644 --- a/app/plugins/Transmogrify/src/Lib/Util/RawSqlQueries.php +++ b/app/plugins/Transmogrify/src/Lib/Util/RawSqlQueries.php @@ -180,6 +180,32 @@ public static function couSqlSelect(string $tableName, bool $isMySQL): string { return str_replace('{table}', $tableName, $sqlTemplate); } + /** + * Select history records that are "current" (no changelog link) and whose + * org_identity_id is either NULL or refers to an included Org Identity + * (has a current link to a non-null co_person_id). + * + * @param string $tableName + * @param bool $isMySQL Unused here; kept for consistent signature + * @return string + */ + public static function historyRecordsSqlSelect(string $tableName, bool $isMySQL): string { + return str_replace('{table}', $tableName, RawSqlQueries::HISTORY_RECORDS_SQL_SELECT); + } + + /** + * Select job history records that are "current" (no changelog link) and whose + * org_identity_id is either NULL or refers to an included Org Identity + * (has a current link to a non-null co_person_id). + * + * @param string $tableName + * @param bool $isMySQL Unused here; kept for consistent signature + * @return string + */ + public static function jobHistoryRecordsSqlSelect(string $tableName, bool $isMySQL): string { + return str_replace('{table}', $tableName, RawSqlQueries::HISTORY_RECORDS_SQL_SELECT); + } + /** * Return SQL used to select COUs from inbound database. * @@ -320,11 +346,6 @@ public static function orgidentitiesSqlSelect(string $tableName, bool $isMySQL): // // Unfortunately PostgreSQL and MySQL do not define the same aggregate functions so we need a unique // SQL template for each below. - - /** - * MySQL template for recursive CTE query to select COU records ordered by generation - * Uses GROUP_CONCAT for string aggregation - */ final const COU_SQL_SELECT_TEMPLATE_MYSQL = <<