Job history records+servers

app/plugins/Transmogrify/README.md

@@ -13,7 +13,10 @@ It is designed to be:
   - Source DB (used by Transmogrify) and target DB must both be reachable. Transmogrify initializes two Doctrine DBAL connections internally.
 - The default tables mapping file is at:
   - app/plugins/Transmogrify/config/schema/tables.json
-
+- Actions
+  - Finalize any pending Jobs. Jobs in Queued or  Progress state will be skipped.
+  - Restore extended type defaults
+  - Run the health checks.
 
 ## Command
 Invoke from your app root:

app/plugins/Transmogrify/config/schema/tables.json

@@ -80,7 +80,8 @@
             "person_picker_email_type": null,
             "person_picker_identifier_type": null,
             "person_picker_display_types": null,
-            "group_create_admin_only": null
+            "group_create_admin_only": null,
+            "t_and_c_return_url_allowlist": null
         }
     },
     "authentication_events": {
@@ -283,6 +284,7 @@
     "history_records": {
         "source": "cm_history_records",
         "displayField": "id",
+        "sqlSelect": "historyRecordsSqlSelect",
         "fieldMap": {
             "actor_co_person_id": "actor_person_id",
             "co_person_id": "person_id",
@@ -312,6 +314,7 @@
     "job_history_records": {
         "source": "cm_co_job_history_records",
         "displayField": "id",
+        "sqlSelect": "jobHistoryRecordsSqlSelect",
         "fieldMap": {
             "co_job_id": "job_id",
             "co_person_id": "person_id",
@@ -324,7 +327,8 @@
         "addChangelog": false,
         "cache": ["co_id", "status"],
         "fieldMap": {
-            "plugin": "&mapServerTypeToPlugin"
+            "plugin": "&mapServerTypeToPlugin",
+            "server_type": null
         }
     }
 }

app/plugins/Transmogrify/src/Command/TransmogrifyCommand.php

@@ -341,6 +341,9 @@ public function execute(Arguments $args, ConsoleIo $io): int
       // Fetch the inbound data.
       $stmt = $this->inconn->executeQuery($insql);
 
+      /*
+       * PROGRESS STARTING
+       **/
       $progress = new CommandLinePrinter($io, 'green', 50, true);
       // If a custom SELECT is used, count the exact result set; otherwise count the whole table
       if (!empty($this->tables[$t]['sqlSelect'])) {
@@ -384,21 +387,11 @@ public function execute(Arguments $args, ConsoleIo $io): int
 
           // Insert the transformed row into the target database
           if($this->cache['skipInsert'][$outboundQualifiedTableName] === false) {
-            $fkOutboundQualifiedTableName = StringUtilities::classNameToForeignKey($outboundQualifiedTableName);
-            if (
-              isset($this->cache['rejected'])
-              && !empty($this->cache['rejected'][$outboundQualifiedTableName][$row[$fkOutboundQualifiedTableName]])
-            ) {
-              // This row will be rejected because it references a parent record that does not exist.
-              // The parent record has been rejected before, so we can't insert this record.
-              $this->cache['rejected'][$outboundQualifiedTableName][$row['id']] = $row;
-              $io->warning(sprintf(
-                'Skipping record %d in table %s - parent record does not exist',
-                $row['id'] ?? 0,
-                $t
-              ));
+            // Check if a parent record for this row was previously rejected; if so, skip this insert
+            if ($this->skipIfRejectedParent(currentTable: $t, row: $row, progress: $progress)) {
               continue;
             }
+
             $this->outconn->insert($outboundQualifiedTableName, $row);
             // Execute any post-processing hooks after successful insertion
             $this->runPostRowHook($t, $origRow, $row);
@@ -412,20 +405,22 @@ public function execute(Arguments $args, ConsoleIo $io): int
           // did not load, perhaps because it was associated with an Org Identity
           // not linked to a CO Person that was not migrated.
           $warns++;
+          $this->cache['rejected'][$outboundQualifiedTableName][$row['id']] = $row;
           $progress->warn("Skipping $t record " . $row['id'] . " due to invalid foreign key: " . $e->getMessage());
           $io->ask('Press <enter> to continue...');
         }
         catch(\InvalidArgumentException $e) {
           // If we can't find a value for mapping we skip the record
           // (ie: mapLegacyFieldNames basically requires a successful mapping)
           $warns++;
+          $this->cache['rejected'][$outboundQualifiedTableName][$row['id']] = $row;
           $progress->warn("Skipping $t record " . $row['id'] . ": " . $e->getMessage());
           $io->ask('Press <enter> to continue...');
         }
         catch(\Exception $e) {
           $err++;
           $progress->error("$t record " . $row['id'] . ": " . $e->getMessage());
-          $io->ask('Press <enter> to continue...');
+          $progress->ask('Press <enter> to continue...');
         }
 
         $tally++;
@@ -436,6 +431,10 @@ public function execute(Arguments $args, ConsoleIo $io): int
       }
 
       $progress->finish();
+      /**
+       * FINISH PROGRESS
+       */
+
       // Output final warning and error counts for the table
       $io->warning(sprintf('Warnings: %d', $warns));
       $io->error(sprintf('Errors: %d', $err));
@@ -641,4 +640,88 @@ protected function tableExists(string $tableName): bool
     $tableList = $dbSchemaManager->listTableNames();
     return in_array($tableName, $tableList);
   }
+
+  /**
+   * Check whether this row references a rejected record and, if so, mark it rejected and warn.
+   * This preserves the original self-reference check and adds a generic parent-table check.
+   *
+   * Self-reference (original semantics):
+   *   if (cache['rejected'][qualifiedCurrentTable][row[singular(currentTable)_id]]) then skip
+   *
+   * Cross-table parent:
+   *   find a *_id in the row that corresponds to a known target table (eg, job_id -> jobs),
+   *   then if (cache['rejected'][qualifiedParentTable][row[parent_fk]]) skip.
+   *
+   * @param string              $currentTable Logical target table name (eg, 'job_history_records')
+   * @param array               $row          Row to insert
+   * @param \Transmogrify\Lib\Util\CommandLinePrinter $progress Progress printer for warnings
+   * @return bool                             True if the row should be skipped, false otherwise
+   */
+  private function skipIfRejectedParent(string $currentTable, array $row, \Transmogrify\Lib\Util\CommandLinePrinter $progress): bool
+  {
+    if (!isset($this->cache['rejected'])) {
+      return false;
+    }
+
+    // Compute qualified table names once
+    $qualifiedCurrent = $this->outconn->qualifyTableName($currentTable);
+
+    // 1) Self-reference check (preserves the original semantics)
+    //    With the original code, fkOutboundQualifiedTableName was derived from the table,
+    //    effectively matching "<singular(currentTable)>_id".
+    $selfFk = StringUtilities::classNameToForeignKey($currentTable);
+    if (
+      isset($row[$selfFk]) &&
+      !empty($this->cache['rejected'][$qualifiedCurrent][$row[$selfFk]])
+    ) {
+      $childId = $row['id'] ?? null;
+      if ($childId !== null) {
+        $this->cache['rejected'][$qualifiedCurrent][$childId] = $row;
+      }
+      $progress->warn(sprintf(
+        'Skipping record %d in table %s - parent %s(%d) was rejected (self-reference)',
+        (int)($childId ?? 0),
+        $currentTable,
+        $currentTable,
+        (int)$row[$selfFk]
+      ));
+      return true;
+    }
+
+    // 2) Cross-table parents: check ALL candidate *_id columns
+    foreach ($row as $col => $val) {
+      if ($val === null) { continue; }
+      if (!is_string($col) || !str_ends_with($col, '_id')) { continue; }
+      if ($col === 'id' || str_ends_with($col, '_type_id')) { continue; }
+
+      $base = substr($col, 0, -3);
+      $candidate = Inflector::pluralize(Inflector::underscore($base));
+
+      // Skip self-table here (already handled)
+      if ($candidate === $currentTable) { continue; }
+
+      // Only check known target tables
+      if (!isset($this->tables[$candidate])) { continue; }
+
+      $qualifiedParent = $this->outconn->qualifyTableName($candidate);
+      $parentId = $val;
+
+      if (!empty($this->cache['rejected'][$qualifiedParent][$parentId])) {
+        $childId = $row['id'] ?? null;
+        if ($childId !== null) {
+          $this->cache['rejected'][$qualifiedCurrent][$childId] = $row;
+        }
+        $progress->warn(sprintf(
+          'Skipping record %d in table %s - parent %s(%d) was rejected',
+          (int)($childId ?? 0),
+          $currentTable,
+          $candidate,
+          (int)$parentId
+        ));
+        return true;
+      }
+    }
+
+    return false;
+  }
 }

app/plugins/Transmogrify/src/Lib/Traits/CacheTrait.php

@@ -104,15 +104,13 @@ protected function findCoId(array $row): int
 
       isset($row['group_id']) => $this->getCoIdFromGroupId((int)$row['group_id']),
 
-      isset($row['person_role_id']) => $this->getCoIdFromPersonRoleId((int)$row['person_role_id']),
-
-      // Legacy/preRow
+      // Legacy/preRow: org_identity_id follows the same External Identity path
       isset($row['org_identity_id']) => $this->getCoIdFromExternalIdentityId((int)$row['org_identity_id']),
 
       isset($row['co_person_id']) => $this->getCoIdFromPersonId((int)$row['co_person_id']),
 
       isset($row['co_group_id']) => $this->getCoIdFromGroupId((int)$row['co_group_id']),
-
+      
       isset($row['co_person_role_id']) => $this->getCoIdFromPersonRoleId((int)$row['co_person_role_id']),
 
       default => null,
@@ -185,15 +183,15 @@ private function getCoIdFromExternalIdentityId(int $externalIdentityId): ?int
   /**
    * Resolve a CO ID from a Person Role ID via cache.
    *
-   * @param int $personRoleId Person Role ID to lookup
+   * @param int $personRoleId Person Role ID to resolve
    * @return int|null CO ID if found, null otherwise
    * @since COmanage Registry v5.2.0
    */
   private function getCoIdFromPersonRoleId(int $personRoleId): ?int
   {
     if (isset($this->cache['person_roles']['id'][$personRoleId]['person_id'])) {
-      $peronId = (int)$this->cache['person_roles']['id'][$personRoleId]['person_id'];
-      return $this->getCoIdFromPersonId($peronId);
+      $personId = (int)$this->cache['person_roles']['id'][$personRoleId]['person_id'];
+      return $this->getCoIdFromPersonId($personId);
     }
     return null;
   }

app/plugins/Transmogrify/src/Lib/Util/CommandLinePrinter.php

@@ -101,6 +101,55 @@ public function message(string $message, string $level = 'info'): void
     $this->messageLines += substr_count($lines, "\n");
   }
 
+  /**
+   * Prompt the user for input while keeping the bar/message layout intact.
+   * Returns the provided answer (or null on EOF).
+   */
+  public function ask(string $prompt, ?string $default = null): ?string
+  {
+    // Ensure the message area exists (one line below the bar)
+    if ($this->messageLines === 0) {
+      $this->rawWrite(PHP_EOL);
+    }
+
+    $answer = null;
+
+    if ($this->io) {
+      // ConsoleIo handles rendering the prompt and reading input
+      $answer = $this->io->ask($prompt, $default);
+    } else {
+      // Fallback to STDOUT/STDIN
+      $this->rawWrite($prompt . ' ');
+      $line = fgets(STDIN);
+      $answer = ($line === false) ? null : rtrim($line, "\r\n");
+      if ($answer === null && $default !== null) {
+        $answer = $default;
+      }
+      // Ensure the cursor advances to the next line after the prompt
+      $this->rawWrite(PHP_EOL);
+    }
+
+    // A prompt line was added to the message area
+    $this->messageLines += 1;
+
+    // Redraw progress bar and return cursor to the end of the message area
+    $this->rawWrite("\033[u");                     // restore to saved bar position
+    $this->rawWrite("\r" . $this->formatBar($this->current));
+    $this->rawWrite("\033[s");                     // save bar position again
+    $this->rawWrite("\033[" . $this->messageLines . "B\r"); // move down to message area
+
+    return $answer;
+  }
+
+  /**
+   * Convenience: prompt to continue (ENTER).
+   */
+  public function pause(string $prompt = 'Press <enter> to continue...'): void
+  {
+    $this->ask($prompt, '');
+  }
+
+
   private function colorizeLevel(string $level, string $message): string
   {
     $level = strtolower($level);