Skip to content
Jump to bottom

[NO JIRA] Transmogrification_improvements #13

Merged
merged 1 commit into from
Apr 27, 2022
+73 −19
Merged

[NO JIRA] Transmogrification_improvements #13

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions app/resources/locales/en_US/error.po
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,9 @@ msgstr "COU Parent ID not valid"
msgid "db.config"
msgstr "Invalid database configuration \"{0}\""

msgid "default.conflict"
msgstr "Default values already imported"

msgid "delete.active"
msgstr "This record is in Active status and cannot be deleted"

Expand Down
63 changes: 45 additions & 18 deletions app/src/Command/TransmogrifyCommand.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,8 @@
use Doctrine\DBAL\Exception\ForeignKeyConstraintViolationException;

class TransmogrifyCommand extends Command {
use \App\Lib\Traits\LabeledLogTrait;

// Tables must be listed in order of primary key dependencies.
protected $tables = [
'cos' => [
Expand Down Expand Up @@ -263,6 +265,7 @@ class TransmogrifyCommand extends Command {
'co_person_id' => 'person_id',
'org_identity_id' => 'external_identity_id',
'actor_co_person_id' => 'actor_person_id',
'actor_api_user_id' => 'actor_api_user_id',
'co_person_role_id' => 'person_role_id',
// XXX temporary until tables are migrated
'co_group_id' => null,
Expand Down Expand Up @@ -389,7 +392,7 @@ public function execute(Arguments $args, ConsoleIo $io) {
if(!empty($atables) && !in_array($t, $atables))
continue;

$io->out("===" . $t . "===");
$io->info(Inflector::classify($t) . "(" . $t . ")");

// Run any pre processing functions for the table.

Expand All @@ -401,18 +404,27 @@ public function execute(Arguments $args, ConsoleIo $io) {

$count = $this->inconn->fetchOne("SELECT COUNT(*) FROM " . $this->tables[$t]['source']);

$io->out("= Processing " . $count . " records");

$insql = "SELECT * FROM " . $this->tables[$t]['source'] . " ORDER BY id ASC";
$stmt = $this->inconn->query($insql);

// Check if the table contains data
$Model = $this->getTableLocator()->get($t);
if($Model->find()->count() > 0) {
$io->warning("Skipping Transmogrification. Table is not empty. Drop the database (or truncate) and start over.");
$this->llog("debug", "WARNING: Skipping Transmogrification. Table(" . $t . ") is not empty. Drop the database (or truncate) and start over.");
continue;
}

$tally = 0;
$warns = 0;
$err = 0;

while($row = $stmt->fetch()) {
if(!empty($row[ $this->tables[$t]['displayField'] ])) {
$io->out("$t " . $row[ $this->tables[$t]['displayField'] ] . "...", 0);
// Use this in the message. Modify last
$this->llog("debug", "$t " . $row[ $this->tables[$t]['displayField'] ]);
}

try {
// Make a copy of the original data for any post processing followups
$origRow = $row;
Expand All @@ -423,7 +435,7 @@ public function execute(Arguments $args, ConsoleIo $io) {
$this->fixBooleans($t, $row);

$this->mapFields($t, $row);

$this->outconn->insert($t, $row);

$this->cacheResults($t, $row);
Expand All @@ -441,27 +453,35 @@ public function execute(Arguments $args, ConsoleIo $io) {
// load this record. This can happen, eg, because the source_field_id
// did not load, perhaps because it was associated with an Org Identity
// not linked to a CO Person that was not migrated.

$io->err("WARNING: Skipping record " . $row['id'] . " due to invalid foreign key: " . $e->getMessage());
$warns++;
$this->llog("debug", "WARNING: Skipping record " . $row['id'] . " due to invalid foreign key: " . $e->getMessage());
}
catch(\InvalidArgumentException $e) {
// If we can't find a value for mapping we skip the record
// (ie: mapFields basically requires a successful mapping)

$io->err("WARNING: Skipping record " . $row['id'] . ": " . $e->getMessage());
$warns++;
$this->llog("debug", "WARNING: Skipping record " . $row['id'] . ": " . $e->getMessage());
}
catch(\Exception $e) {
$io->err("ERROR: Record " . $row['id'] . ": " . $e->getMessage());
$err++;
$this->llog("debug", "ERROR: Record " . $row['id'] . ": " . $e->getMessage());
}

$tally++;
$io->out(floor(($tally * 100)/$count) . "% done");
$this->cliLogPercentage($tally, $count);
}

$max = $this->inconn->fetchOne('SELECT MAX(id) FROM ' . $this->tables[$t]['source']);
$max++;

$io->out("= New max: " . $max);
$stdout_msg = "(New max: " . $max . ")";
if($warns > 0) {
$stdout_msg .= "<warning>(Warnings: " . $warns . ")</warning>";
}
if($err > 0) {
$stdout_msg .= "<error>(Errors: " . $err . ")</error>";
}

$io->out($stdout_msg);

// Strictly speaking we should use prepared statements, but we control the
// data here, and also we're executing a maintenance operation (so query
Expand Down Expand Up @@ -598,8 +618,11 @@ protected function insertDefaultSettings() {

foreach($emptySettings as $coId) {
// Insert a default row into CoSettings for this CO ID

$CoSettings->addDefaults($coId);
try {
$CoSettings->addDefaults($coId);
} catch (\ConflictException $e) {
// skip
}
}
}
}
Expand Down Expand Up @@ -905,8 +928,12 @@ protected function processExtendedAttributes() {
// Extended Attributes were not changelog enabled
$this->fixChangelog('ad_hoc_attributes', $adhocRow, true);
$this->fixBooleans('ad_hoc_attributes', $adhocRow);

$this->outconn->insert('ad_hoc_attributes', $adhocRow);

try {
$this->outconn->insert('ad_hoc_attributes', $adhocRow);
} catch (\Doctrine\DBAL\Exception\UniqueConstraintViolationException $e) {
$this->llog("debug", "record already exists: " . print_r($adhocRow, true));
}
}
}
}
Expand Down
16 changes: 16 additions & 0 deletions app/src/Lib/Traits/LabeledLogTrait.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,4 +65,20 @@ public function llog(string $level, string $msg) {

Log::write($level, $m);
}

/**
* Print formatted cli percentage
*
* @since COmanage Registry v5.0.0
* @param int $done Number of iterations completed
* @param string $total Total number of iterations
* @return string Formated string with line return offset
*/

public function cliLogPercentage(int $done, int $total): void {
$perc = floor(($done / $total) * 100);
$left = 100 - $perc;
$out = sprintf("\033[0G\033[2K[%'={$perc}s>%-{$left}s] - $perc%% -- $done/$total", "", "");
fwrite(STDERR, $out);
}
}
10 changes: 9 additions & 1 deletion app/src/Model/Table/CoSettingsTable.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -173,6 +173,7 @@ public function initialize(array $config): void {
* @since COmanage Registry v5.0.0
* @param int $coId CO ID
* @return int CoSettings ID
* @throws ConflictException when default values exist
*/

public function addDefaults(int $coId): int {
Expand Down Expand Up @@ -210,7 +211,14 @@ public function addDefaults(int $coId): int {
// 'co_theme_id' => null,
// 'global_search_limit' => DEF_GLOBAL_SEARCH_LIMIT
];


// Check if we already have Settings for this CO
$settings = $this->find()->where([ 'co_id' => $defaultSettings['co_id'] ])->first();
// If the record already exists throw an exception
if(!empty($settings->{'id'})) {
throw new \ConflictException(__d('error', 'default.conflict'));
}

$obj = $this->newEntity($defaultSettings);

$this->save($obj);
Expand Down