From cab2db275bf3c5170d37406161451b9b01e1405f Mon Sep 17 00:00:00 2001 From: Scott Koranda Date: Tue, 12 Mar 2024 04:53:35 -0500 Subject: [PATCH] DBAL qualify table names (CFM-356) --- app/src/Command/TransmogrifyCommand.php | 120 ++++++------------ app/src/Lib/Util/DBALConnection.php | 155 ++++++++++++++++++++++++ app/src/Lib/Util/SchemaManager.php | 56 +++------ 3 files changed, 208 insertions(+), 123 deletions(-) create mode 100644 app/src/Lib/Util/DBALConnection.php diff --git a/app/src/Command/TransmogrifyCommand.php b/app/src/Command/TransmogrifyCommand.php index 79d869cbb..07af147f5 100644 --- a/app/src/Command/TransmogrifyCommand.php +++ b/app/src/Command/TransmogrifyCommand.php @@ -39,8 +39,8 @@ use Cake\ORM\TableRegistry; use Cake\Utility\Inflector; use \App\Lib\Util\PaginatedSqlIterator; +use \App\Lib\Util\DBALConnection; -use Doctrine\DBAL\DriverManager; use Doctrine\DBAL\Exception\ForeignKeyConstraintViolationException; class TransmogrifyCommand extends Command { @@ -356,8 +356,6 @@ class TransmogrifyCommand extends Command { // Make some objects more easily accessible protected $inconn = null; protected $outconn = null; - // Cache the driver for ease of workarounds - protected $outdriver = null; // Shell arguments, for easier access protected $args = null; @@ -454,7 +452,9 @@ protected function check_group_membership(array $origRow, array $row) { 'actor_identifier' => $origRow['actor_identifier'] ]; - $this->outconn->insert('group_members', $ownerRow); + $tableName = 'group_members'; + $qualifiedTableName = $this->outconn->qualifyTableName($tableName); + $this->outconn->insert($qualifiedTableName, $ownerRow); } else { $this->io->error("Could not find owners group for CoGroupMember " . $origRow['id']); } @@ -522,74 +522,13 @@ public function execute(Arguments $args, ConsoleIo $io) { // new database. // First, open connections to both old and new databases. - // Use the Cake ConnectionManager to get the database configs to pass to DBAL. - $indb = ConnectionManager::get('transmogrify'); - $incfg = $indb->config(); - - if(empty($incfg)) { - throw new \InvalidArgumentException(__d('error', 'db.config', ["transmogrify"])); - } - - $outdb = ConnectionManager::get('default'); - $outcfg = $outdb->config(); - - if(empty($outcfg)) { - throw new \InvalidArgumentException(__d('error', 'db.config', ["default"])); - } - - $inconfig = new \Doctrine\DBAL\Configuration(); - - $cargs = [ - 'dbname' => $incfg['database'], - 'user' => $incfg['username'], - 'password' => $incfg['password'], - 'host' => $incfg['host'], - 'driver' => ($incfg['driver'] == 'Cake\Database\Driver\Postgres' ? "pdo_pgsql" : "mysqli") - ]; - - // For MySQL SSL - if(!empty($incfg['ssl_ca'])) { - // mysqli supports SSL configuration - $cargs['ssl_ca'] = $incfg['ssl_ca']; - } - - $this->inconn = DriverManager::getConnection($cargs, $inconfig); - - $outconfig = new \Doctrine\DBAL\Configuration(); - - $cargs = [ - 'dbname' => $outcfg['database'], - 'user' => $outcfg['username'], - 'password' => $outcfg['password'], - 'host' => $outcfg['host'], - 'driver' => ($outcfg['driver'] == 'Cake\Database\Driver\Postgres' ? "pdo_pgsql" : "mysqli") - ]; - - // For MySQL SSL - if(!empty($outcfg['ssl_ca'])) { - // mysqli supports SSL configuration - $cargs['ssl_ca'] = $outcfg['ssl_ca']; - } - - $this->outconn = DriverManager::getConnection($cargs, $outconfig); - $this->outdriver = $cargs['driver']; - + $this->inconn = DBALConnection::factory($io, 'transmogrify'); + $this->outconn = DBALConnection::factory($io, 'default'); + // We accept a list of table names, mostly for testing purposes $atables = $args->getArguments(); - $schemaPrefix = ''; - - if($this->outdriver == 'mysqli') { - // We prefix the database to the table to avoid having to quote table names - // that match (MySQL) reserved keywords (in particular "groups"). While - // theoretically Postgres supports the same notation, it seems to cause - // more problems than it solves. - - $schemaPrefix = $outcfg['database'] . '.'; - } - // Register the current version for future upgrade purposes - $targetVersion = rtrim(file_get_contents(CONFIG . DS . "VERSION")); $metaTable = $this->getTableLocator()->get('Meta'); @@ -610,9 +549,10 @@ public function execute(Arguments $args, ConsoleIo $io) { $this->$p(); } - $count = $this->inconn->fetchOne("SELECT COUNT(*) FROM " . $this->tables[$t]['source']); + $qualifiedTableName = $this->inconn->qualifyTableName($this->tables[$t]['source']); + $count = $this->inconn->fetchOne("SELECT COUNT(*) FROM " . $qualifiedTableName); - $insql = "SELECT * FROM " . $this->tables[$t]['source'] . " ORDER BY id ASC"; + $insql = "SELECT * FROM " . $qualifiedTableName . " ORDER BY id ASC"; $stmt = $this->inconn->executeQuery($insql); // Check if the table contains data @@ -650,7 +590,8 @@ public function execute(Arguments $args, ConsoleIo $io) { $this->mapFields($t, $row); - $this->outconn->insert($schemaPrefix.$t, $row); + $qualifiedTableName = $this->outconn->qualifyTableName($t); + $this->outconn->insert($qualifiedTableName, $row); $this->cacheResults($t, $row); @@ -695,17 +636,18 @@ public function execute(Arguments $args, ConsoleIo $io) { $io->out("(Errors: " . $err . ")"); // Reset sequence to next value after current max. - $max = $this->outconn->fetchOne('SELECT MAX(id) FROM ' . $t); + $qualifiedTableName = $this->outconn->qualifyTableName($t); + $max = $this->outconn->fetchOne('SELECT MAX(id) FROM ' . $qualifiedTableName); $max++; - $this->io->info("Resetting sequence for $t to $max"); + $this->io->info("Resetting sequence for $qualifiedTableName to $max"); // Strictly speaking we should use prepared statements, but we control the // data here, and also we're executing a maintenance operation (so query // optimization is less important) - if($this->outdriver == 'mysqli') { - $outsql = "ALTER TABLE `" . $t . "` AUTO_INCREMENT = " . $max; + if($this->outconn->isMySQL()) { + $outsql = "ALTER TABLE $qualifiedTableName AUTO_INCREMENT = " . $max; } else { - $outsql = "ALTER SEQUENCE " . $t . "_id_seq RESTART WITH " . $max; + $outsql = "ALTER SEQUENCE " . $qualifiedTableName . "_id_seq RESTART WITH " . $max; } $this->outconn->executeQuery($outsql); @@ -812,7 +754,7 @@ protected function fixBooleans(string $table, array &$row) { // this issue: https://github.com/doctrine/dbal/issues/1847 // We need to (more generically than this hack) convert from boolean to char // to avoid errors on insert - if($this->outdriver == 'mysqli') { + if($this->outconn->isMySQL()) { $row[$a] = ($row[$a] ? '1' : '0'); } else { $row[$a] = ($row[$a] ? 't' : 'f'); @@ -1068,7 +1010,9 @@ protected function map_login_identifiers(array $origRow, array $row) { $this->fixBooleans('identifiers', $copiedRow); try { - $this->outconn->insert('identifiers', $copiedRow); + $tableName = 'identifiers'; + $qualifiedTableName = $this->outconn->qualifyTableName($tableName); + $this->outconn->insert($qualifiedTableName, $copiedRow); } catch (\Doctrine\DBAL\Exception\UniqueConstraintViolationException $e) { $this->io->warning("record already exists: " . print_r($copiedRow, true)); } @@ -1151,7 +1095,9 @@ protected function map_org_identity_co_person_id(array $row) { $this->io->info('Populating org identity map...'); // We pull deleted rows because we might be migrating deleted rows - $mapsql = "SELECT * FROM cm_co_org_identity_links"; + $tableName = "cm_co_org_identity_links"; + $qualifiedTableName = $this->inconn->qualifyTableName($tableName); + $mapsql = "SELECT * FROM $qualifiedTableName"; $stmt = $this->inconn->query($mapsql); while($r = $stmt->fetch()) { @@ -1254,7 +1200,9 @@ protected function processExtendedAttributes() { // First, pull the old Extended Attribute configuration. $extendedAttrs = []; - $insql = "SELECT * FROM cm_co_extended_attributes ORDER BY id ASC"; + $tableName = "cm_co_extended_attributes"; + $qualifiedTableName = $this->inconn->qualifyTableName($tableName); + $insql = "SELECT * FROM $qualifiedTableName ORDER BY id ASC"; $stmt = $this->inconn->query($insql); while($row = $stmt->fetch()) { @@ -1267,7 +1215,9 @@ protected function processExtendedAttributes() { } foreach(array_keys($extendedAttrs) as $coId) { - $insql = "SELECT * FROM cm_co" . $coId . "_person_extended_attributes"; + $tableName = "cm_co" . $coId . "_person_extended_attributes"; + $qualifiedTableName = $this->inconn->qualifyTableName($tableName); + $insql = "SELECT * FROM $qualifiedTableName"; $stmt = $this->inconn->query($insql); while($eaRow = $stmt->fetch()) { @@ -1288,7 +1238,9 @@ protected function processExtendedAttributes() { $this->fixBooleans('ad_hoc_attributes', $adhocRow); try { - $this->outconn->insert('ad_hoc_attributes', $adhocRow); + $tableName = 'ad_hoc_attributes'; + $qualifiedTableName = $this->outconn->qualifyTableName($tableName); + $this->outconn->insert($qualifiedTableName, $adhocRow); } catch (\Doctrine\DBAL\Exception\UniqueConstraintViolationException $e) { $this->io->warning("record already exists: " . print_r($adhocRow, true)); } @@ -1346,6 +1298,8 @@ protected function split_external_identity(array $origRow, array $row) { // Since we're creating a new row, we have to manually fix up booleans $roleRow['deleted'] = ($roleRow['deleted'] ? 't' : 'f'); - $this->outconn->insert('external_identity_roles', $roleRow); + $tableName = 'external_identity_roles'; + $qualifiedTableName = $this->outconn->qualifyTableName($tableName); + $this->outconn->insert($qualifiedTableName, $roleRow); } } diff --git a/app/src/Lib/Util/DBALConnection.php b/app/src/Lib/Util/DBALConnection.php new file mode 100644 index 000000000..1a3803597 --- /dev/null +++ b/app/src/Lib/Util/DBALConnection.php @@ -0,0 +1,155 @@ +config(); + + // DBAL Configuration instance to pass into DBAL connection factory + $config = new Configuration(); + + // Translate from CakePHP param names to DBAL param names. + $cfargs = [ + 'dbname' => $cfg['database'], + 'user' => $cfg['username'], + 'password' => $cfg['password'], + 'host' => $cfg['host'], + 'driver' => ($cfg['driver'] == 'Cake\Database\Driver\Postgres' ? "pdo_pgsql" : "mysqli") + ]; + + // For MySQL SSL + if(!empty($cfg['ssl_ca'])) { + $cfargs['ssl_ca'] = $cfg['ssl_ca']; + } + + // Signal to DBAL factory to create an instance of this class. + $cfargs['wrapperClass'] = self::class; + + if($io) { + $io->out("Connecting to database " . $cfg['database'] . " as " + . $cfg['username'] . "@" . $cfg['host']); + } + + // Use the DBAL factory to open a connection but return instance + // of this class. + $conn = DriverManager::getConnection($cfargs, $config); + + // The CakePHP database configuration supports using a PostgreSQL schema. If + // defined signal that we should prefix tables with the PostgreSQL schema and a dot '.'. + if(!empty($cfg['schema'])) { + $conn->pgSchema = $cfg['schema']; + } + + if($io) { + $conn->io = $io; + } + $conn->driver = $cfg['driver']; + + return $conn; + } + + /** + * Is the database MySQL or MariaDB? + * + * @since COmanage Registry v5.0.0 + * @return boolean + */ + public function isMySQL() { + return ($this->driver == 'Cake\Database\Driver\Mysql'); + } + + /** + * Is the database PostgreSQL + * + * @since COmanage Registry v5.0.0 + * @return boolean + */ + public function isPostgreSQL() { + return ($this->driver == 'Cake\Database\Driver\Postgres'); + } + + /** + * Qualify database table name. + * + * @since COmanage Registry v5.0.0 + * @param string $tableName Unqualified table name + * @return string Qualified table name + */ + + public function qualifyTableName($tableName) { + switch ($this->driver) { + case 'Cake\Database\Driver\Postgres': + if($this->pgSchema) { + $qualifiedTableName = $this->pgSchema . '.' . $tableName; + } + break; + + case 'Cake\Database\Driver\Mysql': + $qualifiedTableName = $this->getDatabase() . '.' . $tableName; + break; + + default: + $qualifiedTableName = $tableName; + + } + + return $qualifiedTableName; + } +} diff --git a/app/src/Lib/Util/SchemaManager.php b/app/src/Lib/Util/SchemaManager.php index dabc5d8f9..905d9dfaa 100644 --- a/app/src/Lib/Util/SchemaManager.php +++ b/app/src/Lib/Util/SchemaManager.php @@ -30,10 +30,7 @@ namespace App\Lib\Util; use Cake\Console\ConsoleIo; -use Cake\Datasource\ConnectionInterface; -use Cake\Datasource\ConnectionManager; -use Doctrine\DBAL\DriverManager; use Doctrine\DBAL\Schema\Comparator; use Doctrine\DBAL\Schema\Schema; use Doctrine\DBAL\Schema\SchemaDiff; @@ -49,15 +46,12 @@ class SchemaManager { use \App\Lib\Traits\LabeledLogTrait; - // If we're in DatabaseCommand, the Console for output + // Console for output protected $io = null; // The database connection protected $conn = null; - // The database driver in use - protected $driver = null; - // The column library from the main config protected $columnLibrary = null; @@ -70,34 +64,11 @@ class SchemaManager { */ public function __construct(?ConsoleIo $io=null, string $connection='default') { - if($io) $this->io = $io; - - // Use the ConnectionManager to get the database config to pass to DBAL. - $db = ConnectionManager::get($connection); - - // $db is a ConnectionInterface object - $cfg = $db->config(); - - $config = new \Doctrine\DBAL\Configuration(); - - $cfargs = [ - 'dbname' => $cfg['database'], - 'user' => $cfg['username'], - 'password' => $cfg['password'], - 'host' => $cfg['host'], - 'driver' => ($cfg['driver'] == 'Cake\Database\Driver\Postgres' ? "pdo_pgsql" : "mysqli") - ]; - - // For MySQL SSL - if(!empty($cfg['ssl_ca'])) { - $cfargs['ssl_ca'] = $cfg['ssl_ca']; + if($io) { + $this->io = $io; } - - if($this->io) $this->io->out("Connecting to database " . $cfg['database'] . " as " - . $cfg['username'] . "@" . $cfg['host']); - - $this->conn = DriverManager::getConnection($cfargs, $config); - $this->driver = $cfg['driver']; + + $this->conn = DBALConnection::factory($io, $connection); } /** @@ -179,11 +150,12 @@ protected function processSchema( string $tablePrefix="" ) { $schema = new Schema(); - + // Walk through $schemaConfig and build our schema in DBAL format. foreach($schemaConfig->tables as $tName => $tCfg) { - $table = $schema->createTable($tablePrefix.$tName); + $qualifiedTableName = $this->conn->qualifyTableName($tablePrefix.$tName); + $table = $schema->createTable($qualifiedTableName); foreach($tCfg->columns as $cName => $cCfg) { // We allow "inherited" definitions from the fieldLibrary, so merge together @@ -222,7 +194,8 @@ protected function processSchema( } if(isset($colCfg->foreignkey)) { - $table->addForeignKeyConstraint($tablePrefix.$colCfg->foreignkey->table, + $foreignTableName = $this->conn->qualifyTableName($tablePrefix.$colCfg->foreignkey->table); + $table->addForeignKeyConstraint($foreignTableName, [$cName], [$colCfg->foreignkey->column], [], @@ -242,10 +215,11 @@ protected function processSchema( foreach($tCfg->mvea as $m) { $mColumn = $m . "_id"; $fkTable = \Cake\Utility\Inflector::tableize($m); + $foreignTableName = $this->conn->qualifyTableName($tablePrefix.$fkTable); // Insert a foreign key to this model and index it $table->addColumn($mColumn, "integer", ['notnull' => false]); - $table->addForeignKeyConstraint($tablePrefix.$fkTable, [$mColumn], ['id'], [], $tablePrefix.$tName . "_" . $mColumn . "_fkey"); + $table->addForeignKeyConstraint($foreignTableName, [$mColumn], ['id'], [], $tablePrefix.$tName . "_" . $mColumn . "_fkey"); $table->addIndex([$mColumn], $tablePrefix.$tName . "_im" . $i++); } @@ -279,10 +253,11 @@ protected function processSchema( if(isset($tCfg->sourced) && $tCfg->sourced) { $sColumn = "source_" . $tablePrefix.\Cake\Utility\Inflector::singularize($tName) . "_id"; + $foreignTableName = $this->conn->qualifyTableName($tablePrefix.$tName); // Insert a foreign key to this model and index it $table->addColumn($sColumn, "integer", ['notnull' => false]); - $table->addForeignKeyConstraint($tablePrefix.$tName, [$sColumn], ['id'], [], $tablePrefix.$tName . "_" . $sColumn . "_fkey"); + $table->addForeignKeyConstraint($foreignTableName, [$sColumn], ['id'], [], $tablePrefix.$tName . "_" . $sColumn . "_fkey"); $table->addIndex([$sColumn], $tablePrefix.$tName . "_im" . $i++); } @@ -335,7 +310,7 @@ protected function processSchema( foreach($diffSql as $sql) { if($this->io) $this->io->out($sql); - if($this->driver == 'Cake\Database\Driver\Postgres' + if($this->conn->driver == 'Cake\Database\Driver\Postgres' && preg_match("/^DROP SEQUENCE [a-z]*_id_seq/", $sql)) { // Remove the DROP SEQUENCE statements in $fromSql because they're Postgres automagic // being misinterpreted. (Note toSaveSql might mask this now.) @@ -360,4 +335,5 @@ protected function processSchema( // bin/cake schema_cache build --connection default // but so far we don't have an example indicating it's needed. } + } \ No newline at end of file